drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37
  38 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  39 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  40 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  42 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  43                                              int write);
  44 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  45                                                      uint64_t offset,
  46                                                      uint64_t size);
  47 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  48 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  49 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  50                                            unsigned alignment);
  51 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  52 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  53                                 struct drm_i915_gem_pwrite *args,
  54                                 struct drm_file *file_priv);
  55 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  56
  57 static LIST_HEAD(shrink_list);
  58 static DEFINE_SPINLOCK(shrink_list_lock);
  59
  60 static inline bool
  61 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  62 {
  63         return obj_priv->gtt_space &&
  64                 !obj_priv->active &&
  65                 obj_priv->pin_count == 0;
  66 }
  67
  68 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  69                      unsigned long end)
  70 {
  71         drm_i915_private_t *dev_priv = dev->dev_private;
  72
  73         if (start >= end ||
  74             (start & (PAGE_SIZE - 1)) != 0 ||
  75             (end & (PAGE_SIZE - 1)) != 0) {
  76                 return -EINVAL;
  77         }
  78
  79         drm_mm_init(&dev_priv->mm.gtt_space, start,
  80                     end - start);
  81
  82         dev->gtt_total = (uint32_t) (end - start);
  83
  84         return 0;
  85 }
  86
  87 int
  88 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  89                     struct drm_file *file_priv)
  90 {
  91         struct drm_i915_gem_init *args = data;
  92         int ret;
  93
  94         mutex_lock(&dev->struct_mutex);
  95         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  96         mutex_unlock(&dev->struct_mutex);
  97
  98         return ret;
  99 }
 100
 101 int
 102 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 103                             struct drm_file *file_priv)
 104 {
 105         struct drm_i915_gem_get_aperture *args = data;
 106
 107         if (!(dev->driver->driver_features & DRIVER_GEM))
 108                 return -ENODEV;
 109
 110         args->aper_size = dev->gtt_total;
 111         args->aper_available_size = (args->aper_size -
 112                                      atomic_read(&dev->pin_memory));
 113
 114         return 0;
 115 }
 116
 117
 118 /**
 119  * Creates a new mm object and returns a handle to it.
 120  */
 121 int
 122 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 123                       struct drm_file *file_priv)
 124 {
 125         struct drm_i915_gem_create *args = data;
 126         struct drm_gem_object *obj;
 127         int ret;
 128         u32 handle;
 129
 130         args->size = roundup(args->size, PAGE_SIZE);
 131
 132         /* Allocate the new object */
 133         obj = i915_gem_alloc_object(dev, args->size);
 134         if (obj == NULL)
 135                 return -ENOMEM;
 136
 137         ret = drm_gem_handle_create(file_priv, obj, &handle);
 138         if (ret) {
 139                 drm_gem_object_unreference_unlocked(obj);
 140                 return ret;
 141         }
 142
 143         /* Sink the floating reference from kref_init(handlecount) */
 144         drm_gem_object_handle_unreference_unlocked(obj);
 145
 146         args->handle = handle;
 147         return 0;
 148 }
 149
 150 static inline int
 151 fast_shmem_read(struct page **pages,
 152                 loff_t page_base, int page_offset,
 153                 char __user *data,
 154                 int length)
 155 {
 156         char __iomem *vaddr;
 157         int unwritten;
 158
 159         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 160         if (vaddr == NULL)
 161                 return -ENOMEM;
 162         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 163         kunmap_atomic(vaddr, KM_USER0);
 164
 165         if (unwritten)
 166                 return -EFAULT;
 167
 168         return 0;
 169 }
 170
 171 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 172 {
 173         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 174         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 175
 176         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 177                 obj_priv->tiling_mode != I915_TILING_NONE;
 178 }
 179
 180 static inline void
 181 slow_shmem_copy(struct page *dst_page,
 182                 int dst_offset,
 183                 struct page *src_page,
 184                 int src_offset,
 185                 int length)
 186 {
 187         char *dst_vaddr, *src_vaddr;
 188
 189         dst_vaddr = kmap(dst_page);
 190         src_vaddr = kmap(src_page);
 191
 192         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 193
 194         kunmap(src_page);
 195         kunmap(dst_page);
 196 }
 197
 198 static inline void
 199 slow_shmem_bit17_copy(struct page *gpu_page,
 200                       int gpu_offset,
 201                       struct page *cpu_page,
 202                       int cpu_offset,
 203                       int length,
 204                       int is_read)
 205 {
 206         char *gpu_vaddr, *cpu_vaddr;
 207
 208         /* Use the unswizzled path if this page isn't affected. */
 209         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 210                 if (is_read)
 211                         return slow_shmem_copy(cpu_page, cpu_offset,
 212                                                gpu_page, gpu_offset, length);
 213                 else
 214                         return slow_shmem_copy(gpu_page, gpu_offset,
 215                                                cpu_page, cpu_offset, length);
 216         }
 217
 218         gpu_vaddr = kmap(gpu_page);
 219         cpu_vaddr = kmap(cpu_page);
 220
 221         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 222          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 223          */
 224         while (length > 0) {
 225                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 226                 int this_length = min(cacheline_end - gpu_offset, length);
 227                 int swizzled_gpu_offset = gpu_offset ^ 64;
 228
 229                 if (is_read) {
 230                         memcpy(cpu_vaddr + cpu_offset,
 231                                gpu_vaddr + swizzled_gpu_offset,
 232                                this_length);
 233                 } else {
 234                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 235                                cpu_vaddr + cpu_offset,
 236                                this_length);
 237                 }
 238                 cpu_offset += this_length;
 239                 gpu_offset += this_length;
 240                 length -= this_length;
 241         }
 242
 243         kunmap(cpu_page);
 244         kunmap(gpu_page);
 245 }
 246
 247 /**
 248  * This is the fast shmem pread path, which attempts to copy_from_user directly
 249  * from the backing pages of the object to the user's address space.  On a
 250  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 251  */
 252 static int
 253 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 254                           struct drm_i915_gem_pread *args,
 255                           struct drm_file *file_priv)
 256 {
 257         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 258         ssize_t remain;
 259         loff_t offset, page_base;
 260         char __user *user_data;
 261         int page_offset, page_length;
 262         int ret;
 263
 264         user_data = (char __user *) (uintptr_t) args->data_ptr;
 265         remain = args->size;
 266
 267         mutex_lock(&dev->struct_mutex);
 268
 269         ret = i915_gem_object_get_pages(obj, 0);
 270         if (ret != 0)
 271                 goto fail_unlock;
 272
 273         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 274                                                         args->size);
 275         if (ret != 0)
 276                 goto fail_put_pages;
 277
 278         obj_priv = to_intel_bo(obj);
 279         offset = args->offset;
 280
 281         while (remain > 0) {
 282                 /* Operation in this page
 283                  *
 284                  * page_base = page offset within aperture
 285                  * page_offset = offset within page
 286                  * page_length = bytes to copy for this page
 287                  */
 288                 page_base = (offset & ~(PAGE_SIZE-1));
 289                 page_offset = offset & (PAGE_SIZE-1);
 290                 page_length = remain;
 291                 if ((page_offset + remain) > PAGE_SIZE)
 292                         page_length = PAGE_SIZE - page_offset;
 293
 294                 ret = fast_shmem_read(obj_priv->pages,
 295                                       page_base, page_offset,
 296                                       user_data, page_length);
 297                 if (ret)
 298                         goto fail_put_pages;
 299
 300                 remain -= page_length;
 301                 user_data += page_length;
 302                 offset += page_length;
 303         }
 304
 305 fail_put_pages:
 306         i915_gem_object_put_pages(obj);
 307 fail_unlock:
 308         mutex_unlock(&dev->struct_mutex);
 309
 310         return ret;
 311 }
 312
 313 static int
 314 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 315 {
 316         int ret;
 317
 318         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 319
 320         /* If we've insufficient memory to map in the pages, attempt
 321          * to make some space by throwing out some old buffers.
 322          */
 323         if (ret == -ENOMEM) {
 324                 struct drm_device *dev = obj->dev;
 325
 326                 ret = i915_gem_evict_something(dev, obj->size,
 327                                                i915_gem_get_gtt_alignment(obj));
 328                 if (ret)
 329                         return ret;
 330
 331                 ret = i915_gem_object_get_pages(obj, 0);
 332         }
 333
 334         return ret;
 335 }
 336
 337 /**
 338  * This is the fallback shmem pread path, which allocates temporary storage
 339  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 340  * can copy out of the object's backing pages while holding the struct mutex
 341  * and not take page faults.
 342  */
 343 static int
 344 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 345                           struct drm_i915_gem_pread *args,
 346                           struct drm_file *file_priv)
 347 {
 348         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 349         struct mm_struct *mm = current->mm;
 350         struct page **user_pages;
 351         ssize_t remain;
 352         loff_t offset, pinned_pages, i;
 353         loff_t first_data_page, last_data_page, num_pages;
 354         int shmem_page_index, shmem_page_offset;
 355         int data_page_index,  data_page_offset;
 356         int page_length;
 357         int ret;
 358         uint64_t data_ptr = args->data_ptr;
 359         int do_bit17_swizzling;
 360
 361         remain = args->size;
 362
 363         /* Pin the user pages containing the data.  We can't fault while
 364          * holding the struct mutex, yet we want to hold it while
 365          * dereferencing the user data.
 366          */
 367         first_data_page = data_ptr / PAGE_SIZE;
 368         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 369         num_pages = last_data_page - first_data_page + 1;
 370
 371         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 372         if (user_pages == NULL)
 373                 return -ENOMEM;
 374
 375         down_read(&mm->mmap_sem);
 376         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 377                                       num_pages, 1, 0, user_pages, NULL);
 378         up_read(&mm->mmap_sem);
 379         if (pinned_pages < num_pages) {
 380                 ret = -EFAULT;
 381                 goto fail_put_user_pages;
 382         }
 383
 384         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 385
 386         mutex_lock(&dev->struct_mutex);
 387
 388         ret = i915_gem_object_get_pages_or_evict(obj);
 389         if (ret)
 390                 goto fail_unlock;
 391
 392         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 393                                                         args->size);
 394         if (ret != 0)
 395                 goto fail_put_pages;
 396
 397         obj_priv = to_intel_bo(obj);
 398         offset = args->offset;
 399
 400         while (remain > 0) {
 401                 /* Operation in this page
 402                  *
 403                  * shmem_page_index = page number within shmem file
 404                  * shmem_page_offset = offset within page in shmem file
 405                  * data_page_index = page number in get_user_pages return
 406                  * data_page_offset = offset with data_page_index page.
 407                  * page_length = bytes to copy for this page
 408                  */
 409                 shmem_page_index = offset / PAGE_SIZE;
 410                 shmem_page_offset = offset & ~PAGE_MASK;
 411                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 412                 data_page_offset = data_ptr & ~PAGE_MASK;
 413
 414                 page_length = remain;
 415                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 416                         page_length = PAGE_SIZE - shmem_page_offset;
 417                 if ((data_page_offset + page_length) > PAGE_SIZE)
 418                         page_length = PAGE_SIZE - data_page_offset;
 419
 420                 if (do_bit17_swizzling) {
 421                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 422                                               shmem_page_offset,
 423                                               user_pages[data_page_index],
 424                                               data_page_offset,
 425                                               page_length,
 426                                               1);
 427                 } else {
 428                         slow_shmem_copy(user_pages[data_page_index],
 429                                         data_page_offset,
 430                                         obj_priv->pages[shmem_page_index],
 431                                         shmem_page_offset,
 432                                         page_length);
 433                 }
 434
 435                 remain -= page_length;
 436                 data_ptr += page_length;
 437                 offset += page_length;
 438         }
 439
 440 fail_put_pages:
 441         i915_gem_object_put_pages(obj);
 442 fail_unlock:
 443         mutex_unlock(&dev->struct_mutex);
 444 fail_put_user_pages:
 445         for (i = 0; i < pinned_pages; i++) {
 446                 SetPageDirty(user_pages[i]);
 447                 page_cache_release(user_pages[i]);
 448         }
 449         drm_free_large(user_pages);
 450
 451         return ret;
 452 }
 453
 454 /**
 455  * Reads data from the object referenced by handle.
 456  *
 457  * On error, the contents of *data are undefined.
 458  */
 459 int
 460 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 461                      struct drm_file *file_priv)
 462 {
 463         struct drm_i915_gem_pread *args = data;
 464         struct drm_gem_object *obj;
 465         struct drm_i915_gem_object *obj_priv;
 466         int ret;
 467
 468         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 469         if (obj == NULL)
 470                 return -ENOENT;
 471         obj_priv = to_intel_bo(obj);
 472
 473         /* Bounds check source.
 474          *
 475          * XXX: This could use review for overflow issues...
 476          */
 477         if (args->offset > obj->size || args->size > obj->size ||
 478             args->offset + args->size > obj->size) {
 479                 drm_gem_object_unreference_unlocked(obj);
 480                 return -EINVAL;
 481         }
 482
 483         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 484                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 485         } else {
 486                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 487                 if (ret != 0)
 488                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 489                                                         file_priv);
 490         }
 491
 492         drm_gem_object_unreference_unlocked(obj);
 493
 494         return ret;
 495 }
 496
 497 /* This is the fast write path which cannot handle
 498  * page faults in the source data
 499  */
 500
 501 static inline int
 502 fast_user_write(struct io_mapping *mapping,
 503                 loff_t page_base, int page_offset,
 504                 char __user *user_data,
 505                 int length)
 506 {
 507         char *vaddr_atomic;
 508         unsigned long unwritten;
 509
 510         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 511         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 512                                                       user_data, length);
 513         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 514         if (unwritten)
 515                 return -EFAULT;
 516         return 0;
 517 }
 518
 519 /* Here's the write path which can sleep for
 520  * page faults
 521  */
 522
 523 static inline void
 524 slow_kernel_write(struct io_mapping *mapping,
 525                   loff_t gtt_base, int gtt_offset,
 526                   struct page *user_page, int user_offset,
 527                   int length)
 528 {
 529         char __iomem *dst_vaddr;
 530         char *src_vaddr;
 531
 532         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 533         src_vaddr = kmap(user_page);
 534
 535         memcpy_toio(dst_vaddr + gtt_offset,
 536                     src_vaddr + user_offset,
 537                     length);
 538
 539         kunmap(user_page);
 540         io_mapping_unmap(dst_vaddr);
 541 }
 542
 543 static inline int
 544 fast_shmem_write(struct page **pages,
 545                  loff_t page_base, int page_offset,
 546                  char __user *data,
 547                  int length)
 548 {
 549         char __iomem *vaddr;
 550         unsigned long unwritten;
 551
 552         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 553         if (vaddr == NULL)
 554                 return -ENOMEM;
 555         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 556         kunmap_atomic(vaddr, KM_USER0);
 557
 558         if (unwritten)
 559                 return -EFAULT;
 560         return 0;
 561 }
 562
 563 /**
 564  * This is the fast pwrite path, where we copy the data directly from the
 565  * user into the GTT, uncached.
 566  */
 567 static int
 568 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 569                          struct drm_i915_gem_pwrite *args,
 570                          struct drm_file *file_priv)
 571 {
 572         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 573         drm_i915_private_t *dev_priv = dev->dev_private;
 574         ssize_t remain;
 575         loff_t offset, page_base;
 576         char __user *user_data;
 577         int page_offset, page_length;
 578         int ret;
 579
 580         user_data = (char __user *) (uintptr_t) args->data_ptr;
 581         remain = args->size;
 582         if (!access_ok(VERIFY_READ, user_data, remain))
 583                 return -EFAULT;
 584
 585
 586         mutex_lock(&dev->struct_mutex);
 587         ret = i915_gem_object_pin(obj, 0);
 588         if (ret) {
 589                 mutex_unlock(&dev->struct_mutex);
 590                 return ret;
 591         }
 592         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 593         if (ret)
 594                 goto fail;
 595
 596         obj_priv = to_intel_bo(obj);
 597         offset = obj_priv->gtt_offset + args->offset;
 598
 599         while (remain > 0) {
 600                 /* Operation in this page
 601                  *
 602                  * page_base = page offset within aperture
 603                  * page_offset = offset within page
 604                  * page_length = bytes to copy for this page
 605                  */
 606                 page_base = (offset & ~(PAGE_SIZE-1));
 607                 page_offset = offset & (PAGE_SIZE-1);
 608                 page_length = remain;
 609                 if ((page_offset + remain) > PAGE_SIZE)
 610                         page_length = PAGE_SIZE - page_offset;
 611
 612                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 613                                        page_offset, user_data, page_length);
 614
 615                 /* If we get a fault while copying data, then (presumably) our
 616                  * source page isn't available.  Return the error and we'll
 617                  * retry in the slow path.
 618                  */
 619                 if (ret)
 620                         goto fail;
 621
 622                 remain -= page_length;
 623                 user_data += page_length;
 624                 offset += page_length;
 625         }
 626
 627 fail:
 628         i915_gem_object_unpin(obj);
 629         mutex_unlock(&dev->struct_mutex);
 630
 631         return ret;
 632 }
 633
 634 /**
 635  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 636  * the memory and maps it using kmap_atomic for copying.
 637  *
 638  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 639  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 640  */
 641 static int
 642 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 643                          struct drm_i915_gem_pwrite *args,
 644                          struct drm_file *file_priv)
 645 {
 646         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 647         drm_i915_private_t *dev_priv = dev->dev_private;
 648         ssize_t remain;
 649         loff_t gtt_page_base, offset;
 650         loff_t first_data_page, last_data_page, num_pages;
 651         loff_t pinned_pages, i;
 652         struct page **user_pages;
 653         struct mm_struct *mm = current->mm;
 654         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 655         int ret;
 656         uint64_t data_ptr = args->data_ptr;
 657
 658         remain = args->size;
 659
 660         /* Pin the user pages containing the data.  We can't fault while
 661          * holding the struct mutex, and all of the pwrite implementations
 662          * want to hold it while dereferencing the user data.
 663          */
 664         first_data_page = data_ptr / PAGE_SIZE;
 665         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 666         num_pages = last_data_page - first_data_page + 1;
 667
 668         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 669         if (user_pages == NULL)
 670                 return -ENOMEM;
 671
 672         down_read(&mm->mmap_sem);
 673         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 674                                       num_pages, 0, 0, user_pages, NULL);
 675         up_read(&mm->mmap_sem);
 676         if (pinned_pages < num_pages) {
 677                 ret = -EFAULT;
 678                 goto out_unpin_pages;
 679         }
 680
 681         mutex_lock(&dev->struct_mutex);
 682         ret = i915_gem_object_pin(obj, 0);
 683         if (ret)
 684                 goto out_unlock;
 685
 686         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 687         if (ret)
 688                 goto out_unpin_object;
 689
 690         obj_priv = to_intel_bo(obj);
 691         offset = obj_priv->gtt_offset + args->offset;
 692
 693         while (remain > 0) {
 694                 /* Operation in this page
 695                  *
 696                  * gtt_page_base = page offset within aperture
 697                  * gtt_page_offset = offset within page in aperture
 698                  * data_page_index = page number in get_user_pages return
 699                  * data_page_offset = offset with data_page_index page.
 700                  * page_length = bytes to copy for this page
 701                  */
 702                 gtt_page_base = offset & PAGE_MASK;
 703                 gtt_page_offset = offset & ~PAGE_MASK;
 704                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 705                 data_page_offset = data_ptr & ~PAGE_MASK;
 706
 707                 page_length = remain;
 708                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 709                         page_length = PAGE_SIZE - gtt_page_offset;
 710                 if ((data_page_offset + page_length) > PAGE_SIZE)
 711                         page_length = PAGE_SIZE - data_page_offset;
 712
 713                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 714                                   gtt_page_base, gtt_page_offset,
 715                                   user_pages[data_page_index],
 716                                   data_page_offset,
 717                                   page_length);
 718
 719                 remain -= page_length;
 720                 offset += page_length;
 721                 data_ptr += page_length;
 722         }
 723
 724 out_unpin_object:
 725         i915_gem_object_unpin(obj);
 726 out_unlock:
 727         mutex_unlock(&dev->struct_mutex);
 728 out_unpin_pages:
 729         for (i = 0; i < pinned_pages; i++)
 730                 page_cache_release(user_pages[i]);
 731         drm_free_large(user_pages);
 732
 733         return ret;
 734 }
 735
 736 /**
 737  * This is the fast shmem pwrite path, which attempts to directly
 738  * copy_from_user into the kmapped pages backing the object.
 739  */
 740 static int
 741 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 742                            struct drm_i915_gem_pwrite *args,
 743                            struct drm_file *file_priv)
 744 {
 745         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 746         ssize_t remain;
 747         loff_t offset, page_base;
 748         char __user *user_data;
 749         int page_offset, page_length;
 750         int ret;
 751
 752         user_data = (char __user *) (uintptr_t) args->data_ptr;
 753         remain = args->size;
 754
 755         mutex_lock(&dev->struct_mutex);
 756
 757         ret = i915_gem_object_get_pages(obj, 0);
 758         if (ret != 0)
 759                 goto fail_unlock;
 760
 761         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 762         if (ret != 0)
 763                 goto fail_put_pages;
 764
 765         obj_priv = to_intel_bo(obj);
 766         offset = args->offset;
 767         obj_priv->dirty = 1;
 768
 769         while (remain > 0) {
 770                 /* Operation in this page
 771                  *
 772                  * page_base = page offset within aperture
 773                  * page_offset = offset within page
 774                  * page_length = bytes to copy for this page
 775                  */
 776                 page_base = (offset & ~(PAGE_SIZE-1));
 777                 page_offset = offset & (PAGE_SIZE-1);
 778                 page_length = remain;
 779                 if ((page_offset + remain) > PAGE_SIZE)
 780                         page_length = PAGE_SIZE - page_offset;
 781
 782                 ret = fast_shmem_write(obj_priv->pages,
 783                                        page_base, page_offset,
 784                                        user_data, page_length);
 785                 if (ret)
 786                         goto fail_put_pages;
 787
 788                 remain -= page_length;
 789                 user_data += page_length;
 790                 offset += page_length;
 791         }
 792
 793 fail_put_pages:
 794         i915_gem_object_put_pages(obj);
 795 fail_unlock:
 796         mutex_unlock(&dev->struct_mutex);
 797
 798         return ret;
 799 }
 800
 801 /**
 802  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 803  * the memory and maps it using kmap_atomic for copying.
 804  *
 805  * This avoids taking mmap_sem for faulting on the user's address while the
 806  * struct_mutex is held.
 807  */
 808 static int
 809 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 810                            struct drm_i915_gem_pwrite *args,
 811                            struct drm_file *file_priv)
 812 {
 813         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 814         struct mm_struct *mm = current->mm;
 815         struct page **user_pages;
 816         ssize_t remain;
 817         loff_t offset, pinned_pages, i;
 818         loff_t first_data_page, last_data_page, num_pages;
 819         int shmem_page_index, shmem_page_offset;
 820         int data_page_index,  data_page_offset;
 821         int page_length;
 822         int ret;
 823         uint64_t data_ptr = args->data_ptr;
 824         int do_bit17_swizzling;
 825
 826         remain = args->size;
 827
 828         /* Pin the user pages containing the data.  We can't fault while
 829          * holding the struct mutex, and all of the pwrite implementations
 830          * want to hold it while dereferencing the user data.
 831          */
 832         first_data_page = data_ptr / PAGE_SIZE;
 833         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 834         num_pages = last_data_page - first_data_page + 1;
 835
 836         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 837         if (user_pages == NULL)
 838                 return -ENOMEM;
 839
 840         down_read(&mm->mmap_sem);
 841         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 842                                       num_pages, 0, 0, user_pages, NULL);
 843         up_read(&mm->mmap_sem);
 844         if (pinned_pages < num_pages) {
 845                 ret = -EFAULT;
 846                 goto fail_put_user_pages;
 847         }
 848
 849         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 850
 851         mutex_lock(&dev->struct_mutex);
 852
 853         ret = i915_gem_object_get_pages_or_evict(obj);
 854         if (ret)
 855                 goto fail_unlock;
 856
 857         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 858         if (ret != 0)
 859                 goto fail_put_pages;
 860
 861         obj_priv = to_intel_bo(obj);
 862         offset = args->offset;
 863         obj_priv->dirty = 1;
 864
 865         while (remain > 0) {
 866                 /* Operation in this page
 867                  *
 868                  * shmem_page_index = page number within shmem file
 869                  * shmem_page_offset = offset within page in shmem file
 870                  * data_page_index = page number in get_user_pages return
 871                  * data_page_offset = offset with data_page_index page.
 872                  * page_length = bytes to copy for this page
 873                  */
 874                 shmem_page_index = offset / PAGE_SIZE;
 875                 shmem_page_offset = offset & ~PAGE_MASK;
 876                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 877                 data_page_offset = data_ptr & ~PAGE_MASK;
 878
 879                 page_length = remain;
 880                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 881                         page_length = PAGE_SIZE - shmem_page_offset;
 882                 if ((data_page_offset + page_length) > PAGE_SIZE)
 883                         page_length = PAGE_SIZE - data_page_offset;
 884
 885                 if (do_bit17_swizzling) {
 886                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 887                                               shmem_page_offset,
 888                                               user_pages[data_page_index],
 889                                               data_page_offset,
 890                                               page_length,
 891                                               0);
 892                 } else {
 893                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 894                                         shmem_page_offset,
 895                                         user_pages[data_page_index],
 896                                         data_page_offset,
 897                                         page_length);
 898                 }
 899
 900                 remain -= page_length;
 901                 data_ptr += page_length;
 902                 offset += page_length;
 903         }
 904
 905 fail_put_pages:
 906         i915_gem_object_put_pages(obj);
 907 fail_unlock:
 908         mutex_unlock(&dev->struct_mutex);
 909 fail_put_user_pages:
 910         for (i = 0; i < pinned_pages; i++)
 911                 page_cache_release(user_pages[i]);
 912         drm_free_large(user_pages);
 913
 914         return ret;
 915 }
 916
 917 /**
 918  * Writes data to the object referenced by handle.
 919  *
 920  * On error, the contents of the buffer that were to be modified are undefined.
 921  */
 922 int
 923 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 924                       struct drm_file *file_priv)
 925 {
 926         struct drm_i915_gem_pwrite *args = data;
 927         struct drm_gem_object *obj;
 928         struct drm_i915_gem_object *obj_priv;
 929         int ret = 0;
 930
 931         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 932         if (obj == NULL)
 933                 return -ENOENT;
 934         obj_priv = to_intel_bo(obj);
 935
 936         /* Bounds check destination.
 937          *
 938          * XXX: This could use review for overflow issues...
 939          */
 940         if (args->offset > obj->size || args->size > obj->size ||
 941             args->offset + args->size > obj->size) {
 942                 drm_gem_object_unreference_unlocked(obj);
 943                 return -EINVAL;
 944         }
 945
 946         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 947          * it would end up going through the fenced access, and we'll get
 948          * different detiling behavior between reading and writing.
 949          * pread/pwrite currently are reading and writing from the CPU
 950          * perspective, requiring manual detiling by the client.
 951          */
 952         if (obj_priv->phys_obj)
 953                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 954         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 955                  dev->gtt_total != 0 &&
 956                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 957                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 958                 if (ret == -EFAULT) {
 959                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 960                                                        file_priv);
 961                 }
 962         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 963                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 964         } else {
 965                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 966                 if (ret == -EFAULT) {
 967                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 968                                                          file_priv);
 969                 }
 970         }
 971
 972 #if WATCH_PWRITE
 973         if (ret)
 974                 DRM_INFO("pwrite failed %d\n", ret);
 975 #endif
 976
 977         drm_gem_object_unreference_unlocked(obj);
 978
 979         return ret;
 980 }
 981
 982 /**
 983  * Called when user space prepares to use an object with the CPU, either
 984  * through the mmap ioctl's mapping or a GTT mapping.
 985  */
 986 int
 987 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 988                           struct drm_file *file_priv)
 989 {
 990         struct drm_i915_private *dev_priv = dev->dev_private;
 991         struct drm_i915_gem_set_domain *args = data;
 992         struct drm_gem_object *obj;
 993         struct drm_i915_gem_object *obj_priv;
 994         uint32_t read_domains = args->read_domains;
 995         uint32_t write_domain = args->write_domain;
 996         int ret;
 997
 998         if (!(dev->driver->driver_features & DRIVER_GEM))
 999                 return -ENODEV;
1000
1001         /* Only handle setting domains to types used by the CPU. */
1002         if (write_domain & I915_GEM_GPU_DOMAINS)
1003                 return -EINVAL;
1004
1005         if (read_domains & I915_GEM_GPU_DOMAINS)
1006                 return -EINVAL;
1007
1008         /* Having something in the write domain implies it's in the read
1009          * domain, and only that read domain.  Enforce that in the request.
1010          */
1011         if (write_domain != 0 && read_domains != write_domain)
1012                 return -EINVAL;
1013
1014         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1015         if (obj == NULL)
1016                 return -ENOENT;
1017         obj_priv = to_intel_bo(obj);
1018
1019         mutex_lock(&dev->struct_mutex);
1020
1021         intel_mark_busy(dev, obj);
1022
1023 #if WATCH_BUF
1024         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1025                  obj, obj->size, read_domains, write_domain);
1026 #endif
1027         if (read_domains & I915_GEM_DOMAIN_GTT) {
1028                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1029
1030                 /* Update the LRU on the fence for the CPU access that's
1031                  * about to occur.
1032                  */
1033                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1034                         struct drm_i915_fence_reg *reg =
1035                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1036                         list_move_tail(&reg->lru_list,
1037                                        &dev_priv->mm.fence_list);
1038                 }
1039
1040                 /* Silently promote "you're not bound, there was nothing to do"
1041                  * to success, since the client was just asking us to
1042                  * make sure everything was done.
1043                  */
1044                 if (ret == -EINVAL)
1045                         ret = 0;
1046         } else {
1047                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1048         }
1049
1050
1051         /* Maintain LRU order of "inactive" objects */
1052         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1053                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1054
1055         drm_gem_object_unreference(obj);
1056         mutex_unlock(&dev->struct_mutex);
1057         return ret;
1058 }
1059
1060 /**
1061  * Called when user space has done writes to this buffer
1062  */
1063 int
1064 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1065                       struct drm_file *file_priv)
1066 {
1067         struct drm_i915_gem_sw_finish *args = data;
1068         struct drm_gem_object *obj;
1069         struct drm_i915_gem_object *obj_priv;
1070         int ret = 0;
1071
1072         if (!(dev->driver->driver_features & DRIVER_GEM))
1073                 return -ENODEV;
1074
1075         mutex_lock(&dev->struct_mutex);
1076         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1077         if (obj == NULL) {
1078                 mutex_unlock(&dev->struct_mutex);
1079                 return -ENOENT;
1080         }
1081
1082 #if WATCH_BUF
1083         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1084                  __func__, args->handle, obj, obj->size);
1085 #endif
1086         obj_priv = to_intel_bo(obj);
1087
1088         /* Pinned buffers may be scanout, so flush the cache */
1089         if (obj_priv->pin_count)
1090                 i915_gem_object_flush_cpu_write_domain(obj);
1091
1092         drm_gem_object_unreference(obj);
1093         mutex_unlock(&dev->struct_mutex);
1094         return ret;
1095 }
1096
1097 /**
1098  * Maps the contents of an object, returning the address it is mapped
1099  * into.
1100  *
1101  * While the mapping holds a reference on the contents of the object, it doesn't
1102  * imply a ref on the object itself.
1103  */
1104 int
1105 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1106                    struct drm_file *file_priv)
1107 {
1108         struct drm_i915_gem_mmap *args = data;
1109         struct drm_gem_object *obj;
1110         loff_t offset;
1111         unsigned long addr;
1112
1113         if (!(dev->driver->driver_features & DRIVER_GEM))
1114                 return -ENODEV;
1115
1116         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1117         if (obj == NULL)
1118                 return -ENOENT;
1119
1120         offset = args->offset;
1121
1122         down_write(&current->mm->mmap_sem);
1123         addr = do_mmap(obj->filp, 0, args->size,
1124                        PROT_READ | PROT_WRITE, MAP_SHARED,
1125                        args->offset);
1126         up_write(&current->mm->mmap_sem);
1127         drm_gem_object_unreference_unlocked(obj);
1128         if (IS_ERR((void *)addr))
1129                 return addr;
1130
1131         args->addr_ptr = (uint64_t) addr;
1132
1133         return 0;
1134 }
1135
1136 /**
1137  * i915_gem_fault - fault a page into the GTT
1138  * vma: VMA in question
1139  * vmf: fault info
1140  *
1141  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1142  * from userspace.  The fault handler takes care of binding the object to
1143  * the GTT (if needed), allocating and programming a fence register (again,
1144  * only if needed based on whether the old reg is still valid or the object
1145  * is tiled) and inserting a new PTE into the faulting process.
1146  *
1147  * Note that the faulting process may involve evicting existing objects
1148  * from the GTT and/or fence registers to make room.  So performance may
1149  * suffer if the GTT working set is large or there are few fence registers
1150  * left.
1151  */
1152 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1153 {
1154         struct drm_gem_object *obj = vma->vm_private_data;
1155         struct drm_device *dev = obj->dev;
1156         drm_i915_private_t *dev_priv = dev->dev_private;
1157         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1158         pgoff_t page_offset;
1159         unsigned long pfn;
1160         int ret = 0;
1161         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1162
1163         /* We don't use vmf->pgoff since that has the fake offset */
1164         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1165                 PAGE_SHIFT;
1166
1167         /* Now bind it into the GTT if needed */
1168         mutex_lock(&dev->struct_mutex);
1169         if (!obj_priv->gtt_space) {
1170                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1171                 if (ret)
1172                         goto unlock;
1173
1174                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1175                 if (ret)
1176                         goto unlock;
1177         }
1178
1179         /* Need a new fence register? */
1180         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1181                 ret = i915_gem_object_get_fence_reg(obj);
1182                 if (ret)
1183                         goto unlock;
1184         }
1185
1186         if (i915_gem_object_is_inactive(obj_priv))
1187                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1188
1189         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1190                 page_offset;
1191
1192         /* Finally, remap it using the new GTT offset */
1193         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1194 unlock:
1195         mutex_unlock(&dev->struct_mutex);
1196
1197         switch (ret) {
1198         case 0:
1199         case -ERESTARTSYS:
1200                 return VM_FAULT_NOPAGE;
1201         case -ENOMEM:
1202         case -EAGAIN:
1203                 return VM_FAULT_OOM;
1204         default:
1205                 return VM_FAULT_SIGBUS;
1206         }
1207 }
1208
1209 /**
1210  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1211  * @obj: obj in question
1212  *
1213  * GEM memory mapping works by handing back to userspace a fake mmap offset
1214  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1215  * up the object based on the offset and sets up the various memory mapping
1216  * structures.
1217  *
1218  * This routine allocates and attaches a fake offset for @obj.
1219  */
1220 static int
1221 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1222 {
1223         struct drm_device *dev = obj->dev;
1224         struct drm_gem_mm *mm = dev->mm_private;
1225         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1226         struct drm_map_list *list;
1227         struct drm_local_map *map;
1228         int ret = 0;
1229
1230         /* Set the object up for mmap'ing */
1231         list = &obj->map_list;
1232         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1233         if (!list->map)
1234                 return -ENOMEM;
1235
1236         map = list->map;
1237         map->type = _DRM_GEM;
1238         map->size = obj->size;
1239         map->handle = obj;
1240
1241         /* Get a DRM GEM mmap offset allocated... */
1242         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1243                                                     obj->size / PAGE_SIZE, 0, 0);
1244         if (!list->file_offset_node) {
1245                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1246                 ret = -ENOMEM;
1247                 goto out_free_list;
1248         }
1249
1250         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1251                                                   obj->size / PAGE_SIZE, 0);
1252         if (!list->file_offset_node) {
1253                 ret = -ENOMEM;
1254                 goto out_free_list;
1255         }
1256
1257         list->hash.key = list->file_offset_node->start;
1258         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1259                 DRM_ERROR("failed to add to map hash\n");
1260                 ret = -ENOMEM;
1261                 goto out_free_mm;
1262         }
1263
1264         /* By now we should be all set, any drm_mmap request on the offset
1265          * below will get to our mmap & fault handler */
1266         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1267
1268         return 0;
1269
1270 out_free_mm:
1271         drm_mm_put_block(list->file_offset_node);
1272 out_free_list:
1273         kfree(list->map);
1274
1275         return ret;
1276 }
1277
1278 /**
1279  * i915_gem_release_mmap - remove physical page mappings
1280  * @obj: obj in question
1281  *
1282  * Preserve the reservation of the mmapping with the DRM core code, but
1283  * relinquish ownership of the pages back to the system.
1284  *
1285  * It is vital that we remove the page mapping if we have mapped a tiled
1286  * object through the GTT and then lose the fence register due to
1287  * resource pressure. Similarly if the object has been moved out of the
1288  * aperture, than pages mapped into userspace must be revoked. Removing the
1289  * mapping will then trigger a page fault on the next user access, allowing
1290  * fixup by i915_gem_fault().
1291  */
1292 void
1293 i915_gem_release_mmap(struct drm_gem_object *obj)
1294 {
1295         struct drm_device *dev = obj->dev;
1296         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1297
1298         if (dev->dev_mapping)
1299                 unmap_mapping_range(dev->dev_mapping,
1300                                     obj_priv->mmap_offset, obj->size, 1);
1301 }
1302
1303 static void
1304 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1305 {
1306         struct drm_device *dev = obj->dev;
1307         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1308         struct drm_gem_mm *mm = dev->mm_private;
1309         struct drm_map_list *list;
1310
1311         list = &obj->map_list;
1312         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1313
1314         if (list->file_offset_node) {
1315                 drm_mm_put_block(list->file_offset_node);
1316                 list->file_offset_node = NULL;
1317         }
1318
1319         if (list->map) {
1320                 kfree(list->map);
1321                 list->map = NULL;
1322         }
1323
1324         obj_priv->mmap_offset = 0;
1325 }
1326
1327 /**
1328  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1329  * @obj: object to check
1330  *
1331  * Return the required GTT alignment for an object, taking into account
1332  * potential fence register mapping if needed.
1333  */
1334 static uint32_t
1335 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1336 {
1337         struct drm_device *dev = obj->dev;
1338         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1339         int start, i;
1340
1341         /*
1342          * Minimum alignment is 4k (GTT page size), but might be greater
1343          * if a fence register is needed for the object.
1344          */
1345         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1346                 return 4096;
1347
1348         /*
1349          * Previous chips need to be aligned to the size of the smallest
1350          * fence register that can contain the object.
1351          */
1352         if (IS_I9XX(dev))
1353                 start = 1024*1024;
1354         else
1355                 start = 512*1024;
1356
1357         for (i = start; i < obj->size; i <<= 1)
1358                 ;
1359
1360         return i;
1361 }
1362
1363 /**
1364  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1365  * @dev: DRM device
1366  * @data: GTT mapping ioctl data
1367  * @file_priv: GEM object info
1368  *
1369  * Simply returns the fake offset to userspace so it can mmap it.
1370  * The mmap call will end up in drm_gem_mmap(), which will set things
1371  * up so we can get faults in the handler above.
1372  *
1373  * The fault handler will take care of binding the object into the GTT
1374  * (since it may have been evicted to make room for something), allocating
1375  * a fence register, and mapping the appropriate aperture address into
1376  * userspace.
1377  */
1378 int
1379 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1380                         struct drm_file *file_priv)
1381 {
1382         struct drm_i915_gem_mmap_gtt *args = data;
1383         struct drm_gem_object *obj;
1384         struct drm_i915_gem_object *obj_priv;
1385         int ret;
1386
1387         if (!(dev->driver->driver_features & DRIVER_GEM))
1388                 return -ENODEV;
1389
1390         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1391         if (obj == NULL)
1392                 return -ENOENT;
1393
1394         mutex_lock(&dev->struct_mutex);
1395
1396         obj_priv = to_intel_bo(obj);
1397
1398         if (obj_priv->madv != I915_MADV_WILLNEED) {
1399                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1400                 drm_gem_object_unreference(obj);
1401                 mutex_unlock(&dev->struct_mutex);
1402                 return -EINVAL;
1403         }
1404
1405
1406         if (!obj_priv->mmap_offset) {
1407                 ret = i915_gem_create_mmap_offset(obj);
1408                 if (ret) {
1409                         drm_gem_object_unreference(obj);
1410                         mutex_unlock(&dev->struct_mutex);
1411                         return ret;
1412                 }
1413         }
1414
1415         args->offset = obj_priv->mmap_offset;
1416
1417         /*
1418          * Pull it into the GTT so that we have a page list (makes the
1419          * initial fault faster and any subsequent flushing possible).
1420          */
1421         if (!obj_priv->agp_mem) {
1422                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1423                 if (ret) {
1424                         drm_gem_object_unreference(obj);
1425                         mutex_unlock(&dev->struct_mutex);
1426                         return ret;
1427                 }
1428         }
1429
1430         drm_gem_object_unreference(obj);
1431         mutex_unlock(&dev->struct_mutex);
1432
1433         return 0;
1434 }
1435
1436 void
1437 i915_gem_object_put_pages(struct drm_gem_object *obj)
1438 {
1439         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1440         int page_count = obj->size / PAGE_SIZE;
1441         int i;
1442
1443         BUG_ON(obj_priv->pages_refcount == 0);
1444         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1445
1446         if (--obj_priv->pages_refcount != 0)
1447                 return;
1448
1449         if (obj_priv->tiling_mode != I915_TILING_NONE)
1450                 i915_gem_object_save_bit_17_swizzle(obj);
1451
1452         if (obj_priv->madv == I915_MADV_DONTNEED)
1453                 obj_priv->dirty = 0;
1454
1455         for (i = 0; i < page_count; i++) {
1456                 if (obj_priv->dirty)
1457                         set_page_dirty(obj_priv->pages[i]);
1458
1459                 if (obj_priv->madv == I915_MADV_WILLNEED)
1460                         mark_page_accessed(obj_priv->pages[i]);
1461
1462                 page_cache_release(obj_priv->pages[i]);
1463         }
1464         obj_priv->dirty = 0;
1465
1466         drm_free_large(obj_priv->pages);
1467         obj_priv->pages = NULL;
1468 }
1469
1470 static void
1471 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1472                                struct intel_ring_buffer *ring)
1473 {
1474         struct drm_device *dev = obj->dev;
1475         drm_i915_private_t *dev_priv = dev->dev_private;
1476         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1477         BUG_ON(ring == NULL);
1478         obj_priv->ring = ring;
1479
1480         /* Add a reference if we're newly entering the active list. */
1481         if (!obj_priv->active) {
1482                 drm_gem_object_reference(obj);
1483                 obj_priv->active = 1;
1484         }
1485         /* Move from whatever list we were on to the tail of execution. */
1486         spin_lock(&dev_priv->mm.active_list_lock);
1487         list_move_tail(&obj_priv->list, &ring->active_list);
1488         spin_unlock(&dev_priv->mm.active_list_lock);
1489         obj_priv->last_rendering_seqno = seqno;
1490 }
1491
1492 static void
1493 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1494 {
1495         struct drm_device *dev = obj->dev;
1496         drm_i915_private_t *dev_priv = dev->dev_private;
1497         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1498
1499         BUG_ON(!obj_priv->active);
1500         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1501         obj_priv->last_rendering_seqno = 0;
1502 }
1503
1504 /* Immediately discard the backing storage */
1505 static void
1506 i915_gem_object_truncate(struct drm_gem_object *obj)
1507 {
1508         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1509         struct inode *inode;
1510
1511         /* Our goal here is to return as much of the memory as
1512          * is possible back to the system as we are called from OOM.
1513          * To do this we must instruct the shmfs to drop all of its
1514          * backing pages, *now*. Here we mirror the actions taken
1515          * when by shmem_delete_inode() to release the backing store.
1516          */
1517         inode = obj->filp->f_path.dentry->d_inode;
1518         truncate_inode_pages(inode->i_mapping, 0);
1519         if (inode->i_op->truncate_range)
1520                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1521
1522         obj_priv->madv = __I915_MADV_PURGED;
1523 }
1524
1525 static inline int
1526 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1527 {
1528         return obj_priv->madv == I915_MADV_DONTNEED;
1529 }
1530
1531 static void
1532 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1533 {
1534         struct drm_device *dev = obj->dev;
1535         drm_i915_private_t *dev_priv = dev->dev_private;
1536         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1537
1538         i915_verify_inactive(dev, __FILE__, __LINE__);
1539         if (obj_priv->pin_count != 0)
1540                 list_del_init(&obj_priv->list);
1541         else
1542                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1543
1544         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1545
1546         obj_priv->last_rendering_seqno = 0;
1547         obj_priv->ring = NULL;
1548         if (obj_priv->active) {
1549                 obj_priv->active = 0;
1550                 drm_gem_object_unreference(obj);
1551         }
1552         i915_verify_inactive(dev, __FILE__, __LINE__);
1553 }
1554
1555 static void
1556 i915_gem_process_flushing_list(struct drm_device *dev,
1557                                uint32_t flush_domains, uint32_t seqno,
1558                                struct intel_ring_buffer *ring)
1559 {
1560         drm_i915_private_t *dev_priv = dev->dev_private;
1561         struct drm_i915_gem_object *obj_priv, *next;
1562
1563         list_for_each_entry_safe(obj_priv, next,
1564                                  &dev_priv->mm.gpu_write_list,
1565                                  gpu_write_list) {
1566                 struct drm_gem_object *obj = &obj_priv->base;
1567
1568                 if ((obj->write_domain & flush_domains) ==
1569                     obj->write_domain &&
1570                     obj_priv->ring->ring_flag == ring->ring_flag) {
1571                         uint32_t old_write_domain = obj->write_domain;
1572
1573                         obj->write_domain = 0;
1574                         list_del_init(&obj_priv->gpu_write_list);
1575                         i915_gem_object_move_to_active(obj, seqno, ring);
1576
1577                         /* update the fence lru list */
1578                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1579                                 struct drm_i915_fence_reg *reg =
1580                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1581                                 list_move_tail(&reg->lru_list,
1582                                                 &dev_priv->mm.fence_list);
1583                         }
1584
1585                         trace_i915_gem_object_change_domain(obj,
1586                                                             obj->read_domains,
1587                                                             old_write_domain);
1588                 }
1589         }
1590 }
1591
1592 uint32_t
1593 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1594                  uint32_t flush_domains, struct intel_ring_buffer *ring)
1595 {
1596         drm_i915_private_t *dev_priv = dev->dev_private;
1597         struct drm_i915_file_private *i915_file_priv = NULL;
1598         struct drm_i915_gem_request *request;
1599         uint32_t seqno;
1600         int was_empty;
1601
1602         if (file_priv != NULL)
1603                 i915_file_priv = file_priv->driver_priv;
1604
1605         request = kzalloc(sizeof(*request), GFP_KERNEL);
1606         if (request == NULL)
1607                 return 0;
1608
1609         seqno = ring->add_request(dev, ring, file_priv, flush_domains);
1610
1611         request->seqno = seqno;
1612         request->ring = ring;
1613         request->emitted_jiffies = jiffies;
1614         was_empty = list_empty(&ring->request_list);
1615         list_add_tail(&request->list, &ring->request_list);
1616
1617         if (i915_file_priv) {
1618                 list_add_tail(&request->client_list,
1619                               &i915_file_priv->mm.request_list);
1620         } else {
1621                 INIT_LIST_HEAD(&request->client_list);
1622         }
1623
1624         /* Associate any objects on the flushing list matching the write
1625          * domain we're flushing with our flush.
1626          */
1627         if (flush_domains != 0)
1628                 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1629
1630         if (!dev_priv->mm.suspended) {
1631                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1632                 if (was_empty)
1633                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1634         }
1635         return seqno;
1636 }
1637
1638 /**
1639  * Command execution barrier
1640  *
1641  * Ensures that all commands in the ring are finished
1642  * before signalling the CPU
1643  */
1644 static uint32_t
1645 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1646 {
1647         uint32_t flush_domains = 0;
1648
1649         /* The sampler always gets flushed on i965 (sigh) */
1650         if (IS_I965G(dev))
1651                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1652
1653         ring->flush(dev, ring,
1654                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1655         return flush_domains;
1656 }
1657
1658 /**
1659  * Moves buffers associated only with the given active seqno from the active
1660  * to inactive list, potentially freeing them.
1661  */
1662 static void
1663 i915_gem_retire_request(struct drm_device *dev,
1664                         struct drm_i915_gem_request *request)
1665 {
1666         drm_i915_private_t *dev_priv = dev->dev_private;
1667
1668         trace_i915_gem_request_retire(dev, request->seqno);
1669
1670         /* Move any buffers on the active list that are no longer referenced
1671          * by the ringbuffer to the flushing/inactive lists as appropriate.
1672          */
1673         spin_lock(&dev_priv->mm.active_list_lock);
1674         while (!list_empty(&request->ring->active_list)) {
1675                 struct drm_gem_object *obj;
1676                 struct drm_i915_gem_object *obj_priv;
1677
1678                 obj_priv = list_first_entry(&request->ring->active_list,
1679                                             struct drm_i915_gem_object,
1680                                             list);
1681                 obj = &obj_priv->base;
1682
1683                 /* If the seqno being retired doesn't match the oldest in the
1684                  * list, then the oldest in the list must still be newer than
1685                  * this seqno.
1686                  */
1687                 if (obj_priv->last_rendering_seqno != request->seqno)
1688                         goto out;
1689
1690 #if WATCH_LRU
1691                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1692                          __func__, request->seqno, obj);
1693 #endif
1694
1695                 if (obj->write_domain != 0)
1696                         i915_gem_object_move_to_flushing(obj);
1697                 else {
1698                         /* Take a reference on the object so it won't be
1699                          * freed while the spinlock is held.  The list
1700                          * protection for this spinlock is safe when breaking
1701                          * the lock like this since the next thing we do
1702                          * is just get the head of the list again.
1703                          */
1704                         drm_gem_object_reference(obj);
1705                         i915_gem_object_move_to_inactive(obj);
1706                         spin_unlock(&dev_priv->mm.active_list_lock);
1707                         drm_gem_object_unreference(obj);
1708                         spin_lock(&dev_priv->mm.active_list_lock);
1709                 }
1710         }
1711 out:
1712         spin_unlock(&dev_priv->mm.active_list_lock);
1713 }
1714
1715 /**
1716  * Returns true if seq1 is later than seq2.
1717  */
1718 bool
1719 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1720 {
1721         return (int32_t)(seq1 - seq2) >= 0;
1722 }
1723
1724 uint32_t
1725 i915_get_gem_seqno(struct drm_device *dev,
1726                    struct intel_ring_buffer *ring)
1727 {
1728         return ring->get_gem_seqno(dev, ring);
1729 }
1730
1731 /**
1732  * This function clears the request list as sequence numbers are passed.
1733  */
1734 static void
1735 i915_gem_retire_requests_ring(struct drm_device *dev,
1736                               struct intel_ring_buffer *ring)
1737 {
1738         drm_i915_private_t *dev_priv = dev->dev_private;
1739         uint32_t seqno;
1740
1741         if (!ring->status_page.page_addr
1742                         || list_empty(&ring->request_list))
1743                 return;
1744
1745         seqno = i915_get_gem_seqno(dev, ring);
1746
1747         while (!list_empty(&ring->request_list)) {
1748                 struct drm_i915_gem_request *request;
1749                 uint32_t retiring_seqno;
1750
1751                 request = list_first_entry(&ring->request_list,
1752                                            struct drm_i915_gem_request,
1753                                            list);
1754                 retiring_seqno = request->seqno;
1755
1756                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1757                     atomic_read(&dev_priv->mm.wedged)) {
1758                         i915_gem_retire_request(dev, request);
1759
1760                         list_del(&request->list);
1761                         list_del(&request->client_list);
1762                         kfree(request);
1763                 } else
1764                         break;
1765         }
1766
1767         if (unlikely (dev_priv->trace_irq_seqno &&
1768                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1769
1770                 ring->user_irq_put(dev, ring);
1771                 dev_priv->trace_irq_seqno = 0;
1772         }
1773 }
1774
1775 void
1776 i915_gem_retire_requests(struct drm_device *dev)
1777 {
1778         drm_i915_private_t *dev_priv = dev->dev_private;
1779
1780         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1781             struct drm_i915_gem_object *obj_priv, *tmp;
1782
1783             /* We must be careful that during unbind() we do not
1784              * accidentally infinitely recurse into retire requests.
1785              * Currently:
1786              *   retire -> free -> unbind -> wait -> retire_ring
1787              */
1788             list_for_each_entry_safe(obj_priv, tmp,
1789                                      &dev_priv->mm.deferred_free_list,
1790                                      list)
1791                     i915_gem_free_object_tail(&obj_priv->base);
1792         }
1793
1794         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1795         if (HAS_BSD(dev))
1796                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1797 }
1798
1799 void
1800 i915_gem_retire_work_handler(struct work_struct *work)
1801 {
1802         drm_i915_private_t *dev_priv;
1803         struct drm_device *dev;
1804
1805         dev_priv = container_of(work, drm_i915_private_t,
1806                                 mm.retire_work.work);
1807         dev = dev_priv->dev;
1808
1809         mutex_lock(&dev->struct_mutex);
1810         i915_gem_retire_requests(dev);
1811
1812         if (!dev_priv->mm.suspended &&
1813                 (!list_empty(&dev_priv->render_ring.request_list) ||
1814                         (HAS_BSD(dev) &&
1815                          !list_empty(&dev_priv->bsd_ring.request_list))))
1816                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1817         mutex_unlock(&dev->struct_mutex);
1818 }
1819
1820 int
1821 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1822                 int interruptible, struct intel_ring_buffer *ring)
1823 {
1824         drm_i915_private_t *dev_priv = dev->dev_private;
1825         u32 ier;
1826         int ret = 0;
1827
1828         BUG_ON(seqno == 0);
1829
1830         if (atomic_read(&dev_priv->mm.wedged))
1831                 return -EIO;
1832
1833         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1834                 if (HAS_PCH_SPLIT(dev))
1835                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1836                 else
1837                         ier = I915_READ(IER);
1838                 if (!ier) {
1839                         DRM_ERROR("something (likely vbetool) disabled "
1840                                   "interrupts, re-enabling\n");
1841                         i915_driver_irq_preinstall(dev);
1842                         i915_driver_irq_postinstall(dev);
1843                 }
1844
1845                 trace_i915_gem_request_wait_begin(dev, seqno);
1846
1847                 ring->waiting_gem_seqno = seqno;
1848                 ring->user_irq_get(dev, ring);
1849                 if (interruptible)
1850                         ret = wait_event_interruptible(ring->irq_queue,
1851                                 i915_seqno_passed(
1852                                         ring->get_gem_seqno(dev, ring), seqno)
1853                                 || atomic_read(&dev_priv->mm.wedged));
1854                 else
1855                         wait_event(ring->irq_queue,
1856                                 i915_seqno_passed(
1857                                         ring->get_gem_seqno(dev, ring), seqno)
1858                                 || atomic_read(&dev_priv->mm.wedged));
1859
1860                 ring->user_irq_put(dev, ring);
1861                 ring->waiting_gem_seqno = 0;
1862
1863                 trace_i915_gem_request_wait_end(dev, seqno);
1864         }
1865         if (atomic_read(&dev_priv->mm.wedged))
1866                 ret = -EIO;
1867
1868         if (ret && ret != -ERESTARTSYS)
1869                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1870                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
1871
1872         /* Directly dispatch request retiring.  While we have the work queue
1873          * to handle this, the waiter on a request often wants an associated
1874          * buffer to have made it to the inactive list, and we would need
1875          * a separate wait queue to handle that.
1876          */
1877         if (ret == 0)
1878                 i915_gem_retire_requests_ring(dev, ring);
1879
1880         return ret;
1881 }
1882
1883 /**
1884  * Waits for a sequence number to be signaled, and cleans up the
1885  * request and object lists appropriately for that event.
1886  */
1887 static int
1888 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1889                 struct intel_ring_buffer *ring)
1890 {
1891         return i915_do_wait_request(dev, seqno, 1, ring);
1892 }
1893
1894 static void
1895 i915_gem_flush(struct drm_device *dev,
1896                uint32_t invalidate_domains,
1897                uint32_t flush_domains)
1898 {
1899         drm_i915_private_t *dev_priv = dev->dev_private;
1900         if (flush_domains & I915_GEM_DOMAIN_CPU)
1901                 drm_agp_chipset_flush(dev);
1902         dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1903                         invalidate_domains,
1904                         flush_domains);
1905
1906         if (HAS_BSD(dev))
1907                 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1908                                 invalidate_domains,
1909                                 flush_domains);
1910 }
1911
1912 /**
1913  * Ensures that all rendering to the object has completed and the object is
1914  * safe to unbind from the GTT or access from the CPU.
1915  */
1916 static int
1917 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1918 {
1919         struct drm_device *dev = obj->dev;
1920         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1921         int ret;
1922
1923         /* This function only exists to support waiting for existing rendering,
1924          * not for emitting required flushes.
1925          */
1926         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1927
1928         /* If there is rendering queued on the buffer being evicted, wait for
1929          * it.
1930          */
1931         if (obj_priv->active) {
1932 #if WATCH_BUF
1933                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1934                           __func__, obj, obj_priv->last_rendering_seqno);
1935 #endif
1936                 ret = i915_wait_request(dev,
1937                                 obj_priv->last_rendering_seqno, obj_priv->ring);
1938                 if (ret != 0)
1939                         return ret;
1940         }
1941
1942         return 0;
1943 }
1944
1945 /**
1946  * Unbinds an object from the GTT aperture.
1947  */
1948 int
1949 i915_gem_object_unbind(struct drm_gem_object *obj)
1950 {
1951         struct drm_device *dev = obj->dev;
1952         drm_i915_private_t *dev_priv = dev->dev_private;
1953         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1954         int ret = 0;
1955
1956 #if WATCH_BUF
1957         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1958         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1959 #endif
1960         if (obj_priv->gtt_space == NULL)
1961                 return 0;
1962
1963         if (obj_priv->pin_count != 0) {
1964                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1965                 return -EINVAL;
1966         }
1967
1968         /* blow away mappings if mapped through GTT */
1969         i915_gem_release_mmap(obj);
1970
1971         /* Move the object to the CPU domain to ensure that
1972          * any possible CPU writes while it's not in the GTT
1973          * are flushed when we go to remap it. This will
1974          * also ensure that all pending GPU writes are finished
1975          * before we unbind.
1976          */
1977         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1978         if (ret == -ERESTARTSYS)
1979                 return ret;
1980         /* Continue on if we fail due to EIO, the GPU is hung so we
1981          * should be safe and we need to cleanup or else we might
1982          * cause memory corruption through use-after-free.
1983          */
1984
1985         /* release the fence reg _after_ flushing */
1986         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1987                 i915_gem_clear_fence_reg(obj);
1988
1989         if (obj_priv->agp_mem != NULL) {
1990                 drm_unbind_agp(obj_priv->agp_mem);
1991                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1992                 obj_priv->agp_mem = NULL;
1993         }
1994
1995         i915_gem_object_put_pages(obj);
1996         BUG_ON(obj_priv->pages_refcount);
1997
1998         if (obj_priv->gtt_space) {
1999                 atomic_dec(&dev->gtt_count);
2000                 atomic_sub(obj->size, &dev->gtt_memory);
2001
2002                 drm_mm_put_block(obj_priv->gtt_space);
2003                 obj_priv->gtt_space = NULL;
2004         }
2005
2006         /* Remove ourselves from the LRU list if present. */
2007         spin_lock(&dev_priv->mm.active_list_lock);
2008         if (!list_empty(&obj_priv->list))
2009                 list_del_init(&obj_priv->list);
2010         spin_unlock(&dev_priv->mm.active_list_lock);
2011
2012         if (i915_gem_object_is_purgeable(obj_priv))
2013                 i915_gem_object_truncate(obj);
2014
2015         trace_i915_gem_object_unbind(obj);
2016
2017         return ret;
2018 }
2019
2020 int
2021 i915_gpu_idle(struct drm_device *dev)
2022 {
2023         drm_i915_private_t *dev_priv = dev->dev_private;
2024         bool lists_empty;
2025         uint32_t seqno1, seqno2;
2026         int ret;
2027
2028         spin_lock(&dev_priv->mm.active_list_lock);
2029         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2030                        list_empty(&dev_priv->render_ring.active_list) &&
2031                        (!HAS_BSD(dev) ||
2032                         list_empty(&dev_priv->bsd_ring.active_list)));
2033         spin_unlock(&dev_priv->mm.active_list_lock);
2034
2035         if (lists_empty)
2036                 return 0;
2037
2038         /* Flush everything onto the inactive list. */
2039         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2040         seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2041                         &dev_priv->render_ring);
2042         if (seqno1 == 0)
2043                 return -ENOMEM;
2044         ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2045
2046         if (HAS_BSD(dev)) {
2047                 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2048                                 &dev_priv->bsd_ring);
2049                 if (seqno2 == 0)
2050                         return -ENOMEM;
2051
2052                 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2053                 if (ret)
2054                         return ret;
2055         }
2056
2057
2058         return ret;
2059 }
2060
2061 int
2062 i915_gem_object_get_pages(struct drm_gem_object *obj,
2063                           gfp_t gfpmask)
2064 {
2065         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2066         int page_count, i;
2067         struct address_space *mapping;
2068         struct inode *inode;
2069         struct page *page;
2070
2071         BUG_ON(obj_priv->pages_refcount
2072                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2073
2074         if (obj_priv->pages_refcount++ != 0)
2075                 return 0;
2076
2077         /* Get the list of pages out of our struct file.  They'll be pinned
2078          * at this point until we release them.
2079          */
2080         page_count = obj->size / PAGE_SIZE;
2081         BUG_ON(obj_priv->pages != NULL);
2082         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2083         if (obj_priv->pages == NULL) {
2084                 obj_priv->pages_refcount--;
2085                 return -ENOMEM;
2086         }
2087
2088         inode = obj->filp->f_path.dentry->d_inode;
2089         mapping = inode->i_mapping;
2090         for (i = 0; i < page_count; i++) {
2091                 page = read_cache_page_gfp(mapping, i,
2092                                            GFP_HIGHUSER |
2093                                            __GFP_COLD |
2094                                            __GFP_RECLAIMABLE |
2095                                            gfpmask);
2096                 if (IS_ERR(page))
2097                         goto err_pages;
2098
2099                 obj_priv->pages[i] = page;
2100         }
2101
2102         if (obj_priv->tiling_mode != I915_TILING_NONE)
2103                 i915_gem_object_do_bit_17_swizzle(obj);
2104
2105         return 0;
2106
2107 err_pages:
2108         while (i--)
2109                 page_cache_release(obj_priv->pages[i]);
2110
2111         drm_free_large(obj_priv->pages);
2112         obj_priv->pages = NULL;
2113         obj_priv->pages_refcount--;
2114         return PTR_ERR(page);
2115 }
2116
2117 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2118 {
2119         struct drm_gem_object *obj = reg->obj;
2120         struct drm_device *dev = obj->dev;
2121         drm_i915_private_t *dev_priv = dev->dev_private;
2122         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2123         int regnum = obj_priv->fence_reg;
2124         uint64_t val;
2125
2126         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2127                     0xfffff000) << 32;
2128         val |= obj_priv->gtt_offset & 0xfffff000;
2129         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2130                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2131
2132         if (obj_priv->tiling_mode == I915_TILING_Y)
2133                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2134         val |= I965_FENCE_REG_VALID;
2135
2136         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2137 }
2138
2139 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2140 {
2141         struct drm_gem_object *obj = reg->obj;
2142         struct drm_device *dev = obj->dev;
2143         drm_i915_private_t *dev_priv = dev->dev_private;
2144         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2145         int regnum = obj_priv->fence_reg;
2146         uint64_t val;
2147
2148         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2149                     0xfffff000) << 32;
2150         val |= obj_priv->gtt_offset & 0xfffff000;
2151         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2152         if (obj_priv->tiling_mode == I915_TILING_Y)
2153                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2154         val |= I965_FENCE_REG_VALID;
2155
2156         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2157 }
2158
2159 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2160 {
2161         struct drm_gem_object *obj = reg->obj;
2162         struct drm_device *dev = obj->dev;
2163         drm_i915_private_t *dev_priv = dev->dev_private;
2164         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2165         int regnum = obj_priv->fence_reg;
2166         int tile_width;
2167         uint32_t fence_reg, val;
2168         uint32_t pitch_val;
2169
2170         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2171             (obj_priv->gtt_offset & (obj->size - 1))) {
2172                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2173                      __func__, obj_priv->gtt_offset, obj->size);
2174                 return;
2175         }
2176
2177         if (obj_priv->tiling_mode == I915_TILING_Y &&
2178             HAS_128_BYTE_Y_TILING(dev))
2179                 tile_width = 128;
2180         else
2181                 tile_width = 512;
2182
2183         /* Note: pitch better be a power of two tile widths */
2184         pitch_val = obj_priv->stride / tile_width;
2185         pitch_val = ffs(pitch_val) - 1;
2186
2187         if (obj_priv->tiling_mode == I915_TILING_Y &&
2188             HAS_128_BYTE_Y_TILING(dev))
2189                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2190         else
2191                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2192
2193         val = obj_priv->gtt_offset;
2194         if (obj_priv->tiling_mode == I915_TILING_Y)
2195                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2196         val |= I915_FENCE_SIZE_BITS(obj->size);
2197         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2198         val |= I830_FENCE_REG_VALID;
2199
2200         if (regnum < 8)
2201                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2202         else
2203                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2204         I915_WRITE(fence_reg, val);
2205 }
2206
2207 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2208 {
2209         struct drm_gem_object *obj = reg->obj;
2210         struct drm_device *dev = obj->dev;
2211         drm_i915_private_t *dev_priv = dev->dev_private;
2212         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2213         int regnum = obj_priv->fence_reg;
2214         uint32_t val;
2215         uint32_t pitch_val;
2216         uint32_t fence_size_bits;
2217
2218         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2219             (obj_priv->gtt_offset & (obj->size - 1))) {
2220                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2221                      __func__, obj_priv->gtt_offset);
2222                 return;
2223         }
2224
2225         pitch_val = obj_priv->stride / 128;
2226         pitch_val = ffs(pitch_val) - 1;
2227         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2228
2229         val = obj_priv->gtt_offset;
2230         if (obj_priv->tiling_mode == I915_TILING_Y)
2231                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2232         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2233         WARN_ON(fence_size_bits & ~0x00000f00);
2234         val |= fence_size_bits;
2235         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2236         val |= I830_FENCE_REG_VALID;
2237
2238         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2239 }
2240
2241 static int i915_find_fence_reg(struct drm_device *dev)
2242 {
2243         struct drm_i915_fence_reg *reg = NULL;
2244         struct drm_i915_gem_object *obj_priv = NULL;
2245         struct drm_i915_private *dev_priv = dev->dev_private;
2246         struct drm_gem_object *obj = NULL;
2247         int i, avail, ret;
2248
2249         /* First try to find a free reg */
2250         avail = 0;
2251         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2252                 reg = &dev_priv->fence_regs[i];
2253                 if (!reg->obj)
2254                         return i;
2255
2256                 obj_priv = to_intel_bo(reg->obj);
2257                 if (!obj_priv->pin_count)
2258                     avail++;
2259         }
2260
2261         if (avail == 0)
2262                 return -ENOSPC;
2263
2264         /* None available, try to steal one or wait for a user to finish */
2265         i = I915_FENCE_REG_NONE;
2266         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2267                             lru_list) {
2268                 obj = reg->obj;
2269                 obj_priv = to_intel_bo(obj);
2270
2271                 if (obj_priv->pin_count)
2272                         continue;
2273
2274                 /* found one! */
2275                 i = obj_priv->fence_reg;
2276                 break;
2277         }
2278
2279         BUG_ON(i == I915_FENCE_REG_NONE);
2280
2281         /* We only have a reference on obj from the active list. put_fence_reg
2282          * might drop that one, causing a use-after-free in it. So hold a
2283          * private reference to obj like the other callers of put_fence_reg
2284          * (set_tiling ioctl) do. */
2285         drm_gem_object_reference(obj);
2286         ret = i915_gem_object_put_fence_reg(obj);
2287         drm_gem_object_unreference(obj);
2288         if (ret != 0)
2289                 return ret;
2290
2291         return i;
2292 }
2293
2294 /**
2295  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2296  * @obj: object to map through a fence reg
2297  *
2298  * When mapping objects through the GTT, userspace wants to be able to write
2299  * to them without having to worry about swizzling if the object is tiled.
2300  *
2301  * This function walks the fence regs looking for a free one for @obj,
2302  * stealing one if it can't find any.
2303  *
2304  * It then sets up the reg based on the object's properties: address, pitch
2305  * and tiling format.
2306  */
2307 int
2308 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2309 {
2310         struct drm_device *dev = obj->dev;
2311         struct drm_i915_private *dev_priv = dev->dev_private;
2312         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2313         struct drm_i915_fence_reg *reg = NULL;
2314         int ret;
2315
2316         /* Just update our place in the LRU if our fence is getting used. */
2317         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2318                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2319                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2320                 return 0;
2321         }
2322
2323         switch (obj_priv->tiling_mode) {
2324         case I915_TILING_NONE:
2325                 WARN(1, "allocating a fence for non-tiled object?\n");
2326                 break;
2327         case I915_TILING_X:
2328                 if (!obj_priv->stride)
2329                         return -EINVAL;
2330                 WARN((obj_priv->stride & (512 - 1)),
2331                      "object 0x%08x is X tiled but has non-512B pitch\n",
2332                      obj_priv->gtt_offset);
2333                 break;
2334         case I915_TILING_Y:
2335                 if (!obj_priv->stride)
2336                         return -EINVAL;
2337                 WARN((obj_priv->stride & (128 - 1)),
2338                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2339                      obj_priv->gtt_offset);
2340                 break;
2341         }
2342
2343         ret = i915_find_fence_reg(dev);
2344         if (ret < 0)
2345                 return ret;
2346
2347         obj_priv->fence_reg = ret;
2348         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2349         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2350
2351         reg->obj = obj;
2352
2353         if (IS_GEN6(dev))
2354                 sandybridge_write_fence_reg(reg);
2355         else if (IS_I965G(dev))
2356                 i965_write_fence_reg(reg);
2357         else if (IS_I9XX(dev))
2358                 i915_write_fence_reg(reg);
2359         else
2360                 i830_write_fence_reg(reg);
2361
2362         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2363                         obj_priv->tiling_mode);
2364
2365         return 0;
2366 }
2367
2368 /**
2369  * i915_gem_clear_fence_reg - clear out fence register info
2370  * @obj: object to clear
2371  *
2372  * Zeroes out the fence register itself and clears out the associated
2373  * data structures in dev_priv and obj_priv.
2374  */
2375 static void
2376 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2377 {
2378         struct drm_device *dev = obj->dev;
2379         drm_i915_private_t *dev_priv = dev->dev_private;
2380         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2381         struct drm_i915_fence_reg *reg =
2382                 &dev_priv->fence_regs[obj_priv->fence_reg];
2383
2384         if (IS_GEN6(dev)) {
2385                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2386                              (obj_priv->fence_reg * 8), 0);
2387         } else if (IS_I965G(dev)) {
2388                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2389         } else {
2390                 uint32_t fence_reg;
2391
2392                 if (obj_priv->fence_reg < 8)
2393                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2394                 else
2395                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2396                                                        8) * 4;
2397
2398                 I915_WRITE(fence_reg, 0);
2399         }
2400
2401         reg->obj = NULL;
2402         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2403         list_del_init(&reg->lru_list);
2404 }
2405
2406 /**
2407  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2408  * to the buffer to finish, and then resets the fence register.
2409  * @obj: tiled object holding a fence register.
2410  *
2411  * Zeroes out the fence register itself and clears out the associated
2412  * data structures in dev_priv and obj_priv.
2413  */
2414 int
2415 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2416 {
2417         struct drm_device *dev = obj->dev;
2418         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2419
2420         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2421                 return 0;
2422
2423         /* If we've changed tiling, GTT-mappings of the object
2424          * need to re-fault to ensure that the correct fence register
2425          * setup is in place.
2426          */
2427         i915_gem_release_mmap(obj);
2428
2429         /* On the i915, GPU access to tiled buffers is via a fence,
2430          * therefore we must wait for any outstanding access to complete
2431          * before clearing the fence.
2432          */
2433         if (!IS_I965G(dev)) {
2434                 int ret;
2435
2436                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2437                 if (ret != 0)
2438                         return ret;
2439
2440                 ret = i915_gem_object_wait_rendering(obj);
2441                 if (ret != 0)
2442                         return ret;
2443         }
2444
2445         i915_gem_object_flush_gtt_write_domain(obj);
2446         i915_gem_clear_fence_reg (obj);
2447
2448         return 0;
2449 }
2450
2451 /**
2452  * Finds free space in the GTT aperture and binds the object there.
2453  */
2454 static int
2455 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2456 {
2457         struct drm_device *dev = obj->dev;
2458         drm_i915_private_t *dev_priv = dev->dev_private;
2459         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2460         struct drm_mm_node *free_space;
2461         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2462         int ret;
2463
2464         if (obj_priv->madv != I915_MADV_WILLNEED) {
2465                 DRM_ERROR("Attempting to bind a purgeable object\n");
2466                 return -EINVAL;
2467         }
2468
2469         if (alignment == 0)
2470                 alignment = i915_gem_get_gtt_alignment(obj);
2471         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2472                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2473                 return -EINVAL;
2474         }
2475
2476         /* If the object is bigger than the entire aperture, reject it early
2477          * before evicting everything in a vain attempt to find space.
2478          */
2479         if (obj->size > dev->gtt_total) {
2480                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2481                 return -E2BIG;
2482         }
2483
2484  search_free:
2485         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2486                                         obj->size, alignment, 0);
2487         if (free_space != NULL) {
2488                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2489                                                        alignment);
2490                 if (obj_priv->gtt_space != NULL)
2491                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2492         }
2493         if (obj_priv->gtt_space == NULL) {
2494                 /* If the gtt is empty and we're still having trouble
2495                  * fitting our object in, we're out of memory.
2496                  */
2497 #if WATCH_LRU
2498                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2499 #endif
2500                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2501                 if (ret)
2502                         return ret;
2503
2504                 goto search_free;
2505         }
2506
2507 #if WATCH_BUF
2508         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2509                  obj->size, obj_priv->gtt_offset);
2510 #endif
2511         ret = i915_gem_object_get_pages(obj, gfpmask);
2512         if (ret) {
2513                 drm_mm_put_block(obj_priv->gtt_space);
2514                 obj_priv->gtt_space = NULL;
2515
2516                 if (ret == -ENOMEM) {
2517                         /* first try to clear up some space from the GTT */
2518                         ret = i915_gem_evict_something(dev, obj->size,
2519                                                        alignment);
2520                         if (ret) {
2521                                 /* now try to shrink everyone else */
2522                                 if (gfpmask) {
2523                                         gfpmask = 0;
2524                                         goto search_free;
2525                                 }
2526
2527                                 return ret;
2528                         }
2529
2530                         goto search_free;
2531                 }
2532
2533                 return ret;
2534         }
2535
2536         /* Create an AGP memory structure pointing at our pages, and bind it
2537          * into the GTT.
2538          */
2539         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2540                                                obj_priv->pages,
2541                                                obj->size >> PAGE_SHIFT,
2542                                                obj_priv->gtt_offset,
2543                                                obj_priv->agp_type);
2544         if (obj_priv->agp_mem == NULL) {
2545                 i915_gem_object_put_pages(obj);
2546                 drm_mm_put_block(obj_priv->gtt_space);
2547                 obj_priv->gtt_space = NULL;
2548
2549                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2550                 if (ret)
2551                         return ret;
2552
2553                 goto search_free;
2554         }
2555         atomic_inc(&dev->gtt_count);
2556         atomic_add(obj->size, &dev->gtt_memory);
2557
2558         /* keep track of bounds object by adding it to the inactive list */
2559         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2560
2561         /* Assert that the object is not currently in any GPU domain. As it
2562          * wasn't in the GTT, there shouldn't be any way it could have been in
2563          * a GPU cache
2564          */
2565         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2566         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2567
2568         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2569
2570         return 0;
2571 }
2572
2573 void
2574 i915_gem_clflush_object(struct drm_gem_object *obj)
2575 {
2576         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2577
2578         /* If we don't have a page list set up, then we're not pinned
2579          * to GPU, and we can ignore the cache flush because it'll happen
2580          * again at bind time.
2581          */
2582         if (obj_priv->pages == NULL)
2583                 return;
2584
2585         trace_i915_gem_object_clflush(obj);
2586
2587         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2588 }
2589
2590 /** Flushes any GPU write domain for the object if it's dirty. */
2591 static int
2592 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2593 {
2594         struct drm_device *dev = obj->dev;
2595         uint32_t old_write_domain;
2596         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2597
2598         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2599                 return 0;
2600
2601         /* Queue the GPU write cache flushing we need. */
2602         old_write_domain = obj->write_domain;
2603         i915_gem_flush(dev, 0, obj->write_domain);
2604         if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2605                 return -ENOMEM;
2606
2607         trace_i915_gem_object_change_domain(obj,
2608                                             obj->read_domains,
2609                                             old_write_domain);
2610         return 0;
2611 }
2612
2613 /** Flushes the GTT write domain for the object if it's dirty. */
2614 static void
2615 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2616 {
2617         uint32_t old_write_domain;
2618
2619         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2620                 return;
2621
2622         /* No actual flushing is required for the GTT write domain.   Writes
2623          * to it immediately go to main memory as far as we know, so there's
2624          * no chipset flush.  It also doesn't land in render cache.
2625          */
2626         old_write_domain = obj->write_domain;
2627         obj->write_domain = 0;
2628
2629         trace_i915_gem_object_change_domain(obj,
2630                                             obj->read_domains,
2631                                             old_write_domain);
2632 }
2633
2634 /** Flushes the CPU write domain for the object if it's dirty. */
2635 static void
2636 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2637 {
2638         struct drm_device *dev = obj->dev;
2639         uint32_t old_write_domain;
2640
2641         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2642                 return;
2643
2644         i915_gem_clflush_object(obj);
2645         drm_agp_chipset_flush(dev);
2646         old_write_domain = obj->write_domain;
2647         obj->write_domain = 0;
2648
2649         trace_i915_gem_object_change_domain(obj,
2650                                             obj->read_domains,
2651                                             old_write_domain);
2652 }
2653
2654 int
2655 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2656 {
2657         int ret = 0;
2658
2659         switch (obj->write_domain) {
2660         case I915_GEM_DOMAIN_GTT:
2661                 i915_gem_object_flush_gtt_write_domain(obj);
2662                 break;
2663         case I915_GEM_DOMAIN_CPU:
2664                 i915_gem_object_flush_cpu_write_domain(obj);
2665                 break;
2666         default:
2667                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2668                 break;
2669         }
2670
2671         return ret;
2672 }
2673
2674 /**
2675  * Moves a single object to the GTT read, and possibly write domain.
2676  *
2677  * This function returns when the move is complete, including waiting on
2678  * flushes to occur.
2679  */
2680 int
2681 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2682 {
2683         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2684         uint32_t old_write_domain, old_read_domains;
2685         int ret;
2686
2687         /* Not valid to be called on unbound objects. */
2688         if (obj_priv->gtt_space == NULL)
2689                 return -EINVAL;
2690
2691         ret = i915_gem_object_flush_gpu_write_domain(obj);
2692         if (ret != 0)
2693                 return ret;
2694
2695         /* Wait on any GPU rendering and flushing to occur. */
2696         ret = i915_gem_object_wait_rendering(obj);
2697         if (ret != 0)
2698                 return ret;
2699
2700         old_write_domain = obj->write_domain;
2701         old_read_domains = obj->read_domains;
2702
2703         /* If we're writing through the GTT domain, then CPU and GPU caches
2704          * will need to be invalidated at next use.
2705          */
2706         if (write)
2707                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2708
2709         i915_gem_object_flush_cpu_write_domain(obj);
2710
2711         /* It should now be out of any other write domains, and we can update
2712          * the domain values for our changes.
2713          */
2714         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2715         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2716         if (write) {
2717                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2718                 obj_priv->dirty = 1;
2719         }
2720
2721         trace_i915_gem_object_change_domain(obj,
2722                                             old_read_domains,
2723                                             old_write_domain);
2724
2725         return 0;
2726 }
2727
2728 /*
2729  * Prepare buffer for display plane. Use uninterruptible for possible flush
2730  * wait, as in modesetting process we're not supposed to be interrupted.
2731  */
2732 int
2733 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2734 {
2735         struct drm_device *dev = obj->dev;
2736         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2737         uint32_t old_write_domain, old_read_domains;
2738         int ret;
2739
2740         /* Not valid to be called on unbound objects. */
2741         if (obj_priv->gtt_space == NULL)
2742                 return -EINVAL;
2743
2744         ret = i915_gem_object_flush_gpu_write_domain(obj);
2745         if (ret)
2746                 return ret;
2747
2748         /* Wait on any GPU rendering and flushing to occur. */
2749         if (obj_priv->active) {
2750 #if WATCH_BUF
2751                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2752                           __func__, obj, obj_priv->last_rendering_seqno);
2753 #endif
2754                 ret = i915_do_wait_request(dev,
2755                                 obj_priv->last_rendering_seqno,
2756                                 0,
2757                                 obj_priv->ring);
2758                 if (ret != 0)
2759                         return ret;
2760         }
2761
2762         i915_gem_object_flush_cpu_write_domain(obj);
2763
2764         old_write_domain = obj->write_domain;
2765         old_read_domains = obj->read_domains;
2766
2767         /* It should now be out of any other write domains, and we can update
2768          * the domain values for our changes.
2769          */
2770         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2771         obj->read_domains = I915_GEM_DOMAIN_GTT;
2772         obj->write_domain = I915_GEM_DOMAIN_GTT;
2773         obj_priv->dirty = 1;
2774
2775         trace_i915_gem_object_change_domain(obj,
2776                                             old_read_domains,
2777                                             old_write_domain);
2778
2779         return 0;
2780 }
2781
2782 /**
2783  * Moves a single object to the CPU read, and possibly write domain.
2784  *
2785  * This function returns when the move is complete, including waiting on
2786  * flushes to occur.
2787  */
2788 static int
2789 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2790 {
2791         uint32_t old_write_domain, old_read_domains;
2792         int ret;
2793
2794         ret = i915_gem_object_flush_gpu_write_domain(obj);
2795         if (ret)
2796                 return ret;
2797
2798         /* Wait on any GPU rendering and flushing to occur. */
2799         ret = i915_gem_object_wait_rendering(obj);
2800         if (ret != 0)
2801                 return ret;
2802
2803         i915_gem_object_flush_gtt_write_domain(obj);
2804
2805         /* If we have a partially-valid cache of the object in the CPU,
2806          * finish invalidating it and free the per-page flags.
2807          */
2808         i915_gem_object_set_to_full_cpu_read_domain(obj);
2809
2810         old_write_domain = obj->write_domain;
2811         old_read_domains = obj->read_domains;
2812
2813         /* Flush the CPU cache if it's still invalid. */
2814         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2815                 i915_gem_clflush_object(obj);
2816
2817                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2818         }
2819
2820         /* It should now be out of any other write domains, and we can update
2821          * the domain values for our changes.
2822          */
2823         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2824
2825         /* If we're writing through the CPU, then the GPU read domains will
2826          * need to be invalidated at next use.
2827          */
2828         if (write) {
2829                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2830                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2831         }
2832
2833         trace_i915_gem_object_change_domain(obj,
2834                                             old_read_domains,
2835                                             old_write_domain);
2836
2837         return 0;
2838 }
2839
2840 /*
2841  * Set the next domain for the specified object. This
2842  * may not actually perform the necessary flushing/invaliding though,
2843  * as that may want to be batched with other set_domain operations
2844  *
2845  * This is (we hope) the only really tricky part of gem. The goal
2846  * is fairly simple -- track which caches hold bits of the object
2847  * and make sure they remain coherent. A few concrete examples may
2848  * help to explain how it works. For shorthand, we use the notation
2849  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2850  * a pair of read and write domain masks.
2851  *
2852  * Case 1: the batch buffer
2853  *
2854  *      1. Allocated
2855  *      2. Written by CPU
2856  *      3. Mapped to GTT
2857  *      4. Read by GPU
2858  *      5. Unmapped from GTT
2859  *      6. Freed
2860  *
2861  *      Let's take these a step at a time
2862  *
2863  *      1. Allocated
2864  *              Pages allocated from the kernel may still have
2865  *              cache contents, so we set them to (CPU, CPU) always.
2866  *      2. Written by CPU (using pwrite)
2867  *              The pwrite function calls set_domain (CPU, CPU) and
2868  *              this function does nothing (as nothing changes)
2869  *      3. Mapped by GTT
2870  *              This function asserts that the object is not
2871  *              currently in any GPU-based read or write domains
2872  *      4. Read by GPU
2873  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2874  *              As write_domain is zero, this function adds in the
2875  *              current read domains (CPU+COMMAND, 0).
2876  *              flush_domains is set to CPU.
2877  *              invalidate_domains is set to COMMAND
2878  *              clflush is run to get data out of the CPU caches
2879  *              then i915_dev_set_domain calls i915_gem_flush to
2880  *              emit an MI_FLUSH and drm_agp_chipset_flush
2881  *      5. Unmapped from GTT
2882  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2883  *              flush_domains and invalidate_domains end up both zero
2884  *              so no flushing/invalidating happens
2885  *      6. Freed
2886  *              yay, done
2887  *
2888  * Case 2: The shared render buffer
2889  *
2890  *      1. Allocated
2891  *      2. Mapped to GTT
2892  *      3. Read/written by GPU
2893  *      4. set_domain to (CPU,CPU)
2894  *      5. Read/written by CPU
2895  *      6. Read/written by GPU
2896  *
2897  *      1. Allocated
2898  *              Same as last example, (CPU, CPU)
2899  *      2. Mapped to GTT
2900  *              Nothing changes (assertions find that it is not in the GPU)
2901  *      3. Read/written by GPU
2902  *              execbuffer calls set_domain (RENDER, RENDER)
2903  *              flush_domains gets CPU
2904  *              invalidate_domains gets GPU
2905  *              clflush (obj)
2906  *              MI_FLUSH and drm_agp_chipset_flush
2907  *      4. set_domain (CPU, CPU)
2908  *              flush_domains gets GPU
2909  *              invalidate_domains gets CPU
2910  *              wait_rendering (obj) to make sure all drawing is complete.
2911  *              This will include an MI_FLUSH to get the data from GPU
2912  *              to memory
2913  *              clflush (obj) to invalidate the CPU cache
2914  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2915  *      5. Read/written by CPU
2916  *              cache lines are loaded and dirtied
2917  *      6. Read written by GPU
2918  *              Same as last GPU access
2919  *
2920  * Case 3: The constant buffer
2921  *
2922  *      1. Allocated
2923  *      2. Written by CPU
2924  *      3. Read by GPU
2925  *      4. Updated (written) by CPU again
2926  *      5. Read by GPU
2927  *
2928  *      1. Allocated
2929  *              (CPU, CPU)
2930  *      2. Written by CPU
2931  *              (CPU, CPU)
2932  *      3. Read by GPU
2933  *              (CPU+RENDER, 0)
2934  *              flush_domains = CPU
2935  *              invalidate_domains = RENDER
2936  *              clflush (obj)
2937  *              MI_FLUSH
2938  *              drm_agp_chipset_flush
2939  *      4. Updated (written) by CPU again
2940  *              (CPU, CPU)
2941  *              flush_domains = 0 (no previous write domain)
2942  *              invalidate_domains = 0 (no new read domains)
2943  *      5. Read by GPU
2944  *              (CPU+RENDER, 0)
2945  *              flush_domains = CPU
2946  *              invalidate_domains = RENDER
2947  *              clflush (obj)
2948  *              MI_FLUSH
2949  *              drm_agp_chipset_flush
2950  */
2951 static void
2952 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2953 {
2954         struct drm_device               *dev = obj->dev;
2955         drm_i915_private_t              *dev_priv = dev->dev_private;
2956         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2957         uint32_t                        invalidate_domains = 0;
2958         uint32_t                        flush_domains = 0;
2959         uint32_t                        old_read_domains;
2960
2961         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2962         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2963
2964         intel_mark_busy(dev, obj);
2965
2966 #if WATCH_BUF
2967         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2968                  __func__, obj,
2969                  obj->read_domains, obj->pending_read_domains,
2970                  obj->write_domain, obj->pending_write_domain);
2971 #endif
2972         /*
2973          * If the object isn't moving to a new write domain,
2974          * let the object stay in multiple read domains
2975          */
2976         if (obj->pending_write_domain == 0)
2977                 obj->pending_read_domains |= obj->read_domains;
2978         else
2979                 obj_priv->dirty = 1;
2980
2981         /*
2982          * Flush the current write domain if
2983          * the new read domains don't match. Invalidate
2984          * any read domains which differ from the old
2985          * write domain
2986          */
2987         if (obj->write_domain &&
2988             obj->write_domain != obj->pending_read_domains) {
2989                 flush_domains |= obj->write_domain;
2990                 invalidate_domains |=
2991                         obj->pending_read_domains & ~obj->write_domain;
2992         }
2993         /*
2994          * Invalidate any read caches which may have
2995          * stale data. That is, any new read domains.
2996          */
2997         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
2998         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
2999 #if WATCH_BUF
3000                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3001                          __func__, flush_domains, invalidate_domains);
3002 #endif
3003                 i915_gem_clflush_object(obj);
3004         }
3005
3006         old_read_domains = obj->read_domains;
3007
3008         /* The actual obj->write_domain will be updated with
3009          * pending_write_domain after we emit the accumulated flush for all
3010          * of our domain changes in execbuffers (which clears objects'
3011          * write_domains).  So if we have a current write domain that we
3012          * aren't changing, set pending_write_domain to that.
3013          */
3014         if (flush_domains == 0 && obj->pending_write_domain == 0)
3015                 obj->pending_write_domain = obj->write_domain;
3016         obj->read_domains = obj->pending_read_domains;
3017
3018         if (flush_domains & I915_GEM_GPU_DOMAINS) {
3019                 if (obj_priv->ring == &dev_priv->render_ring)
3020                         dev_priv->flush_rings |= FLUSH_RENDER_RING;
3021                 else if (obj_priv->ring == &dev_priv->bsd_ring)
3022                         dev_priv->flush_rings |= FLUSH_BSD_RING;
3023         }
3024
3025         dev->invalidate_domains |= invalidate_domains;
3026         dev->flush_domains |= flush_domains;
3027 #if WATCH_BUF
3028         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3029                  __func__,
3030                  obj->read_domains, obj->write_domain,
3031                  dev->invalidate_domains, dev->flush_domains);
3032 #endif
3033
3034         trace_i915_gem_object_change_domain(obj,
3035                                             old_read_domains,
3036                                             obj->write_domain);
3037 }
3038
3039 /**
3040  * Moves the object from a partially CPU read to a full one.
3041  *
3042  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3043  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3044  */
3045 static void
3046 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3047 {
3048         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3049
3050         if (!obj_priv->page_cpu_valid)
3051                 return;
3052
3053         /* If we're partially in the CPU read domain, finish moving it in.
3054          */
3055         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3056                 int i;
3057
3058                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3059                         if (obj_priv->page_cpu_valid[i])
3060                                 continue;
3061                         drm_clflush_pages(obj_priv->pages + i, 1);
3062                 }
3063         }
3064
3065         /* Free the page_cpu_valid mappings which are now stale, whether
3066          * or not we've got I915_GEM_DOMAIN_CPU.
3067          */
3068         kfree(obj_priv->page_cpu_valid);
3069         obj_priv->page_cpu_valid = NULL;
3070 }
3071
3072 /**
3073  * Set the CPU read domain on a range of the object.
3074  *
3075  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3076  * not entirely valid.  The page_cpu_valid member of the object flags which
3077  * pages have been flushed, and will be respected by
3078  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3079  * of the whole object.
3080  *
3081  * This function returns when the move is complete, including waiting on
3082  * flushes to occur.
3083  */
3084 static int
3085 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3086                                           uint64_t offset, uint64_t size)
3087 {
3088         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3089         uint32_t old_read_domains;
3090         int i, ret;
3091
3092         if (offset == 0 && size == obj->size)
3093                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3094
3095         ret = i915_gem_object_flush_gpu_write_domain(obj);
3096         if (ret)
3097                 return ret;
3098
3099         /* Wait on any GPU rendering and flushing to occur. */
3100         ret = i915_gem_object_wait_rendering(obj);
3101         if (ret != 0)
3102                 return ret;
3103         i915_gem_object_flush_gtt_write_domain(obj);
3104
3105         /* If we're already fully in the CPU read domain, we're done. */
3106         if (obj_priv->page_cpu_valid == NULL &&
3107             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3108                 return 0;
3109
3110         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3111          * newly adding I915_GEM_DOMAIN_CPU
3112          */
3113         if (obj_priv->page_cpu_valid == NULL) {
3114                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3115                                                    GFP_KERNEL);
3116                 if (obj_priv->page_cpu_valid == NULL)
3117                         return -ENOMEM;
3118         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3119                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3120
3121         /* Flush the cache on any pages that are still invalid from the CPU's
3122          * perspective.
3123          */
3124         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3125              i++) {
3126                 if (obj_priv->page_cpu_valid[i])
3127                         continue;
3128
3129                 drm_clflush_pages(obj_priv->pages + i, 1);
3130
3131                 obj_priv->page_cpu_valid[i] = 1;
3132         }
3133
3134         /* It should now be out of any other write domains, and we can update
3135          * the domain values for our changes.
3136          */
3137         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3138
3139         old_read_domains = obj->read_domains;
3140         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3141
3142         trace_i915_gem_object_change_domain(obj,
3143                                             old_read_domains,
3144                                             obj->write_domain);
3145
3146         return 0;
3147 }
3148
3149 /**
3150  * Pin an object to the GTT and evaluate the relocations landing in it.
3151  */
3152 static int
3153 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3154                                  struct drm_file *file_priv,
3155                                  struct drm_i915_gem_exec_object2 *entry,
3156                                  struct drm_i915_gem_relocation_entry *relocs)
3157 {
3158         struct drm_device *dev = obj->dev;
3159         drm_i915_private_t *dev_priv = dev->dev_private;
3160         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3161         int i, ret;
3162         void __iomem *reloc_page;
3163         bool need_fence;
3164
3165         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3166                      obj_priv->tiling_mode != I915_TILING_NONE;
3167
3168         /* Check fence reg constraints and rebind if necessary */
3169         if (need_fence &&
3170             !i915_gem_object_fence_offset_ok(obj,
3171                                              obj_priv->tiling_mode)) {
3172                 ret = i915_gem_object_unbind(obj);
3173                 if (ret)
3174                         return ret;
3175         }
3176
3177         /* Choose the GTT offset for our buffer and put it there. */
3178         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3179         if (ret)
3180                 return ret;
3181
3182         /*
3183          * Pre-965 chips need a fence register set up in order to
3184          * properly handle blits to/from tiled surfaces.
3185          */
3186         if (need_fence) {
3187                 ret = i915_gem_object_get_fence_reg(obj);
3188                 if (ret != 0) {
3189                         i915_gem_object_unpin(obj);
3190                         return ret;
3191                 }
3192         }
3193
3194         entry->offset = obj_priv->gtt_offset;
3195
3196         /* Apply the relocations, using the GTT aperture to avoid cache
3197          * flushing requirements.
3198          */
3199         for (i = 0; i < entry->relocation_count; i++) {
3200                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3201                 struct drm_gem_object *target_obj;
3202                 struct drm_i915_gem_object *target_obj_priv;
3203                 uint32_t reloc_val, reloc_offset;
3204                 uint32_t __iomem *reloc_entry;
3205
3206                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3207                                                    reloc->target_handle);
3208                 if (target_obj == NULL) {
3209                         i915_gem_object_unpin(obj);
3210                         return -ENOENT;
3211                 }
3212                 target_obj_priv = to_intel_bo(target_obj);
3213
3214 #if WATCH_RELOC
3215                 DRM_INFO("%s: obj %p offset %08x target %d "
3216                          "read %08x write %08x gtt %08x "
3217                          "presumed %08x delta %08x\n",
3218                          __func__,
3219                          obj,
3220                          (int) reloc->offset,
3221                          (int) reloc->target_handle,
3222                          (int) reloc->read_domains,
3223                          (int) reloc->write_domain,
3224                          (int) target_obj_priv->gtt_offset,
3225                          (int) reloc->presumed_offset,
3226                          reloc->delta);
3227 #endif
3228
3229                 /* The target buffer should have appeared before us in the
3230                  * exec_object list, so it should have a GTT space bound by now.
3231                  */
3232                 if (target_obj_priv->gtt_space == NULL) {
3233                         DRM_ERROR("No GTT space found for object %d\n",
3234                                   reloc->target_handle);
3235                         drm_gem_object_unreference(target_obj);
3236                         i915_gem_object_unpin(obj);
3237                         return -EINVAL;
3238                 }
3239
3240                 /* Validate that the target is in a valid r/w GPU domain */
3241                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3242                         DRM_ERROR("reloc with multiple write domains: "
3243                                   "obj %p target %d offset %d "
3244                                   "read %08x write %08x",
3245                                   obj, reloc->target_handle,
3246                                   (int) reloc->offset,
3247                                   reloc->read_domains,
3248                                   reloc->write_domain);
3249                         return -EINVAL;
3250                 }
3251                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3252                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3253                         DRM_ERROR("reloc with read/write CPU domains: "
3254                                   "obj %p target %d offset %d "
3255                                   "read %08x write %08x",
3256                                   obj, reloc->target_handle,
3257                                   (int) reloc->offset,
3258                                   reloc->read_domains,
3259                                   reloc->write_domain);
3260                         drm_gem_object_unreference(target_obj);
3261                         i915_gem_object_unpin(obj);
3262                         return -EINVAL;
3263                 }
3264                 if (reloc->write_domain && target_obj->pending_write_domain &&
3265                     reloc->write_domain != target_obj->pending_write_domain) {
3266                         DRM_ERROR("Write domain conflict: "
3267                                   "obj %p target %d offset %d "
3268                                   "new %08x old %08x\n",
3269                                   obj, reloc->target_handle,
3270                                   (int) reloc->offset,
3271                                   reloc->write_domain,
3272                                   target_obj->pending_write_domain);
3273                         drm_gem_object_unreference(target_obj);
3274                         i915_gem_object_unpin(obj);
3275                         return -EINVAL;
3276                 }
3277
3278                 target_obj->pending_read_domains |= reloc->read_domains;
3279                 target_obj->pending_write_domain |= reloc->write_domain;
3280
3281                 /* If the relocation already has the right value in it, no
3282                  * more work needs to be done.
3283                  */
3284                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3285                         drm_gem_object_unreference(target_obj);
3286                         continue;
3287                 }
3288
3289                 /* Check that the relocation address is valid... */
3290                 if (reloc->offset > obj->size - 4) {
3291                         DRM_ERROR("Relocation beyond object bounds: "
3292                                   "obj %p target %d offset %d size %d.\n",
3293                                   obj, reloc->target_handle,
3294                                   (int) reloc->offset, (int) obj->size);
3295                         drm_gem_object_unreference(target_obj);
3296                         i915_gem_object_unpin(obj);
3297                         return -EINVAL;
3298                 }
3299                 if (reloc->offset & 3) {
3300                         DRM_ERROR("Relocation not 4-byte aligned: "
3301                                   "obj %p target %d offset %d.\n",
3302                                   obj, reloc->target_handle,
3303                                   (int) reloc->offset);
3304                         drm_gem_object_unreference(target_obj);
3305                         i915_gem_object_unpin(obj);
3306                         return -EINVAL;
3307                 }
3308
3309                 /* and points to somewhere within the target object. */
3310                 if (reloc->delta >= target_obj->size) {
3311                         DRM_ERROR("Relocation beyond target object bounds: "
3312                                   "obj %p target %d delta %d size %d.\n",
3313                                   obj, reloc->target_handle,
3314                                   (int) reloc->delta, (int) target_obj->size);
3315                         drm_gem_object_unreference(target_obj);
3316                         i915_gem_object_unpin(obj);
3317                         return -EINVAL;
3318                 }
3319
3320                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3321                 if (ret != 0) {
3322                         drm_gem_object_unreference(target_obj);
3323                         i915_gem_object_unpin(obj);
3324                         return -EINVAL;
3325                 }
3326
3327                 /* Map the page containing the relocation we're going to
3328                  * perform.
3329                  */
3330                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3331                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3332                                                       (reloc_offset &
3333                                                        ~(PAGE_SIZE - 1)),
3334                                                       KM_USER0);
3335                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3336                                                    (reloc_offset & (PAGE_SIZE - 1)));
3337                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3338
3339 #if WATCH_BUF
3340                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3341                           obj, (unsigned int) reloc->offset,
3342                           readl(reloc_entry), reloc_val);
3343 #endif
3344                 writel(reloc_val, reloc_entry);
3345                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3346
3347                 /* The updated presumed offset for this entry will be
3348                  * copied back out to the user.
3349                  */
3350                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3351
3352                 drm_gem_object_unreference(target_obj);
3353         }
3354
3355 #if WATCH_BUF
3356         if (0)
3357                 i915_gem_dump_object(obj, 128, __func__, ~0);
3358 #endif
3359         return 0;
3360 }
3361
3362 /* Throttle our rendering by waiting until the ring has completed our requests
3363  * emitted over 20 msec ago.
3364  *
3365  * Note that if we were to use the current jiffies each time around the loop,
3366  * we wouldn't escape the function with any frames outstanding if the time to
3367  * render a frame was over 20ms.
3368  *
3369  * This should get us reasonable parallelism between CPU and GPU but also
3370  * relatively low latency when blocking on a particular request to finish.
3371  */
3372 static int
3373 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3374 {
3375         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3376         int ret = 0;
3377         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3378
3379         mutex_lock(&dev->struct_mutex);
3380         while (!list_empty(&i915_file_priv->mm.request_list)) {
3381                 struct drm_i915_gem_request *request;
3382
3383                 request = list_first_entry(&i915_file_priv->mm.request_list,
3384                                            struct drm_i915_gem_request,
3385                                            client_list);
3386
3387                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3388                         break;
3389
3390                 ret = i915_wait_request(dev, request->seqno, request->ring);
3391                 if (ret != 0)
3392                         break;
3393         }
3394         mutex_unlock(&dev->struct_mutex);
3395
3396         return ret;
3397 }
3398
3399 static int
3400 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3401                               uint32_t buffer_count,
3402                               struct drm_i915_gem_relocation_entry **relocs)
3403 {
3404         uint32_t reloc_count = 0, reloc_index = 0, i;
3405         int ret;
3406
3407         *relocs = NULL;
3408         for (i = 0; i < buffer_count; i++) {
3409                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3410                         return -EINVAL;
3411                 reloc_count += exec_list[i].relocation_count;
3412         }
3413
3414         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3415         if (*relocs == NULL) {
3416                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3417                 return -ENOMEM;
3418         }
3419
3420         for (i = 0; i < buffer_count; i++) {
3421                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3422
3423                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3424
3425                 ret = copy_from_user(&(*relocs)[reloc_index],
3426                                      user_relocs,
3427                                      exec_list[i].relocation_count *
3428                                      sizeof(**relocs));
3429                 if (ret != 0) {
3430                         drm_free_large(*relocs);
3431                         *relocs = NULL;
3432                         return -EFAULT;
3433                 }
3434
3435                 reloc_index += exec_list[i].relocation_count;
3436         }
3437
3438         return 0;
3439 }
3440
3441 static int
3442 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3443                             uint32_t buffer_count,
3444                             struct drm_i915_gem_relocation_entry *relocs)
3445 {
3446         uint32_t reloc_count = 0, i;
3447         int ret = 0;
3448
3449         if (relocs == NULL)
3450             return 0;
3451
3452         for (i = 0; i < buffer_count; i++) {
3453                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3454                 int unwritten;
3455
3456                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3457
3458                 unwritten = copy_to_user(user_relocs,
3459                                          &relocs[reloc_count],
3460                                          exec_list[i].relocation_count *
3461                                          sizeof(*relocs));
3462
3463                 if (unwritten) {
3464                         ret = -EFAULT;
3465                         goto err;
3466                 }
3467
3468                 reloc_count += exec_list[i].relocation_count;
3469         }
3470
3471 err:
3472         drm_free_large(relocs);
3473
3474         return ret;
3475 }
3476
3477 static int
3478 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3479                            uint64_t exec_offset)
3480 {
3481         uint32_t exec_start, exec_len;
3482
3483         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3484         exec_len = (uint32_t) exec->batch_len;
3485
3486         if ((exec_start | exec_len) & 0x7)
3487                 return -EINVAL;
3488
3489         if (!exec_start)
3490                 return -EINVAL;
3491
3492         return 0;
3493 }
3494
3495 static int
3496 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3497                                struct drm_gem_object **object_list,
3498                                int count)
3499 {
3500         drm_i915_private_t *dev_priv = dev->dev_private;
3501         struct drm_i915_gem_object *obj_priv;
3502         DEFINE_WAIT(wait);
3503         int i, ret = 0;
3504
3505         for (;;) {
3506                 prepare_to_wait(&dev_priv->pending_flip_queue,
3507                                 &wait, TASK_INTERRUPTIBLE);
3508                 for (i = 0; i < count; i++) {
3509                         obj_priv = to_intel_bo(object_list[i]);
3510                         if (atomic_read(&obj_priv->pending_flip) > 0)
3511                                 break;
3512                 }
3513                 if (i == count)
3514                         break;
3515
3516                 if (!signal_pending(current)) {
3517                         mutex_unlock(&dev->struct_mutex);
3518                         schedule();
3519                         mutex_lock(&dev->struct_mutex);
3520                         continue;
3521                 }
3522                 ret = -ERESTARTSYS;
3523                 break;
3524         }
3525         finish_wait(&dev_priv->pending_flip_queue, &wait);
3526
3527         return ret;
3528 }
3529
3530
3531 int
3532 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3533                        struct drm_file *file_priv,
3534                        struct drm_i915_gem_execbuffer2 *args,
3535                        struct drm_i915_gem_exec_object2 *exec_list)
3536 {
3537         drm_i915_private_t *dev_priv = dev->dev_private;
3538         struct drm_gem_object **object_list = NULL;
3539         struct drm_gem_object *batch_obj;
3540         struct drm_i915_gem_object *obj_priv;
3541         struct drm_clip_rect *cliprects = NULL;
3542         struct drm_i915_gem_relocation_entry *relocs = NULL;
3543         int ret = 0, ret2, i, pinned = 0;
3544         uint64_t exec_offset;
3545         uint32_t seqno, flush_domains, reloc_index;
3546         int pin_tries, flips;
3547
3548         struct intel_ring_buffer *ring = NULL;
3549
3550 #if WATCH_EXEC
3551         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3552                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3553 #endif
3554         if (args->flags & I915_EXEC_BSD) {
3555                 if (!HAS_BSD(dev)) {
3556                         DRM_ERROR("execbuf with wrong flag\n");
3557                         return -EINVAL;
3558                 }
3559                 ring = &dev_priv->bsd_ring;
3560         } else {
3561                 ring = &dev_priv->render_ring;
3562         }
3563
3564         if (args->buffer_count < 1) {
3565                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3566                 return -EINVAL;
3567         }
3568         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3569         if (object_list == NULL) {
3570                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3571                           args->buffer_count);
3572                 ret = -ENOMEM;
3573                 goto pre_mutex_err;
3574         }
3575
3576         if (args->num_cliprects != 0) {
3577                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3578                                     GFP_KERNEL);
3579                 if (cliprects == NULL) {
3580                         ret = -ENOMEM;
3581                         goto pre_mutex_err;
3582                 }
3583
3584                 ret = copy_from_user(cliprects,
3585                                      (struct drm_clip_rect __user *)
3586                                      (uintptr_t) args->cliprects_ptr,
3587                                      sizeof(*cliprects) * args->num_cliprects);
3588                 if (ret != 0) {
3589                         DRM_ERROR("copy %d cliprects failed: %d\n",
3590                                   args->num_cliprects, ret);
3591                         goto pre_mutex_err;
3592                 }
3593         }
3594
3595         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3596                                             &relocs);
3597         if (ret != 0)
3598                 goto pre_mutex_err;
3599
3600         mutex_lock(&dev->struct_mutex);
3601
3602         i915_verify_inactive(dev, __FILE__, __LINE__);
3603
3604         if (atomic_read(&dev_priv->mm.wedged)) {
3605                 mutex_unlock(&dev->struct_mutex);
3606                 ret = -EIO;
3607                 goto pre_mutex_err;
3608         }
3609
3610         if (dev_priv->mm.suspended) {
3611                 mutex_unlock(&dev->struct_mutex);
3612                 ret = -EBUSY;
3613                 goto pre_mutex_err;
3614         }
3615
3616         /* Look up object handles */
3617         flips = 0;
3618         for (i = 0; i < args->buffer_count; i++) {
3619                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3620                                                        exec_list[i].handle);
3621                 if (object_list[i] == NULL) {
3622                         DRM_ERROR("Invalid object handle %d at index %d\n",
3623                                    exec_list[i].handle, i);
3624                         /* prevent error path from reading uninitialized data */
3625                         args->buffer_count = i + 1;
3626                         ret = -ENOENT;
3627                         goto err;
3628                 }
3629
3630                 obj_priv = to_intel_bo(object_list[i]);
3631                 if (obj_priv->in_execbuffer) {
3632                         DRM_ERROR("Object %p appears more than once in object list\n",
3633                                    object_list[i]);
3634                         /* prevent error path from reading uninitialized data */
3635                         args->buffer_count = i + 1;
3636                         ret = -EINVAL;
3637                         goto err;
3638                 }
3639                 obj_priv->in_execbuffer = true;
3640                 flips += atomic_read(&obj_priv->pending_flip);
3641         }
3642
3643         if (flips > 0) {
3644                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3645                                                      args->buffer_count);
3646                 if (ret)
3647                         goto err;
3648         }
3649
3650         /* Pin and relocate */
3651         for (pin_tries = 0; ; pin_tries++) {
3652                 ret = 0;
3653                 reloc_index = 0;
3654
3655                 for (i = 0; i < args->buffer_count; i++) {
3656                         object_list[i]->pending_read_domains = 0;
3657                         object_list[i]->pending_write_domain = 0;
3658                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3659                                                                file_priv,
3660                                                                &exec_list[i],
3661                                                                &relocs[reloc_index]);
3662                         if (ret)
3663                                 break;
3664                         pinned = i + 1;
3665                         reloc_index += exec_list[i].relocation_count;
3666                 }
3667                 /* success */
3668                 if (ret == 0)
3669                         break;
3670
3671                 /* error other than GTT full, or we've already tried again */
3672                 if (ret != -ENOSPC || pin_tries >= 1) {
3673                         if (ret != -ERESTARTSYS) {
3674                                 unsigned long long total_size = 0;
3675                                 int num_fences = 0;
3676                                 for (i = 0; i < args->buffer_count; i++) {
3677                                         obj_priv = to_intel_bo(object_list[i]);
3678
3679                                         total_size += object_list[i]->size;
3680                                         num_fences +=
3681                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3682                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3683                                 }
3684                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3685                                           pinned+1, args->buffer_count,
3686                                           total_size, num_fences,
3687                                           ret);
3688                                 DRM_ERROR("%d objects [%d pinned], "
3689                                           "%d object bytes [%d pinned], "
3690                                           "%d/%d gtt bytes\n",
3691                                           atomic_read(&dev->object_count),
3692                                           atomic_read(&dev->pin_count),
3693                                           atomic_read(&dev->object_memory),
3694                                           atomic_read(&dev->pin_memory),
3695                                           atomic_read(&dev->gtt_memory),
3696                                           dev->gtt_total);
3697                         }
3698                         goto err;
3699                 }
3700
3701                 /* unpin all of our buffers */
3702                 for (i = 0; i < pinned; i++)
3703                         i915_gem_object_unpin(object_list[i]);
3704                 pinned = 0;
3705
3706                 /* evict everyone we can from the aperture */
3707                 ret = i915_gem_evict_everything(dev);
3708                 if (ret && ret != -ENOSPC)
3709                         goto err;
3710         }
3711
3712         /* Set the pending read domains for the batch buffer to COMMAND */
3713         batch_obj = object_list[args->buffer_count-1];
3714         if (batch_obj->pending_write_domain) {
3715                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3716                 ret = -EINVAL;
3717                 goto err;
3718         }
3719         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3720
3721         /* Sanity check the batch buffer, prior to moving objects */
3722         exec_offset = exec_list[args->buffer_count - 1].offset;
3723         ret = i915_gem_check_execbuffer (args, exec_offset);
3724         if (ret != 0) {
3725                 DRM_ERROR("execbuf with invalid offset/length\n");
3726                 goto err;
3727         }
3728
3729         i915_verify_inactive(dev, __FILE__, __LINE__);
3730
3731         /* Zero the global flush/invalidate flags. These
3732          * will be modified as new domains are computed
3733          * for each object
3734          */
3735         dev->invalidate_domains = 0;
3736         dev->flush_domains = 0;
3737         dev_priv->flush_rings = 0;
3738
3739         for (i = 0; i < args->buffer_count; i++) {
3740                 struct drm_gem_object *obj = object_list[i];
3741
3742                 /* Compute new gpu domains and update invalidate/flush */
3743                 i915_gem_object_set_to_gpu_domain(obj);
3744         }
3745
3746         i915_verify_inactive(dev, __FILE__, __LINE__);
3747
3748         if (dev->invalidate_domains | dev->flush_domains) {
3749 #if WATCH_EXEC
3750                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3751                           __func__,
3752                          dev->invalidate_domains,
3753                          dev->flush_domains);
3754 #endif
3755                 i915_gem_flush(dev,
3756                                dev->invalidate_domains,
3757                                dev->flush_domains);
3758                 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3759                         (void)i915_add_request(dev, file_priv,
3760                                                dev->flush_domains,
3761                                                &dev_priv->render_ring);
3762                 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3763                         (void)i915_add_request(dev, file_priv,
3764                                                dev->flush_domains,
3765                                                &dev_priv->bsd_ring);
3766         }
3767
3768         for (i = 0; i < args->buffer_count; i++) {
3769                 struct drm_gem_object *obj = object_list[i];
3770                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3771                 uint32_t old_write_domain = obj->write_domain;
3772
3773                 obj->write_domain = obj->pending_write_domain;
3774                 if (obj->write_domain)
3775                         list_move_tail(&obj_priv->gpu_write_list,
3776                                        &dev_priv->mm.gpu_write_list);
3777                 else
3778                         list_del_init(&obj_priv->gpu_write_list);
3779
3780                 trace_i915_gem_object_change_domain(obj,
3781                                                     obj->read_domains,
3782                                                     old_write_domain);
3783         }
3784
3785         i915_verify_inactive(dev, __FILE__, __LINE__);
3786
3787 #if WATCH_COHERENCY
3788         for (i = 0; i < args->buffer_count; i++) {
3789                 i915_gem_object_check_coherency(object_list[i],
3790                                                 exec_list[i].handle);
3791         }
3792 #endif
3793
3794 #if WATCH_EXEC
3795         i915_gem_dump_object(batch_obj,
3796                               args->batch_len,
3797                               __func__,
3798                               ~0);
3799 #endif
3800
3801         /* Exec the batchbuffer */
3802         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3803                         cliprects, exec_offset);
3804         if (ret) {
3805                 DRM_ERROR("dispatch failed %d\n", ret);
3806                 goto err;
3807         }
3808
3809         /*
3810          * Ensure that the commands in the batch buffer are
3811          * finished before the interrupt fires
3812          */
3813         flush_domains = i915_retire_commands(dev, ring);
3814
3815         i915_verify_inactive(dev, __FILE__, __LINE__);
3816
3817         /*
3818          * Get a seqno representing the execution of the current buffer,
3819          * which we can wait on.  We would like to mitigate these interrupts,
3820          * likely by only creating seqnos occasionally (so that we have
3821          * *some* interrupts representing completion of buffers that we can
3822          * wait on when trying to clear up gtt space).
3823          */
3824         seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3825         BUG_ON(seqno == 0);
3826         for (i = 0; i < args->buffer_count; i++) {
3827                 struct drm_gem_object *obj = object_list[i];
3828                 obj_priv = to_intel_bo(obj);
3829
3830                 i915_gem_object_move_to_active(obj, seqno, ring);
3831 #if WATCH_LRU
3832                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3833 #endif
3834         }
3835 #if WATCH_LRU
3836         i915_dump_lru(dev, __func__);
3837 #endif
3838
3839         i915_verify_inactive(dev, __FILE__, __LINE__);
3840
3841 err:
3842         for (i = 0; i < pinned; i++)
3843                 i915_gem_object_unpin(object_list[i]);
3844
3845         for (i = 0; i < args->buffer_count; i++) {
3846                 if (object_list[i]) {
3847                         obj_priv = to_intel_bo(object_list[i]);
3848                         obj_priv->in_execbuffer = false;
3849                 }
3850                 drm_gem_object_unreference(object_list[i]);
3851         }
3852
3853         mutex_unlock(&dev->struct_mutex);
3854
3855 pre_mutex_err:
3856         /* Copy the updated relocations out regardless of current error
3857          * state.  Failure to update the relocs would mean that the next
3858          * time userland calls execbuf, it would do so with presumed offset
3859          * state that didn't match the actual object state.
3860          */
3861         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3862                                            relocs);
3863         if (ret2 != 0) {
3864                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3865
3866                 if (ret == 0)
3867                         ret = ret2;
3868         }
3869
3870         drm_free_large(object_list);
3871         kfree(cliprects);
3872
3873         return ret;
3874 }
3875
3876 /*
3877  * Legacy execbuffer just creates an exec2 list from the original exec object
3878  * list array and passes it to the real function.
3879  */
3880 int
3881 i915_gem_execbuffer(struct drm_device *dev, void *data,
3882                     struct drm_file *file_priv)
3883 {
3884         struct drm_i915_gem_execbuffer *args = data;
3885         struct drm_i915_gem_execbuffer2 exec2;
3886         struct drm_i915_gem_exec_object *exec_list = NULL;
3887         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3888         int ret, i;
3889
3890 #if WATCH_EXEC
3891         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3892                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3893 #endif
3894
3895         if (args->buffer_count < 1) {
3896                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3897                 return -EINVAL;
3898         }
3899
3900         /* Copy in the exec list from userland */
3901         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3902         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3903         if (exec_list == NULL || exec2_list == NULL) {
3904                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3905                           args->buffer_count);
3906                 drm_free_large(exec_list);
3907                 drm_free_large(exec2_list);
3908                 return -ENOMEM;
3909         }
3910         ret = copy_from_user(exec_list,
3911                              (struct drm_i915_relocation_entry __user *)
3912                              (uintptr_t) args->buffers_ptr,
3913                              sizeof(*exec_list) * args->buffer_count);
3914         if (ret != 0) {
3915                 DRM_ERROR("copy %d exec entries failed %d\n",
3916                           args->buffer_count, ret);
3917                 drm_free_large(exec_list);
3918                 drm_free_large(exec2_list);
3919                 return -EFAULT;
3920         }
3921
3922         for (i = 0; i < args->buffer_count; i++) {
3923                 exec2_list[i].handle = exec_list[i].handle;
3924                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3925                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3926                 exec2_list[i].alignment = exec_list[i].alignment;
3927                 exec2_list[i].offset = exec_list[i].offset;
3928                 if (!IS_I965G(dev))
3929                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3930                 else
3931                         exec2_list[i].flags = 0;
3932         }
3933
3934         exec2.buffers_ptr = args->buffers_ptr;
3935         exec2.buffer_count = args->buffer_count;
3936         exec2.batch_start_offset = args->batch_start_offset;
3937         exec2.batch_len = args->batch_len;
3938         exec2.DR1 = args->DR1;
3939         exec2.DR4 = args->DR4;
3940         exec2.num_cliprects = args->num_cliprects;
3941         exec2.cliprects_ptr = args->cliprects_ptr;
3942         exec2.flags = I915_EXEC_RENDER;
3943
3944         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3945         if (!ret) {
3946                 /* Copy the new buffer offsets back to the user's exec list. */
3947                 for (i = 0; i < args->buffer_count; i++)
3948                         exec_list[i].offset = exec2_list[i].offset;
3949                 /* ... and back out to userspace */
3950                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3951                                    (uintptr_t) args->buffers_ptr,
3952                                    exec_list,
3953                                    sizeof(*exec_list) * args->buffer_count);
3954                 if (ret) {
3955                         ret = -EFAULT;
3956                         DRM_ERROR("failed to copy %d exec entries "
3957                                   "back to user (%d)\n",
3958                                   args->buffer_count, ret);
3959                 }
3960         }
3961
3962         drm_free_large(exec_list);
3963         drm_free_large(exec2_list);
3964         return ret;
3965 }
3966
3967 int
3968 i915_gem_execbuffer2(struct drm_device *dev, void *data,
3969                      struct drm_file *file_priv)
3970 {
3971         struct drm_i915_gem_execbuffer2 *args = data;
3972         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3973         int ret;
3974
3975 #if WATCH_EXEC
3976         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3977                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3978 #endif
3979
3980         if (args->buffer_count < 1) {
3981                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
3982                 return -EINVAL;
3983         }
3984
3985         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3986         if (exec2_list == NULL) {
3987                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3988                           args->buffer_count);
3989                 return -ENOMEM;
3990         }
3991         ret = copy_from_user(exec2_list,
3992                              (struct drm_i915_relocation_entry __user *)
3993                              (uintptr_t) args->buffers_ptr,
3994                              sizeof(*exec2_list) * args->buffer_count);
3995         if (ret != 0) {
3996                 DRM_ERROR("copy %d exec entries failed %d\n",
3997                           args->buffer_count, ret);
3998                 drm_free_large(exec2_list);
3999                 return -EFAULT;
4000         }
4001
4002         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4003         if (!ret) {
4004                 /* Copy the new buffer offsets back to the user's exec list. */
4005                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4006                                    (uintptr_t) args->buffers_ptr,
4007                                    exec2_list,
4008                                    sizeof(*exec2_list) * args->buffer_count);
4009                 if (ret) {
4010                         ret = -EFAULT;
4011                         DRM_ERROR("failed to copy %d exec entries "
4012                                   "back to user (%d)\n",
4013                                   args->buffer_count, ret);
4014                 }
4015         }
4016
4017         drm_free_large(exec2_list);
4018         return ret;
4019 }
4020
4021 int
4022 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4023 {
4024         struct drm_device *dev = obj->dev;
4025         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4026         int ret;
4027
4028         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4029
4030         i915_verify_inactive(dev, __FILE__, __LINE__);
4031
4032         if (obj_priv->gtt_space != NULL) {
4033                 if (alignment == 0)
4034                         alignment = i915_gem_get_gtt_alignment(obj);
4035                 if (obj_priv->gtt_offset & (alignment - 1)) {
4036                         WARN(obj_priv->pin_count,
4037                              "bo is already pinned with incorrect alignment:"
4038                              " offset=%x, req.alignment=%x\n",
4039                              obj_priv->gtt_offset, alignment);
4040                         ret = i915_gem_object_unbind(obj);
4041                         if (ret)
4042                                 return ret;
4043                 }
4044         }
4045
4046         if (obj_priv->gtt_space == NULL) {
4047                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4048                 if (ret)
4049                         return ret;
4050         }
4051
4052         obj_priv->pin_count++;
4053
4054         /* If the object is not active and not pending a flush,
4055          * remove it from the inactive list
4056          */
4057         if (obj_priv->pin_count == 1) {
4058                 atomic_inc(&dev->pin_count);
4059                 atomic_add(obj->size, &dev->pin_memory);
4060                 if (!obj_priv->active &&
4061                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4062                         list_del_init(&obj_priv->list);
4063         }
4064         i915_verify_inactive(dev, __FILE__, __LINE__);
4065
4066         return 0;
4067 }
4068
4069 void
4070 i915_gem_object_unpin(struct drm_gem_object *obj)
4071 {
4072         struct drm_device *dev = obj->dev;
4073         drm_i915_private_t *dev_priv = dev->dev_private;
4074         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4075
4076         i915_verify_inactive(dev, __FILE__, __LINE__);
4077         obj_priv->pin_count--;
4078         BUG_ON(obj_priv->pin_count < 0);
4079         BUG_ON(obj_priv->gtt_space == NULL);
4080
4081         /* If the object is no longer pinned, and is
4082          * neither active nor being flushed, then stick it on
4083          * the inactive list
4084          */
4085         if (obj_priv->pin_count == 0) {
4086                 if (!obj_priv->active &&
4087                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4088                         list_move_tail(&obj_priv->list,
4089                                        &dev_priv->mm.inactive_list);
4090                 atomic_dec(&dev->pin_count);
4091                 atomic_sub(obj->size, &dev->pin_memory);
4092         }
4093         i915_verify_inactive(dev, __FILE__, __LINE__);
4094 }
4095
4096 int
4097 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4098                    struct drm_file *file_priv)
4099 {
4100         struct drm_i915_gem_pin *args = data;
4101         struct drm_gem_object *obj;
4102         struct drm_i915_gem_object *obj_priv;
4103         int ret;
4104
4105         mutex_lock(&dev->struct_mutex);
4106
4107         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4108         if (obj == NULL) {
4109                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4110                           args->handle);
4111                 mutex_unlock(&dev->struct_mutex);
4112                 return -ENOENT;
4113         }
4114         obj_priv = to_intel_bo(obj);
4115
4116         if (obj_priv->madv != I915_MADV_WILLNEED) {
4117                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4118                 drm_gem_object_unreference(obj);
4119                 mutex_unlock(&dev->struct_mutex);
4120                 return -EINVAL;
4121         }
4122
4123         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4124                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4125                           args->handle);
4126                 drm_gem_object_unreference(obj);
4127                 mutex_unlock(&dev->struct_mutex);
4128                 return -EINVAL;
4129         }
4130
4131         obj_priv->user_pin_count++;
4132         obj_priv->pin_filp = file_priv;
4133         if (obj_priv->user_pin_count == 1) {
4134                 ret = i915_gem_object_pin(obj, args->alignment);
4135                 if (ret != 0) {
4136                         drm_gem_object_unreference(obj);
4137                         mutex_unlock(&dev->struct_mutex);
4138                         return ret;
4139                 }
4140         }
4141
4142         /* XXX - flush the CPU caches for pinned objects
4143          * as the X server doesn't manage domains yet
4144          */
4145         i915_gem_object_flush_cpu_write_domain(obj);
4146         args->offset = obj_priv->gtt_offset;
4147         drm_gem_object_unreference(obj);
4148         mutex_unlock(&dev->struct_mutex);
4149
4150         return 0;
4151 }
4152
4153 int
4154 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4155                      struct drm_file *file_priv)
4156 {
4157         struct drm_i915_gem_pin *args = data;
4158         struct drm_gem_object *obj;
4159         struct drm_i915_gem_object *obj_priv;
4160
4161         mutex_lock(&dev->struct_mutex);
4162
4163         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4164         if (obj == NULL) {
4165                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4166                           args->handle);
4167                 mutex_unlock(&dev->struct_mutex);
4168                 return -ENOENT;
4169         }
4170
4171         obj_priv = to_intel_bo(obj);
4172         if (obj_priv->pin_filp != file_priv) {
4173                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4174                           args->handle);
4175                 drm_gem_object_unreference(obj);
4176                 mutex_unlock(&dev->struct_mutex);
4177                 return -EINVAL;
4178         }
4179         obj_priv->user_pin_count--;
4180         if (obj_priv->user_pin_count == 0) {
4181                 obj_priv->pin_filp = NULL;
4182                 i915_gem_object_unpin(obj);
4183         }
4184
4185         drm_gem_object_unreference(obj);
4186         mutex_unlock(&dev->struct_mutex);
4187         return 0;
4188 }
4189
4190 int
4191 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4192                     struct drm_file *file_priv)
4193 {
4194         struct drm_i915_gem_busy *args = data;
4195         struct drm_gem_object *obj;
4196         struct drm_i915_gem_object *obj_priv;
4197
4198         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4199         if (obj == NULL) {
4200                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4201                           args->handle);
4202                 return -ENOENT;
4203         }
4204
4205         mutex_lock(&dev->struct_mutex);
4206
4207         /* Count all active objects as busy, even if they are currently not used
4208          * by the gpu. Users of this interface expect objects to eventually
4209          * become non-busy without any further actions, therefore emit any
4210          * necessary flushes here.
4211          */
4212         obj_priv = to_intel_bo(obj);
4213         args->busy = obj_priv->active;
4214         if (args->busy) {
4215                 /* Unconditionally flush objects, even when the gpu still uses this
4216                  * object. Userspace calling this function indicates that it wants to
4217                  * use this buffer rather sooner than later, so issuing the required
4218                  * flush earlier is beneficial.
4219                  */
4220                 if (obj->write_domain) {
4221                         i915_gem_flush(dev, 0, obj->write_domain);
4222                         (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4223                 }
4224
4225                 /* Update the active list for the hardware's current position.
4226                  * Otherwise this only updates on a delayed timer or when irqs
4227                  * are actually unmasked, and our working set ends up being
4228                  * larger than required.
4229                  */
4230                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4231
4232                 args->busy = obj_priv->active;
4233         }
4234
4235         drm_gem_object_unreference(obj);
4236         mutex_unlock(&dev->struct_mutex);
4237         return 0;
4238 }
4239
4240 int
4241 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4242                         struct drm_file *file_priv)
4243 {
4244     return i915_gem_ring_throttle(dev, file_priv);
4245 }
4246
4247 int
4248 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4249                        struct drm_file *file_priv)
4250 {
4251         struct drm_i915_gem_madvise *args = data;
4252         struct drm_gem_object *obj;
4253         struct drm_i915_gem_object *obj_priv;
4254
4255         switch (args->madv) {
4256         case I915_MADV_DONTNEED:
4257         case I915_MADV_WILLNEED:
4258             break;
4259         default:
4260             return -EINVAL;
4261         }
4262
4263         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4264         if (obj == NULL) {
4265                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4266                           args->handle);
4267                 return -ENOENT;
4268         }
4269
4270         mutex_lock(&dev->struct_mutex);
4271         obj_priv = to_intel_bo(obj);
4272
4273         if (obj_priv->pin_count) {
4274                 drm_gem_object_unreference(obj);
4275                 mutex_unlock(&dev->struct_mutex);
4276
4277                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4278                 return -EINVAL;
4279         }
4280
4281         if (obj_priv->madv != __I915_MADV_PURGED)
4282                 obj_priv->madv = args->madv;
4283
4284         /* if the object is no longer bound, discard its backing storage */
4285         if (i915_gem_object_is_purgeable(obj_priv) &&
4286             obj_priv->gtt_space == NULL)
4287                 i915_gem_object_truncate(obj);
4288
4289         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4290
4291         drm_gem_object_unreference(obj);
4292         mutex_unlock(&dev->struct_mutex);
4293
4294         return 0;
4295 }
4296
4297 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4298                                               size_t size)
4299 {
4300         struct drm_i915_gem_object *obj;
4301
4302         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4303         if (obj == NULL)
4304                 return NULL;
4305
4306         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4307                 kfree(obj);
4308                 return NULL;
4309         }
4310
4311         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4312         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4313
4314         obj->agp_type = AGP_USER_MEMORY;
4315         obj->base.driver_private = NULL;
4316         obj->fence_reg = I915_FENCE_REG_NONE;
4317         INIT_LIST_HEAD(&obj->list);
4318         INIT_LIST_HEAD(&obj->gpu_write_list);
4319         obj->madv = I915_MADV_WILLNEED;
4320
4321         trace_i915_gem_object_create(&obj->base);
4322
4323         return &obj->base;
4324 }
4325
4326 int i915_gem_init_object(struct drm_gem_object *obj)
4327 {
4328         BUG();
4329
4330         return 0;
4331 }
4332
4333 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4334 {
4335         struct drm_device *dev = obj->dev;
4336         drm_i915_private_t *dev_priv = dev->dev_private;
4337         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4338         int ret;
4339
4340         ret = i915_gem_object_unbind(obj);
4341         if (ret == -ERESTARTSYS) {
4342                 list_move(&obj_priv->list,
4343                           &dev_priv->mm.deferred_free_list);
4344                 return;
4345         }
4346
4347         if (obj_priv->mmap_offset)
4348                 i915_gem_free_mmap_offset(obj);
4349
4350         drm_gem_object_release(obj);
4351
4352         kfree(obj_priv->page_cpu_valid);
4353         kfree(obj_priv->bit_17);
4354         kfree(obj_priv);
4355 }
4356
4357 void i915_gem_free_object(struct drm_gem_object *obj)
4358 {
4359         struct drm_device *dev = obj->dev;
4360         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4361
4362         trace_i915_gem_object_destroy(obj);
4363
4364         while (obj_priv->pin_count > 0)
4365                 i915_gem_object_unpin(obj);
4366
4367         if (obj_priv->phys_obj)
4368                 i915_gem_detach_phys_object(dev, obj);
4369
4370         i915_gem_free_object_tail(obj);
4371 }
4372
4373 int
4374 i915_gem_idle(struct drm_device *dev)
4375 {
4376         drm_i915_private_t *dev_priv = dev->dev_private;
4377         int ret;
4378
4379         mutex_lock(&dev->struct_mutex);
4380
4381         if (dev_priv->mm.suspended ||
4382                         (dev_priv->render_ring.gem_object == NULL) ||
4383                         (HAS_BSD(dev) &&
4384                          dev_priv->bsd_ring.gem_object == NULL)) {
4385                 mutex_unlock(&dev->struct_mutex);
4386                 return 0;
4387         }
4388
4389         ret = i915_gpu_idle(dev);
4390         if (ret) {
4391                 mutex_unlock(&dev->struct_mutex);
4392                 return ret;
4393         }
4394
4395         /* Under UMS, be paranoid and evict. */
4396         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4397                 ret = i915_gem_evict_inactive(dev);
4398                 if (ret) {
4399                         mutex_unlock(&dev->struct_mutex);
4400                         return ret;
4401                 }
4402         }
4403
4404         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4405          * We need to replace this with a semaphore, or something.
4406          * And not confound mm.suspended!
4407          */
4408         dev_priv->mm.suspended = 1;
4409         del_timer(&dev_priv->hangcheck_timer);
4410
4411         i915_kernel_lost_context(dev);
4412         i915_gem_cleanup_ringbuffer(dev);
4413
4414         mutex_unlock(&dev->struct_mutex);
4415
4416         /* Cancel the retire work handler, which should be idle now. */
4417         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4418
4419         return 0;
4420 }
4421
4422 /*
4423  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4424  * over cache flushing.
4425  */
4426 static int
4427 i915_gem_init_pipe_control(struct drm_device *dev)
4428 {
4429         drm_i915_private_t *dev_priv = dev->dev_private;
4430         struct drm_gem_object *obj;
4431         struct drm_i915_gem_object *obj_priv;
4432         int ret;
4433
4434         obj = i915_gem_alloc_object(dev, 4096);
4435         if (obj == NULL) {
4436                 DRM_ERROR("Failed to allocate seqno page\n");
4437                 ret = -ENOMEM;
4438                 goto err;
4439         }
4440         obj_priv = to_intel_bo(obj);
4441         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4442
4443         ret = i915_gem_object_pin(obj, 4096);
4444         if (ret)
4445                 goto err_unref;
4446
4447         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4448         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4449         if (dev_priv->seqno_page == NULL)
4450                 goto err_unpin;
4451
4452         dev_priv->seqno_obj = obj;
4453         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4454
4455         return 0;
4456
4457 err_unpin:
4458         i915_gem_object_unpin(obj);
4459 err_unref:
4460         drm_gem_object_unreference(obj);
4461 err:
4462         return ret;
4463 }
4464
4465
4466 static void
4467 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4468 {
4469         drm_i915_private_t *dev_priv = dev->dev_private;
4470         struct drm_gem_object *obj;
4471         struct drm_i915_gem_object *obj_priv;
4472
4473         obj = dev_priv->seqno_obj;
4474         obj_priv = to_intel_bo(obj);
4475         kunmap(obj_priv->pages[0]);
4476         i915_gem_object_unpin(obj);
4477         drm_gem_object_unreference(obj);
4478         dev_priv->seqno_obj = NULL;
4479
4480         dev_priv->seqno_page = NULL;
4481 }
4482
4483 int
4484 i915_gem_init_ringbuffer(struct drm_device *dev)
4485 {
4486         drm_i915_private_t *dev_priv = dev->dev_private;
4487         int ret;
4488
4489         dev_priv->render_ring = render_ring;
4490
4491         if (!I915_NEED_GFX_HWS(dev)) {
4492                 dev_priv->render_ring.status_page.page_addr
4493                         = dev_priv->status_page_dmah->vaddr;
4494                 memset(dev_priv->render_ring.status_page.page_addr,
4495                                 0, PAGE_SIZE);
4496         }
4497
4498         if (HAS_PIPE_CONTROL(dev)) {
4499                 ret = i915_gem_init_pipe_control(dev);
4500                 if (ret)
4501                         return ret;
4502         }
4503
4504         ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4505         if (ret)
4506                 goto cleanup_pipe_control;
4507
4508         if (HAS_BSD(dev)) {
4509                 dev_priv->bsd_ring = bsd_ring;
4510                 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4511                 if (ret)
4512                         goto cleanup_render_ring;
4513         }
4514
4515         dev_priv->next_seqno = 1;
4516
4517         return 0;
4518
4519 cleanup_render_ring:
4520         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4521 cleanup_pipe_control:
4522         if (HAS_PIPE_CONTROL(dev))
4523                 i915_gem_cleanup_pipe_control(dev);
4524         return ret;
4525 }
4526
4527 void
4528 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4529 {
4530         drm_i915_private_t *dev_priv = dev->dev_private;
4531
4532         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4533         if (HAS_BSD(dev))
4534                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4535         if (HAS_PIPE_CONTROL(dev))
4536                 i915_gem_cleanup_pipe_control(dev);
4537 }
4538
4539 int
4540 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4541                        struct drm_file *file_priv)
4542 {
4543         drm_i915_private_t *dev_priv = dev->dev_private;
4544         int ret;
4545
4546         if (drm_core_check_feature(dev, DRIVER_MODESET))
4547                 return 0;
4548
4549         if (atomic_read(&dev_priv->mm.wedged)) {
4550                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4551                 atomic_set(&dev_priv->mm.wedged, 0);
4552         }
4553
4554         mutex_lock(&dev->struct_mutex);
4555         dev_priv->mm.suspended = 0;
4556
4557         ret = i915_gem_init_ringbuffer(dev);
4558         if (ret != 0) {
4559                 mutex_unlock(&dev->struct_mutex);
4560                 return ret;
4561         }
4562
4563         spin_lock(&dev_priv->mm.active_list_lock);
4564         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4565         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4566         spin_unlock(&dev_priv->mm.active_list_lock);
4567
4568         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4569         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4570         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4571         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4572         mutex_unlock(&dev->struct_mutex);
4573
4574         ret = drm_irq_install(dev);
4575         if (ret)
4576                 goto cleanup_ringbuffer;
4577
4578         return 0;
4579
4580 cleanup_ringbuffer:
4581         mutex_lock(&dev->struct_mutex);
4582         i915_gem_cleanup_ringbuffer(dev);
4583         dev_priv->mm.suspended = 1;
4584         mutex_unlock(&dev->struct_mutex);
4585
4586         return ret;
4587 }
4588
4589 int
4590 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4591                        struct drm_file *file_priv)
4592 {
4593         if (drm_core_check_feature(dev, DRIVER_MODESET))
4594                 return 0;
4595
4596         drm_irq_uninstall(dev);
4597         return i915_gem_idle(dev);
4598 }
4599
4600 void
4601 i915_gem_lastclose(struct drm_device *dev)
4602 {
4603         int ret;
4604
4605         if (drm_core_check_feature(dev, DRIVER_MODESET))
4606                 return;
4607
4608         ret = i915_gem_idle(dev);
4609         if (ret)
4610                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4611 }
4612
4613 void
4614 i915_gem_load(struct drm_device *dev)
4615 {
4616         int i;
4617         drm_i915_private_t *dev_priv = dev->dev_private;
4618
4619         spin_lock_init(&dev_priv->mm.active_list_lock);
4620         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4621         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4622         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4623         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4624         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4625         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4626         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4627         if (HAS_BSD(dev)) {
4628                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4629                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4630         }
4631         for (i = 0; i < 16; i++)
4632                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4633         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4634                           i915_gem_retire_work_handler);
4635         spin_lock(&shrink_list_lock);
4636         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4637         spin_unlock(&shrink_list_lock);
4638
4639         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4640         if (IS_GEN3(dev)) {
4641                 u32 tmp = I915_READ(MI_ARB_STATE);
4642                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4643                         /* arb state is a masked write, so set bit + bit in mask */
4644                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4645                         I915_WRITE(MI_ARB_STATE, tmp);
4646                 }
4647         }
4648
4649         /* Old X drivers will take 0-2 for front, back, depth buffers */
4650         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4651                 dev_priv->fence_reg_start = 3;
4652
4653         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4654                 dev_priv->num_fence_regs = 16;
4655         else
4656                 dev_priv->num_fence_regs = 8;
4657
4658         /* Initialize fence registers to zero */
4659         if (IS_I965G(dev)) {
4660                 for (i = 0; i < 16; i++)
4661                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4662         } else {
4663                 for (i = 0; i < 8; i++)
4664                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4665                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4666                         for (i = 0; i < 8; i++)
4667                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4668         }
4669         i915_gem_detect_bit_6_swizzle(dev);
4670         init_waitqueue_head(&dev_priv->pending_flip_queue);
4671 }
4672
4673 /*
4674  * Create a physically contiguous memory object for this object
4675  * e.g. for cursor + overlay regs
4676  */
4677 int i915_gem_init_phys_object(struct drm_device *dev,
4678                               int id, int size, int align)
4679 {
4680         drm_i915_private_t *dev_priv = dev->dev_private;
4681         struct drm_i915_gem_phys_object *phys_obj;
4682         int ret;
4683
4684         if (dev_priv->mm.phys_objs[id - 1] || !size)
4685                 return 0;
4686
4687         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4688         if (!phys_obj)
4689                 return -ENOMEM;
4690
4691         phys_obj->id = id;
4692
4693         phys_obj->handle = drm_pci_alloc(dev, size, align);
4694         if (!phys_obj->handle) {
4695                 ret = -ENOMEM;
4696                 goto kfree_obj;
4697         }
4698 #ifdef CONFIG_X86
4699         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4700 #endif
4701
4702         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4703
4704         return 0;
4705 kfree_obj:
4706         kfree(phys_obj);
4707         return ret;
4708 }
4709
4710 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4711 {
4712         drm_i915_private_t *dev_priv = dev->dev_private;
4713         struct drm_i915_gem_phys_object *phys_obj;
4714
4715         if (!dev_priv->mm.phys_objs[id - 1])
4716                 return;
4717
4718         phys_obj = dev_priv->mm.phys_objs[id - 1];
4719         if (phys_obj->cur_obj) {
4720                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4721         }
4722
4723 #ifdef CONFIG_X86
4724         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4725 #endif
4726         drm_pci_free(dev, phys_obj->handle);
4727         kfree(phys_obj);
4728         dev_priv->mm.phys_objs[id - 1] = NULL;
4729 }
4730
4731 void i915_gem_free_all_phys_object(struct drm_device *dev)
4732 {
4733         int i;
4734
4735         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4736                 i915_gem_free_phys_object(dev, i);
4737 }
4738
4739 void i915_gem_detach_phys_object(struct drm_device *dev,
4740                                  struct drm_gem_object *obj)
4741 {
4742         struct drm_i915_gem_object *obj_priv;
4743         int i;
4744         int ret;
4745         int page_count;
4746
4747         obj_priv = to_intel_bo(obj);
4748         if (!obj_priv->phys_obj)
4749                 return;
4750
4751         ret = i915_gem_object_get_pages(obj, 0);
4752         if (ret)
4753                 goto out;
4754
4755         page_count = obj->size / PAGE_SIZE;
4756
4757         for (i = 0; i < page_count; i++) {
4758                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4759                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4760
4761                 memcpy(dst, src, PAGE_SIZE);
4762                 kunmap_atomic(dst, KM_USER0);
4763         }
4764         drm_clflush_pages(obj_priv->pages, page_count);
4765         drm_agp_chipset_flush(dev);
4766
4767         i915_gem_object_put_pages(obj);
4768 out:
4769         obj_priv->phys_obj->cur_obj = NULL;
4770         obj_priv->phys_obj = NULL;
4771 }
4772
4773 int
4774 i915_gem_attach_phys_object(struct drm_device *dev,
4775                             struct drm_gem_object *obj,
4776                             int id,
4777                             int align)
4778 {
4779         drm_i915_private_t *dev_priv = dev->dev_private;
4780         struct drm_i915_gem_object *obj_priv;
4781         int ret = 0;
4782         int page_count;
4783         int i;
4784
4785         if (id > I915_MAX_PHYS_OBJECT)
4786                 return -EINVAL;
4787
4788         obj_priv = to_intel_bo(obj);
4789
4790         if (obj_priv->phys_obj) {
4791                 if (obj_priv->phys_obj->id == id)
4792                         return 0;
4793                 i915_gem_detach_phys_object(dev, obj);
4794         }
4795
4796         /* create a new object */
4797         if (!dev_priv->mm.phys_objs[id - 1]) {
4798                 ret = i915_gem_init_phys_object(dev, id,
4799                                                 obj->size, align);
4800                 if (ret) {
4801                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4802                         goto out;
4803                 }
4804         }
4805
4806         /* bind to the object */
4807         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4808         obj_priv->phys_obj->cur_obj = obj;
4809
4810         ret = i915_gem_object_get_pages(obj, 0);
4811         if (ret) {
4812                 DRM_ERROR("failed to get page list\n");
4813                 goto out;
4814         }
4815
4816         page_count = obj->size / PAGE_SIZE;
4817
4818         for (i = 0; i < page_count; i++) {
4819                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4820                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4821
4822                 memcpy(dst, src, PAGE_SIZE);
4823                 kunmap_atomic(src, KM_USER0);
4824         }
4825
4826         i915_gem_object_put_pages(obj);
4827
4828         return 0;
4829 out:
4830         return ret;
4831 }
4832
4833 static int
4834 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4835                      struct drm_i915_gem_pwrite *args,
4836                      struct drm_file *file_priv)
4837 {
4838         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4839         void *obj_addr;
4840         int ret;
4841         char __user *user_data;
4842
4843         user_data = (char __user *) (uintptr_t) args->data_ptr;
4844         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4845
4846         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4847         ret = copy_from_user(obj_addr, user_data, args->size);
4848         if (ret)
4849                 return -EFAULT;
4850
4851         drm_agp_chipset_flush(dev);
4852         return 0;
4853 }
4854
4855 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4856 {
4857         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4858
4859         /* Clean up our request list when the client is going away, so that
4860          * later retire_requests won't dereference our soon-to-be-gone
4861          * file_priv.
4862          */
4863         mutex_lock(&dev->struct_mutex);
4864         while (!list_empty(&i915_file_priv->mm.request_list))
4865                 list_del_init(i915_file_priv->mm.request_list.next);
4866         mutex_unlock(&dev->struct_mutex);
4867 }
4868
4869 static int
4870 i915_gpu_is_active(struct drm_device *dev)
4871 {
4872         drm_i915_private_t *dev_priv = dev->dev_private;
4873         int lists_empty;
4874
4875         spin_lock(&dev_priv->mm.active_list_lock);
4876         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4877                       list_empty(&dev_priv->render_ring.active_list);
4878         if (HAS_BSD(dev))
4879                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4880         spin_unlock(&dev_priv->mm.active_list_lock);
4881
4882         return !lists_empty;
4883 }
4884
4885 static int
4886 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
4887 {
4888         drm_i915_private_t *dev_priv, *next_dev;
4889         struct drm_i915_gem_object *obj_priv, *next_obj;
4890         int cnt = 0;
4891         int would_deadlock = 1;
4892
4893         /* "fast-path" to count number of available objects */
4894         if (nr_to_scan == 0) {
4895                 spin_lock(&shrink_list_lock);
4896                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4897                         struct drm_device *dev = dev_priv->dev;
4898
4899                         if (mutex_trylock(&dev->struct_mutex)) {
4900                                 list_for_each_entry(obj_priv,
4901                                                     &dev_priv->mm.inactive_list,
4902                                                     list)
4903                                         cnt++;
4904                                 mutex_unlock(&dev->struct_mutex);
4905                         }
4906                 }
4907                 spin_unlock(&shrink_list_lock);
4908
4909                 return (cnt / 100) * sysctl_vfs_cache_pressure;
4910         }
4911
4912         spin_lock(&shrink_list_lock);
4913
4914 rescan:
4915         /* first scan for clean buffers */
4916         list_for_each_entry_safe(dev_priv, next_dev,
4917                                  &shrink_list, mm.shrink_list) {
4918                 struct drm_device *dev = dev_priv->dev;
4919
4920                 if (! mutex_trylock(&dev->struct_mutex))
4921                         continue;
4922
4923                 spin_unlock(&shrink_list_lock);
4924                 i915_gem_retire_requests(dev);
4925
4926                 list_for_each_entry_safe(obj_priv, next_obj,
4927                                          &dev_priv->mm.inactive_list,
4928                                          list) {
4929                         if (i915_gem_object_is_purgeable(obj_priv)) {
4930                                 i915_gem_object_unbind(&obj_priv->base);
4931                                 if (--nr_to_scan <= 0)
4932                                         break;
4933                         }
4934                 }
4935
4936                 spin_lock(&shrink_list_lock);
4937                 mutex_unlock(&dev->struct_mutex);
4938
4939                 would_deadlock = 0;
4940
4941                 if (nr_to_scan <= 0)
4942                         break;
4943         }
4944
4945         /* second pass, evict/count anything still on the inactive list */
4946         list_for_each_entry_safe(dev_priv, next_dev,
4947                                  &shrink_list, mm.shrink_list) {
4948                 struct drm_device *dev = dev_priv->dev;
4949
4950                 if (! mutex_trylock(&dev->struct_mutex))
4951                         continue;
4952
4953                 spin_unlock(&shrink_list_lock);
4954
4955                 list_for_each_entry_safe(obj_priv, next_obj,
4956                                          &dev_priv->mm.inactive_list,
4957                                          list) {
4958                         if (nr_to_scan > 0) {
4959                                 i915_gem_object_unbind(&obj_priv->base);
4960                                 nr_to_scan--;
4961                         } else
4962                                 cnt++;
4963                 }
4964
4965                 spin_lock(&shrink_list_lock);
4966                 mutex_unlock(&dev->struct_mutex);
4967
4968                 would_deadlock = 0;
4969         }
4970
4971         if (nr_to_scan) {
4972                 int active = 0;
4973
4974                 /*
4975                  * We are desperate for pages, so as a last resort, wait
4976                  * for the GPU to finish and discard whatever we can.
4977                  * This has a dramatic impact to reduce the number of
4978                  * OOM-killer events whilst running the GPU aggressively.
4979                  */
4980                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4981                         struct drm_device *dev = dev_priv->dev;
4982
4983                         if (!mutex_trylock(&dev->struct_mutex))
4984                                 continue;
4985
4986                         spin_unlock(&shrink_list_lock);
4987
4988                         if (i915_gpu_is_active(dev)) {
4989                                 i915_gpu_idle(dev);
4990                                 active++;
4991                         }
4992
4993                         spin_lock(&shrink_list_lock);
4994                         mutex_unlock(&dev->struct_mutex);
4995                 }
4996
4997                 if (active)
4998                         goto rescan;
4999         }
5000
5001         spin_unlock(&shrink_list_lock);
5002
5003         if (would_deadlock)
5004                 return -1;
5005         else if (cnt > 0)
5006                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5007         else
5008                 return 0;
5009 }
5010
5011 static struct shrinker shrinker = {
5012         .shrink = i915_gem_shrink,
5013         .seeks = DEFAULT_SEEKS,
5014 };
5015
5016 __init void
5017 i915_gem_shrinker_init(void)
5018 {
5019     register_shrinker(&shrinker);
5020 }
5021
5022 __exit void
5023 i915_gem_shrinker_exit(void)
5024 {
5025     unregister_shrinker(&shrinker);
5026 }