drivers/gpu/drm/i915/i915_gem.c

   1 /*
   2  * Copyright © 2008 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *
  26  */
  27
  28 #include "drmP.h"
  29 #include "drm.h"
  30 #include "i915_drm.h"
  31 #include "i915_drv.h"
  32 #include "i915_trace.h"
  33 #include "intel_drv.h"
  34 #include <linux/slab.h>
  35 #include <linux/swap.h>
  36 #include <linux/pci.h>
  37 #include <linux/intel-gtt.h>
  38
  39 static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
  40 static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
  41 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
  42 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
  43 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
  44                                              int write);
  45 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
  46                                                      uint64_t offset,
  47                                                      uint64_t size);
  48 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
  49 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
  50 static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
  51                                            unsigned alignment);
  52 static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
  53 static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
  54                                 struct drm_i915_gem_pwrite *args,
  55                                 struct drm_file *file_priv);
  56 static void i915_gem_free_object_tail(struct drm_gem_object *obj);
  57
  58 static LIST_HEAD(shrink_list);
  59 static DEFINE_SPINLOCK(shrink_list_lock);
  60
  61 static inline bool
  62 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
  63 {
  64         return obj_priv->gtt_space &&
  65                 !obj_priv->active &&
  66                 obj_priv->pin_count == 0;
  67 }
  68
  69 int i915_gem_do_init(struct drm_device *dev, unsigned long start,
  70                      unsigned long end)
  71 {
  72         drm_i915_private_t *dev_priv = dev->dev_private;
  73
  74         if (start >= end ||
  75             (start & (PAGE_SIZE - 1)) != 0 ||
  76             (end & (PAGE_SIZE - 1)) != 0) {
  77                 return -EINVAL;
  78         }
  79
  80         drm_mm_init(&dev_priv->mm.gtt_space, start,
  81                     end - start);
  82
  83         dev->gtt_total = (uint32_t) (end - start);
  84
  85         return 0;
  86 }
  87
  88 int
  89 i915_gem_init_ioctl(struct drm_device *dev, void *data,
  90                     struct drm_file *file_priv)
  91 {
  92         struct drm_i915_gem_init *args = data;
  93         int ret;
  94
  95         mutex_lock(&dev->struct_mutex);
  96         ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
  97         mutex_unlock(&dev->struct_mutex);
  98
  99         return ret;
 100 }
 101
 102 int
 103 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 104                             struct drm_file *file_priv)
 105 {
 106         struct drm_i915_gem_get_aperture *args = data;
 107
 108         if (!(dev->driver->driver_features & DRIVER_GEM))
 109                 return -ENODEV;
 110
 111         args->aper_size = dev->gtt_total;
 112         args->aper_available_size = (args->aper_size -
 113                                      atomic_read(&dev->pin_memory));
 114
 115         return 0;
 116 }
 117
 118
 119 /**
 120  * Creates a new mm object and returns a handle to it.
 121  */
 122 int
 123 i915_gem_create_ioctl(struct drm_device *dev, void *data,
 124                       struct drm_file *file_priv)
 125 {
 126         struct drm_i915_gem_create *args = data;
 127         struct drm_gem_object *obj;
 128         int ret;
 129         u32 handle;
 130
 131         args->size = roundup(args->size, PAGE_SIZE);
 132
 133         /* Allocate the new object */
 134         obj = i915_gem_alloc_object(dev, args->size);
 135         if (obj == NULL)
 136                 return -ENOMEM;
 137
 138         ret = drm_gem_handle_create(file_priv, obj, &handle);
 139         if (ret) {
 140                 drm_gem_object_unreference_unlocked(obj);
 141                 return ret;
 142         }
 143
 144         /* Sink the floating reference from kref_init(handlecount) */
 145         drm_gem_object_handle_unreference_unlocked(obj);
 146
 147         args->handle = handle;
 148         return 0;
 149 }
 150
 151 static inline int
 152 fast_shmem_read(struct page **pages,
 153                 loff_t page_base, int page_offset,
 154                 char __user *data,
 155                 int length)
 156 {
 157         char __iomem *vaddr;
 158         int unwritten;
 159
 160         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 161         if (vaddr == NULL)
 162                 return -ENOMEM;
 163         unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
 164         kunmap_atomic(vaddr, KM_USER0);
 165
 166         if (unwritten)
 167                 return -EFAULT;
 168
 169         return 0;
 170 }
 171
 172 static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
 173 {
 174         drm_i915_private_t *dev_priv = obj->dev->dev_private;
 175         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 176
 177         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
 178                 obj_priv->tiling_mode != I915_TILING_NONE;
 179 }
 180
 181 static inline void
 182 slow_shmem_copy(struct page *dst_page,
 183                 int dst_offset,
 184                 struct page *src_page,
 185                 int src_offset,
 186                 int length)
 187 {
 188         char *dst_vaddr, *src_vaddr;
 189
 190         dst_vaddr = kmap(dst_page);
 191         src_vaddr = kmap(src_page);
 192
 193         memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
 194
 195         kunmap(src_page);
 196         kunmap(dst_page);
 197 }
 198
 199 static inline void
 200 slow_shmem_bit17_copy(struct page *gpu_page,
 201                       int gpu_offset,
 202                       struct page *cpu_page,
 203                       int cpu_offset,
 204                       int length,
 205                       int is_read)
 206 {
 207         char *gpu_vaddr, *cpu_vaddr;
 208
 209         /* Use the unswizzled path if this page isn't affected. */
 210         if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
 211                 if (is_read)
 212                         return slow_shmem_copy(cpu_page, cpu_offset,
 213                                                gpu_page, gpu_offset, length);
 214                 else
 215                         return slow_shmem_copy(gpu_page, gpu_offset,
 216                                                cpu_page, cpu_offset, length);
 217         }
 218
 219         gpu_vaddr = kmap(gpu_page);
 220         cpu_vaddr = kmap(cpu_page);
 221
 222         /* Copy the data, XORing A6 with A17 (1). The user already knows he's
 223          * XORing with the other bits (A9 for Y, A9 and A10 for X)
 224          */
 225         while (length > 0) {
 226                 int cacheline_end = ALIGN(gpu_offset + 1, 64);
 227                 int this_length = min(cacheline_end - gpu_offset, length);
 228                 int swizzled_gpu_offset = gpu_offset ^ 64;
 229
 230                 if (is_read) {
 231                         memcpy(cpu_vaddr + cpu_offset,
 232                                gpu_vaddr + swizzled_gpu_offset,
 233                                this_length);
 234                 } else {
 235                         memcpy(gpu_vaddr + swizzled_gpu_offset,
 236                                cpu_vaddr + cpu_offset,
 237                                this_length);
 238                 }
 239                 cpu_offset += this_length;
 240                 gpu_offset += this_length;
 241                 length -= this_length;
 242         }
 243
 244         kunmap(cpu_page);
 245         kunmap(gpu_page);
 246 }
 247
 248 /**
 249  * This is the fast shmem pread path, which attempts to copy_from_user directly
 250  * from the backing pages of the object to the user's address space.  On a
 251  * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
 252  */
 253 static int
 254 i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
 255                           struct drm_i915_gem_pread *args,
 256                           struct drm_file *file_priv)
 257 {
 258         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 259         ssize_t remain;
 260         loff_t offset, page_base;
 261         char __user *user_data;
 262         int page_offset, page_length;
 263         int ret;
 264
 265         user_data = (char __user *) (uintptr_t) args->data_ptr;
 266         remain = args->size;
 267
 268         mutex_lock(&dev->struct_mutex);
 269
 270         ret = i915_gem_object_get_pages(obj, 0);
 271         if (ret != 0)
 272                 goto fail_unlock;
 273
 274         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 275                                                         args->size);
 276         if (ret != 0)
 277                 goto fail_put_pages;
 278
 279         obj_priv = to_intel_bo(obj);
 280         offset = args->offset;
 281
 282         while (remain > 0) {
 283                 /* Operation in this page
 284                  *
 285                  * page_base = page offset within aperture
 286                  * page_offset = offset within page
 287                  * page_length = bytes to copy for this page
 288                  */
 289                 page_base = (offset & ~(PAGE_SIZE-1));
 290                 page_offset = offset & (PAGE_SIZE-1);
 291                 page_length = remain;
 292                 if ((page_offset + remain) > PAGE_SIZE)
 293                         page_length = PAGE_SIZE - page_offset;
 294
 295                 ret = fast_shmem_read(obj_priv->pages,
 296                                       page_base, page_offset,
 297                                       user_data, page_length);
 298                 if (ret)
 299                         goto fail_put_pages;
 300
 301                 remain -= page_length;
 302                 user_data += page_length;
 303                 offset += page_length;
 304         }
 305
 306 fail_put_pages:
 307         i915_gem_object_put_pages(obj);
 308 fail_unlock:
 309         mutex_unlock(&dev->struct_mutex);
 310
 311         return ret;
 312 }
 313
 314 static int
 315 i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
 316 {
 317         int ret;
 318
 319         ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
 320
 321         /* If we've insufficient memory to map in the pages, attempt
 322          * to make some space by throwing out some old buffers.
 323          */
 324         if (ret == -ENOMEM) {
 325                 struct drm_device *dev = obj->dev;
 326
 327                 ret = i915_gem_evict_something(dev, obj->size,
 328                                                i915_gem_get_gtt_alignment(obj));
 329                 if (ret)
 330                         return ret;
 331
 332                 ret = i915_gem_object_get_pages(obj, 0);
 333         }
 334
 335         return ret;
 336 }
 337
 338 /**
 339  * This is the fallback shmem pread path, which allocates temporary storage
 340  * in kernel space to copy_to_user into outside of the struct_mutex, so we
 341  * can copy out of the object's backing pages while holding the struct mutex
 342  * and not take page faults.
 343  */
 344 static int
 345 i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
 346                           struct drm_i915_gem_pread *args,
 347                           struct drm_file *file_priv)
 348 {
 349         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 350         struct mm_struct *mm = current->mm;
 351         struct page **user_pages;
 352         ssize_t remain;
 353         loff_t offset, pinned_pages, i;
 354         loff_t first_data_page, last_data_page, num_pages;
 355         int shmem_page_index, shmem_page_offset;
 356         int data_page_index,  data_page_offset;
 357         int page_length;
 358         int ret;
 359         uint64_t data_ptr = args->data_ptr;
 360         int do_bit17_swizzling;
 361
 362         remain = args->size;
 363
 364         /* Pin the user pages containing the data.  We can't fault while
 365          * holding the struct mutex, yet we want to hold it while
 366          * dereferencing the user data.
 367          */
 368         first_data_page = data_ptr / PAGE_SIZE;
 369         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 370         num_pages = last_data_page - first_data_page + 1;
 371
 372         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 373         if (user_pages == NULL)
 374                 return -ENOMEM;
 375
 376         down_read(&mm->mmap_sem);
 377         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 378                                       num_pages, 1, 0, user_pages, NULL);
 379         up_read(&mm->mmap_sem);
 380         if (pinned_pages < num_pages) {
 381                 ret = -EFAULT;
 382                 goto fail_put_user_pages;
 383         }
 384
 385         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 386
 387         mutex_lock(&dev->struct_mutex);
 388
 389         ret = i915_gem_object_get_pages_or_evict(obj);
 390         if (ret)
 391                 goto fail_unlock;
 392
 393         ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
 394                                                         args->size);
 395         if (ret != 0)
 396                 goto fail_put_pages;
 397
 398         obj_priv = to_intel_bo(obj);
 399         offset = args->offset;
 400
 401         while (remain > 0) {
 402                 /* Operation in this page
 403                  *
 404                  * shmem_page_index = page number within shmem file
 405                  * shmem_page_offset = offset within page in shmem file
 406                  * data_page_index = page number in get_user_pages return
 407                  * data_page_offset = offset with data_page_index page.
 408                  * page_length = bytes to copy for this page
 409                  */
 410                 shmem_page_index = offset / PAGE_SIZE;
 411                 shmem_page_offset = offset & ~PAGE_MASK;
 412                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 413                 data_page_offset = data_ptr & ~PAGE_MASK;
 414
 415                 page_length = remain;
 416                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 417                         page_length = PAGE_SIZE - shmem_page_offset;
 418                 if ((data_page_offset + page_length) > PAGE_SIZE)
 419                         page_length = PAGE_SIZE - data_page_offset;
 420
 421                 if (do_bit17_swizzling) {
 422                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 423                                               shmem_page_offset,
 424                                               user_pages[data_page_index],
 425                                               data_page_offset,
 426                                               page_length,
 427                                               1);
 428                 } else {
 429                         slow_shmem_copy(user_pages[data_page_index],
 430                                         data_page_offset,
 431                                         obj_priv->pages[shmem_page_index],
 432                                         shmem_page_offset,
 433                                         page_length);
 434                 }
 435
 436                 remain -= page_length;
 437                 data_ptr += page_length;
 438                 offset += page_length;
 439         }
 440
 441 fail_put_pages:
 442         i915_gem_object_put_pages(obj);
 443 fail_unlock:
 444         mutex_unlock(&dev->struct_mutex);
 445 fail_put_user_pages:
 446         for (i = 0; i < pinned_pages; i++) {
 447                 SetPageDirty(user_pages[i]);
 448                 page_cache_release(user_pages[i]);
 449         }
 450         drm_free_large(user_pages);
 451
 452         return ret;
 453 }
 454
 455 /**
 456  * Reads data from the object referenced by handle.
 457  *
 458  * On error, the contents of *data are undefined.
 459  */
 460 int
 461 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 462                      struct drm_file *file_priv)
 463 {
 464         struct drm_i915_gem_pread *args = data;
 465         struct drm_gem_object *obj;
 466         struct drm_i915_gem_object *obj_priv;
 467         int ret;
 468
 469         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 470         if (obj == NULL)
 471                 return -ENOENT;
 472         obj_priv = to_intel_bo(obj);
 473
 474         /* Bounds check source.  */
 475         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 476                 ret = -EINVAL;
 477                 goto err;
 478         }
 479
 480         if (!access_ok(VERIFY_WRITE,
 481                        (char __user *)(uintptr_t)args->data_ptr,
 482                        args->size)) {
 483                 ret = -EFAULT;
 484                 goto err;
 485         }
 486
 487         if (i915_gem_object_needs_bit17_swizzle(obj)) {
 488                 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
 489         } else {
 490                 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
 491                 if (ret != 0)
 492                         ret = i915_gem_shmem_pread_slow(dev, obj, args,
 493                                                         file_priv);
 494         }
 495
 496 err:
 497         drm_gem_object_unreference_unlocked(obj);
 498         return ret;
 499 }
 500
 501 /* This is the fast write path which cannot handle
 502  * page faults in the source data
 503  */
 504
 505 static inline int
 506 fast_user_write(struct io_mapping *mapping,
 507                 loff_t page_base, int page_offset,
 508                 char __user *user_data,
 509                 int length)
 510 {
 511         char *vaddr_atomic;
 512         unsigned long unwritten;
 513
 514         vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
 515         unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
 516                                                       user_data, length);
 517         io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
 518         if (unwritten)
 519                 return -EFAULT;
 520         return 0;
 521 }
 522
 523 /* Here's the write path which can sleep for
 524  * page faults
 525  */
 526
 527 static inline void
 528 slow_kernel_write(struct io_mapping *mapping,
 529                   loff_t gtt_base, int gtt_offset,
 530                   struct page *user_page, int user_offset,
 531                   int length)
 532 {
 533         char __iomem *dst_vaddr;
 534         char *src_vaddr;
 535
 536         dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
 537         src_vaddr = kmap(user_page);
 538
 539         memcpy_toio(dst_vaddr + gtt_offset,
 540                     src_vaddr + user_offset,
 541                     length);
 542
 543         kunmap(user_page);
 544         io_mapping_unmap(dst_vaddr);
 545 }
 546
 547 static inline int
 548 fast_shmem_write(struct page **pages,
 549                  loff_t page_base, int page_offset,
 550                  char __user *data,
 551                  int length)
 552 {
 553         char __iomem *vaddr;
 554         unsigned long unwritten;
 555
 556         vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
 557         if (vaddr == NULL)
 558                 return -ENOMEM;
 559         unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
 560         kunmap_atomic(vaddr, KM_USER0);
 561
 562         if (unwritten)
 563                 return -EFAULT;
 564         return 0;
 565 }
 566
 567 /**
 568  * This is the fast pwrite path, where we copy the data directly from the
 569  * user into the GTT, uncached.
 570  */
 571 static int
 572 i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 573                          struct drm_i915_gem_pwrite *args,
 574                          struct drm_file *file_priv)
 575 {
 576         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 577         drm_i915_private_t *dev_priv = dev->dev_private;
 578         ssize_t remain;
 579         loff_t offset, page_base;
 580         char __user *user_data;
 581         int page_offset, page_length;
 582         int ret;
 583
 584         user_data = (char __user *) (uintptr_t) args->data_ptr;
 585         remain = args->size;
 586
 587
 588         mutex_lock(&dev->struct_mutex);
 589         ret = i915_gem_object_pin(obj, 0);
 590         if (ret) {
 591                 mutex_unlock(&dev->struct_mutex);
 592                 return ret;
 593         }
 594         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 595         if (ret)
 596                 goto fail;
 597
 598         obj_priv = to_intel_bo(obj);
 599         offset = obj_priv->gtt_offset + args->offset;
 600
 601         while (remain > 0) {
 602                 /* Operation in this page
 603                  *
 604                  * page_base = page offset within aperture
 605                  * page_offset = offset within page
 606                  * page_length = bytes to copy for this page
 607                  */
 608                 page_base = (offset & ~(PAGE_SIZE-1));
 609                 page_offset = offset & (PAGE_SIZE-1);
 610                 page_length = remain;
 611                 if ((page_offset + remain) > PAGE_SIZE)
 612                         page_length = PAGE_SIZE - page_offset;
 613
 614                 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
 615                                        page_offset, user_data, page_length);
 616
 617                 /* If we get a fault while copying data, then (presumably) our
 618                  * source page isn't available.  Return the error and we'll
 619                  * retry in the slow path.
 620                  */
 621                 if (ret)
 622                         goto fail;
 623
 624                 remain -= page_length;
 625                 user_data += page_length;
 626                 offset += page_length;
 627         }
 628
 629 fail:
 630         i915_gem_object_unpin(obj);
 631         mutex_unlock(&dev->struct_mutex);
 632
 633         return ret;
 634 }
 635
 636 /**
 637  * This is the fallback GTT pwrite path, which uses get_user_pages to pin
 638  * the memory and maps it using kmap_atomic for copying.
 639  *
 640  * This code resulted in x11perf -rgb10text consuming about 10% more CPU
 641  * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
 642  */
 643 static int
 644 i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 645                          struct drm_i915_gem_pwrite *args,
 646                          struct drm_file *file_priv)
 647 {
 648         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 649         drm_i915_private_t *dev_priv = dev->dev_private;
 650         ssize_t remain;
 651         loff_t gtt_page_base, offset;
 652         loff_t first_data_page, last_data_page, num_pages;
 653         loff_t pinned_pages, i;
 654         struct page **user_pages;
 655         struct mm_struct *mm = current->mm;
 656         int gtt_page_offset, data_page_offset, data_page_index, page_length;
 657         int ret;
 658         uint64_t data_ptr = args->data_ptr;
 659
 660         remain = args->size;
 661
 662         /* Pin the user pages containing the data.  We can't fault while
 663          * holding the struct mutex, and all of the pwrite implementations
 664          * want to hold it while dereferencing the user data.
 665          */
 666         first_data_page = data_ptr / PAGE_SIZE;
 667         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 668         num_pages = last_data_page - first_data_page + 1;
 669
 670         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 671         if (user_pages == NULL)
 672                 return -ENOMEM;
 673
 674         down_read(&mm->mmap_sem);
 675         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 676                                       num_pages, 0, 0, user_pages, NULL);
 677         up_read(&mm->mmap_sem);
 678         if (pinned_pages < num_pages) {
 679                 ret = -EFAULT;
 680                 goto out_unpin_pages;
 681         }
 682
 683         mutex_lock(&dev->struct_mutex);
 684         ret = i915_gem_object_pin(obj, 0);
 685         if (ret)
 686                 goto out_unlock;
 687
 688         ret = i915_gem_object_set_to_gtt_domain(obj, 1);
 689         if (ret)
 690                 goto out_unpin_object;
 691
 692         obj_priv = to_intel_bo(obj);
 693         offset = obj_priv->gtt_offset + args->offset;
 694
 695         while (remain > 0) {
 696                 /* Operation in this page
 697                  *
 698                  * gtt_page_base = page offset within aperture
 699                  * gtt_page_offset = offset within page in aperture
 700                  * data_page_index = page number in get_user_pages return
 701                  * data_page_offset = offset with data_page_index page.
 702                  * page_length = bytes to copy for this page
 703                  */
 704                 gtt_page_base = offset & PAGE_MASK;
 705                 gtt_page_offset = offset & ~PAGE_MASK;
 706                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 707                 data_page_offset = data_ptr & ~PAGE_MASK;
 708
 709                 page_length = remain;
 710                 if ((gtt_page_offset + page_length) > PAGE_SIZE)
 711                         page_length = PAGE_SIZE - gtt_page_offset;
 712                 if ((data_page_offset + page_length) > PAGE_SIZE)
 713                         page_length = PAGE_SIZE - data_page_offset;
 714
 715                 slow_kernel_write(dev_priv->mm.gtt_mapping,
 716                                   gtt_page_base, gtt_page_offset,
 717                                   user_pages[data_page_index],
 718                                   data_page_offset,
 719                                   page_length);
 720
 721                 remain -= page_length;
 722                 offset += page_length;
 723                 data_ptr += page_length;
 724         }
 725
 726 out_unpin_object:
 727         i915_gem_object_unpin(obj);
 728 out_unlock:
 729         mutex_unlock(&dev->struct_mutex);
 730 out_unpin_pages:
 731         for (i = 0; i < pinned_pages; i++)
 732                 page_cache_release(user_pages[i]);
 733         drm_free_large(user_pages);
 734
 735         return ret;
 736 }
 737
 738 /**
 739  * This is the fast shmem pwrite path, which attempts to directly
 740  * copy_from_user into the kmapped pages backing the object.
 741  */
 742 static int
 743 i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
 744                            struct drm_i915_gem_pwrite *args,
 745                            struct drm_file *file_priv)
 746 {
 747         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 748         ssize_t remain;
 749         loff_t offset, page_base;
 750         char __user *user_data;
 751         int page_offset, page_length;
 752         int ret;
 753
 754         user_data = (char __user *) (uintptr_t) args->data_ptr;
 755         remain = args->size;
 756
 757         mutex_lock(&dev->struct_mutex);
 758
 759         ret = i915_gem_object_get_pages(obj, 0);
 760         if (ret != 0)
 761                 goto fail_unlock;
 762
 763         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 764         if (ret != 0)
 765                 goto fail_put_pages;
 766
 767         obj_priv = to_intel_bo(obj);
 768         offset = args->offset;
 769         obj_priv->dirty = 1;
 770
 771         while (remain > 0) {
 772                 /* Operation in this page
 773                  *
 774                  * page_base = page offset within aperture
 775                  * page_offset = offset within page
 776                  * page_length = bytes to copy for this page
 777                  */
 778                 page_base = (offset & ~(PAGE_SIZE-1));
 779                 page_offset = offset & (PAGE_SIZE-1);
 780                 page_length = remain;
 781                 if ((page_offset + remain) > PAGE_SIZE)
 782                         page_length = PAGE_SIZE - page_offset;
 783
 784                 ret = fast_shmem_write(obj_priv->pages,
 785                                        page_base, page_offset,
 786                                        user_data, page_length);
 787                 if (ret)
 788                         goto fail_put_pages;
 789
 790                 remain -= page_length;
 791                 user_data += page_length;
 792                 offset += page_length;
 793         }
 794
 795 fail_put_pages:
 796         i915_gem_object_put_pages(obj);
 797 fail_unlock:
 798         mutex_unlock(&dev->struct_mutex);
 799
 800         return ret;
 801 }
 802
 803 /**
 804  * This is the fallback shmem pwrite path, which uses get_user_pages to pin
 805  * the memory and maps it using kmap_atomic for copying.
 806  *
 807  * This avoids taking mmap_sem for faulting on the user's address while the
 808  * struct_mutex is held.
 809  */
 810 static int
 811 i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
 812                            struct drm_i915_gem_pwrite *args,
 813                            struct drm_file *file_priv)
 814 {
 815         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
 816         struct mm_struct *mm = current->mm;
 817         struct page **user_pages;
 818         ssize_t remain;
 819         loff_t offset, pinned_pages, i;
 820         loff_t first_data_page, last_data_page, num_pages;
 821         int shmem_page_index, shmem_page_offset;
 822         int data_page_index,  data_page_offset;
 823         int page_length;
 824         int ret;
 825         uint64_t data_ptr = args->data_ptr;
 826         int do_bit17_swizzling;
 827
 828         remain = args->size;
 829
 830         /* Pin the user pages containing the data.  We can't fault while
 831          * holding the struct mutex, and all of the pwrite implementations
 832          * want to hold it while dereferencing the user data.
 833          */
 834         first_data_page = data_ptr / PAGE_SIZE;
 835         last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
 836         num_pages = last_data_page - first_data_page + 1;
 837
 838         user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
 839         if (user_pages == NULL)
 840                 return -ENOMEM;
 841
 842         down_read(&mm->mmap_sem);
 843         pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
 844                                       num_pages, 0, 0, user_pages, NULL);
 845         up_read(&mm->mmap_sem);
 846         if (pinned_pages < num_pages) {
 847                 ret = -EFAULT;
 848                 goto fail_put_user_pages;
 849         }
 850
 851         do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 852
 853         mutex_lock(&dev->struct_mutex);
 854
 855         ret = i915_gem_object_get_pages_or_evict(obj);
 856         if (ret)
 857                 goto fail_unlock;
 858
 859         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
 860         if (ret != 0)
 861                 goto fail_put_pages;
 862
 863         obj_priv = to_intel_bo(obj);
 864         offset = args->offset;
 865         obj_priv->dirty = 1;
 866
 867         while (remain > 0) {
 868                 /* Operation in this page
 869                  *
 870                  * shmem_page_index = page number within shmem file
 871                  * shmem_page_offset = offset within page in shmem file
 872                  * data_page_index = page number in get_user_pages return
 873                  * data_page_offset = offset with data_page_index page.
 874                  * page_length = bytes to copy for this page
 875                  */
 876                 shmem_page_index = offset / PAGE_SIZE;
 877                 shmem_page_offset = offset & ~PAGE_MASK;
 878                 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
 879                 data_page_offset = data_ptr & ~PAGE_MASK;
 880
 881                 page_length = remain;
 882                 if ((shmem_page_offset + page_length) > PAGE_SIZE)
 883                         page_length = PAGE_SIZE - shmem_page_offset;
 884                 if ((data_page_offset + page_length) > PAGE_SIZE)
 885                         page_length = PAGE_SIZE - data_page_offset;
 886
 887                 if (do_bit17_swizzling) {
 888                         slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
 889                                               shmem_page_offset,
 890                                               user_pages[data_page_index],
 891                                               data_page_offset,
 892                                               page_length,
 893                                               0);
 894                 } else {
 895                         slow_shmem_copy(obj_priv->pages[shmem_page_index],
 896                                         shmem_page_offset,
 897                                         user_pages[data_page_index],
 898                                         data_page_offset,
 899                                         page_length);
 900                 }
 901
 902                 remain -= page_length;
 903                 data_ptr += page_length;
 904                 offset += page_length;
 905         }
 906
 907 fail_put_pages:
 908         i915_gem_object_put_pages(obj);
 909 fail_unlock:
 910         mutex_unlock(&dev->struct_mutex);
 911 fail_put_user_pages:
 912         for (i = 0; i < pinned_pages; i++)
 913                 page_cache_release(user_pages[i]);
 914         drm_free_large(user_pages);
 915
 916         return ret;
 917 }
 918
 919 /**
 920  * Writes data to the object referenced by handle.
 921  *
 922  * On error, the contents of the buffer that were to be modified are undefined.
 923  */
 924 int
 925 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 926                       struct drm_file *file_priv)
 927 {
 928         struct drm_i915_gem_pwrite *args = data;
 929         struct drm_gem_object *obj;
 930         struct drm_i915_gem_object *obj_priv;
 931         int ret = 0;
 932
 933         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
 934         if (obj == NULL)
 935                 return -ENOENT;
 936         obj_priv = to_intel_bo(obj);
 937
 938         /* Bounds check destination. */
 939         if (args->offset > obj->size || args->size > obj->size - args->offset) {
 940                 ret = -EINVAL;
 941                 goto err;
 942         }
 943
 944         if (!access_ok(VERIFY_READ,
 945                        (char __user *)(uintptr_t)args->data_ptr,
 946                        args->size)) {
 947                 ret = -EFAULT;
 948                 goto err;
 949         }
 950
 951         /* We can only do the GTT pwrite on untiled buffers, as otherwise
 952          * it would end up going through the fenced access, and we'll get
 953          * different detiling behavior between reading and writing.
 954          * pread/pwrite currently are reading and writing from the CPU
 955          * perspective, requiring manual detiling by the client.
 956          */
 957         if (obj_priv->phys_obj)
 958                 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
 959         else if (obj_priv->tiling_mode == I915_TILING_NONE &&
 960                  dev->gtt_total != 0 &&
 961                  obj->write_domain != I915_GEM_DOMAIN_CPU) {
 962                 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
 963                 if (ret == -EFAULT) {
 964                         ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
 965                                                        file_priv);
 966                 }
 967         } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
 968                 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
 969         } else {
 970                 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
 971                 if (ret == -EFAULT) {
 972                         ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
 973                                                          file_priv);
 974                 }
 975         }
 976
 977 #if WATCH_PWRITE
 978         if (ret)
 979                 DRM_INFO("pwrite failed %d\n", ret);
 980 #endif
 981
 982 err:
 983         drm_gem_object_unreference_unlocked(obj);
 984         return ret;
 985 }
 986
 987 /**
 988  * Called when user space prepares to use an object with the CPU, either
 989  * through the mmap ioctl's mapping or a GTT mapping.
 990  */
 991 int
 992 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 993                           struct drm_file *file_priv)
 994 {
 995         struct drm_i915_private *dev_priv = dev->dev_private;
 996         struct drm_i915_gem_set_domain *args = data;
 997         struct drm_gem_object *obj;
 998         struct drm_i915_gem_object *obj_priv;
 999         uint32_t read_domains = args->read_domains;
1000         uint32_t write_domain = args->write_domain;
1001         int ret;
1002
1003         if (!(dev->driver->driver_features & DRIVER_GEM))
1004                 return -ENODEV;
1005
1006         /* Only handle setting domains to types used by the CPU. */
1007         if (write_domain & I915_GEM_GPU_DOMAINS)
1008                 return -EINVAL;
1009
1010         if (read_domains & I915_GEM_GPU_DOMAINS)
1011                 return -EINVAL;
1012
1013         /* Having something in the write domain implies it's in the read
1014          * domain, and only that read domain.  Enforce that in the request.
1015          */
1016         if (write_domain != 0 && read_domains != write_domain)
1017                 return -EINVAL;
1018
1019         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1020         if (obj == NULL)
1021                 return -ENOENT;
1022         obj_priv = to_intel_bo(obj);
1023
1024         mutex_lock(&dev->struct_mutex);
1025
1026         intel_mark_busy(dev, obj);
1027
1028 #if WATCH_BUF
1029         DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
1030                  obj, obj->size, read_domains, write_domain);
1031 #endif
1032         if (read_domains & I915_GEM_DOMAIN_GTT) {
1033                 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1034
1035                 /* Update the LRU on the fence for the CPU access that's
1036                  * about to occur.
1037                  */
1038                 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1039                         struct drm_i915_fence_reg *reg =
1040                                 &dev_priv->fence_regs[obj_priv->fence_reg];
1041                         list_move_tail(&reg->lru_list,
1042                                        &dev_priv->mm.fence_list);
1043                 }
1044
1045                 /* Silently promote "you're not bound, there was nothing to do"
1046                  * to success, since the client was just asking us to
1047                  * make sure everything was done.
1048                  */
1049                 if (ret == -EINVAL)
1050                         ret = 0;
1051         } else {
1052                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1053         }
1054
1055
1056         /* Maintain LRU order of "inactive" objects */
1057         if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1058                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1059
1060         drm_gem_object_unreference(obj);
1061         mutex_unlock(&dev->struct_mutex);
1062         return ret;
1063 }
1064
1065 /**
1066  * Called when user space has done writes to this buffer
1067  */
1068 int
1069 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1070                       struct drm_file *file_priv)
1071 {
1072         struct drm_i915_gem_sw_finish *args = data;
1073         struct drm_gem_object *obj;
1074         struct drm_i915_gem_object *obj_priv;
1075         int ret = 0;
1076
1077         if (!(dev->driver->driver_features & DRIVER_GEM))
1078                 return -ENODEV;
1079
1080         mutex_lock(&dev->struct_mutex);
1081         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1082         if (obj == NULL) {
1083                 mutex_unlock(&dev->struct_mutex);
1084                 return -ENOENT;
1085         }
1086
1087 #if WATCH_BUF
1088         DRM_INFO("%s: sw_finish %d (%p %zd)\n",
1089                  __func__, args->handle, obj, obj->size);
1090 #endif
1091         obj_priv = to_intel_bo(obj);
1092
1093         /* Pinned buffers may be scanout, so flush the cache */
1094         if (obj_priv->pin_count)
1095                 i915_gem_object_flush_cpu_write_domain(obj);
1096
1097         drm_gem_object_unreference(obj);
1098         mutex_unlock(&dev->struct_mutex);
1099         return ret;
1100 }
1101
1102 /**
1103  * Maps the contents of an object, returning the address it is mapped
1104  * into.
1105  *
1106  * While the mapping holds a reference on the contents of the object, it doesn't
1107  * imply a ref on the object itself.
1108  */
1109 int
1110 i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1111                    struct drm_file *file_priv)
1112 {
1113         struct drm_i915_gem_mmap *args = data;
1114         struct drm_gem_object *obj;
1115         loff_t offset;
1116         unsigned long addr;
1117
1118         if (!(dev->driver->driver_features & DRIVER_GEM))
1119                 return -ENODEV;
1120
1121         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1122         if (obj == NULL)
1123                 return -ENOENT;
1124
1125         offset = args->offset;
1126
1127         down_write(&current->mm->mmap_sem);
1128         addr = do_mmap(obj->filp, 0, args->size,
1129                        PROT_READ | PROT_WRITE, MAP_SHARED,
1130                        args->offset);
1131         up_write(&current->mm->mmap_sem);
1132         drm_gem_object_unreference_unlocked(obj);
1133         if (IS_ERR((void *)addr))
1134                 return addr;
1135
1136         args->addr_ptr = (uint64_t) addr;
1137
1138         return 0;
1139 }
1140
1141 /**
1142  * i915_gem_fault - fault a page into the GTT
1143  * vma: VMA in question
1144  * vmf: fault info
1145  *
1146  * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1147  * from userspace.  The fault handler takes care of binding the object to
1148  * the GTT (if needed), allocating and programming a fence register (again,
1149  * only if needed based on whether the old reg is still valid or the object
1150  * is tiled) and inserting a new PTE into the faulting process.
1151  *
1152  * Note that the faulting process may involve evicting existing objects
1153  * from the GTT and/or fence registers to make room.  So performance may
1154  * suffer if the GTT working set is large or there are few fence registers
1155  * left.
1156  */
1157 int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1158 {
1159         struct drm_gem_object *obj = vma->vm_private_data;
1160         struct drm_device *dev = obj->dev;
1161         drm_i915_private_t *dev_priv = dev->dev_private;
1162         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1163         pgoff_t page_offset;
1164         unsigned long pfn;
1165         int ret = 0;
1166         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1167
1168         /* We don't use vmf->pgoff since that has the fake offset */
1169         page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1170                 PAGE_SHIFT;
1171
1172         /* Now bind it into the GTT if needed */
1173         mutex_lock(&dev->struct_mutex);
1174         if (!obj_priv->gtt_space) {
1175                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1176                 if (ret)
1177                         goto unlock;
1178
1179                 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1180                 if (ret)
1181                         goto unlock;
1182         }
1183
1184         /* Need a new fence register? */
1185         if (obj_priv->tiling_mode != I915_TILING_NONE) {
1186                 ret = i915_gem_object_get_fence_reg(obj);
1187                 if (ret)
1188                         goto unlock;
1189         }
1190
1191         if (i915_gem_object_is_inactive(obj_priv))
1192                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1193
1194         pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1195                 page_offset;
1196
1197         /* Finally, remap it using the new GTT offset */
1198         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1199 unlock:
1200         mutex_unlock(&dev->struct_mutex);
1201
1202         switch (ret) {
1203         case 0:
1204         case -ERESTARTSYS:
1205                 return VM_FAULT_NOPAGE;
1206         case -ENOMEM:
1207         case -EAGAIN:
1208                 return VM_FAULT_OOM;
1209         default:
1210                 return VM_FAULT_SIGBUS;
1211         }
1212 }
1213
1214 /**
1215  * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1216  * @obj: obj in question
1217  *
1218  * GEM memory mapping works by handing back to userspace a fake mmap offset
1219  * it can use in a subsequent mmap(2) call.  The DRM core code then looks
1220  * up the object based on the offset and sets up the various memory mapping
1221  * structures.
1222  *
1223  * This routine allocates and attaches a fake offset for @obj.
1224  */
1225 static int
1226 i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1227 {
1228         struct drm_device *dev = obj->dev;
1229         struct drm_gem_mm *mm = dev->mm_private;
1230         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1231         struct drm_map_list *list;
1232         struct drm_local_map *map;
1233         int ret = 0;
1234
1235         /* Set the object up for mmap'ing */
1236         list = &obj->map_list;
1237         list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1238         if (!list->map)
1239                 return -ENOMEM;
1240
1241         map = list->map;
1242         map->type = _DRM_GEM;
1243         map->size = obj->size;
1244         map->handle = obj;
1245
1246         /* Get a DRM GEM mmap offset allocated... */
1247         list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1248                                                     obj->size / PAGE_SIZE, 0, 0);
1249         if (!list->file_offset_node) {
1250                 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1251                 ret = -ENOMEM;
1252                 goto out_free_list;
1253         }
1254
1255         list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1256                                                   obj->size / PAGE_SIZE, 0);
1257         if (!list->file_offset_node) {
1258                 ret = -ENOMEM;
1259                 goto out_free_list;
1260         }
1261
1262         list->hash.key = list->file_offset_node->start;
1263         if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1264                 DRM_ERROR("failed to add to map hash\n");
1265                 ret = -ENOMEM;
1266                 goto out_free_mm;
1267         }
1268
1269         /* By now we should be all set, any drm_mmap request on the offset
1270          * below will get to our mmap & fault handler */
1271         obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1272
1273         return 0;
1274
1275 out_free_mm:
1276         drm_mm_put_block(list->file_offset_node);
1277 out_free_list:
1278         kfree(list->map);
1279
1280         return ret;
1281 }
1282
1283 /**
1284  * i915_gem_release_mmap - remove physical page mappings
1285  * @obj: obj in question
1286  *
1287  * Preserve the reservation of the mmapping with the DRM core code, but
1288  * relinquish ownership of the pages back to the system.
1289  *
1290  * It is vital that we remove the page mapping if we have mapped a tiled
1291  * object through the GTT and then lose the fence register due to
1292  * resource pressure. Similarly if the object has been moved out of the
1293  * aperture, than pages mapped into userspace must be revoked. Removing the
1294  * mapping will then trigger a page fault on the next user access, allowing
1295  * fixup by i915_gem_fault().
1296  */
1297 void
1298 i915_gem_release_mmap(struct drm_gem_object *obj)
1299 {
1300         struct drm_device *dev = obj->dev;
1301         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1302
1303         if (dev->dev_mapping)
1304                 unmap_mapping_range(dev->dev_mapping,
1305                                     obj_priv->mmap_offset, obj->size, 1);
1306 }
1307
1308 static void
1309 i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1310 {
1311         struct drm_device *dev = obj->dev;
1312         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1313         struct drm_gem_mm *mm = dev->mm_private;
1314         struct drm_map_list *list;
1315
1316         list = &obj->map_list;
1317         drm_ht_remove_item(&mm->offset_hash, &list->hash);
1318
1319         if (list->file_offset_node) {
1320                 drm_mm_put_block(list->file_offset_node);
1321                 list->file_offset_node = NULL;
1322         }
1323
1324         if (list->map) {
1325                 kfree(list->map);
1326                 list->map = NULL;
1327         }
1328
1329         obj_priv->mmap_offset = 0;
1330 }
1331
1332 /**
1333  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1334  * @obj: object to check
1335  *
1336  * Return the required GTT alignment for an object, taking into account
1337  * potential fence register mapping if needed.
1338  */
1339 static uint32_t
1340 i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1341 {
1342         struct drm_device *dev = obj->dev;
1343         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1344         int start, i;
1345
1346         /*
1347          * Minimum alignment is 4k (GTT page size), but might be greater
1348          * if a fence register is needed for the object.
1349          */
1350         if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1351                 return 4096;
1352
1353         /*
1354          * Previous chips need to be aligned to the size of the smallest
1355          * fence register that can contain the object.
1356          */
1357         if (IS_I9XX(dev))
1358                 start = 1024*1024;
1359         else
1360                 start = 512*1024;
1361
1362         for (i = start; i < obj->size; i <<= 1)
1363                 ;
1364
1365         return i;
1366 }
1367
1368 /**
1369  * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1370  * @dev: DRM device
1371  * @data: GTT mapping ioctl data
1372  * @file_priv: GEM object info
1373  *
1374  * Simply returns the fake offset to userspace so it can mmap it.
1375  * The mmap call will end up in drm_gem_mmap(), which will set things
1376  * up so we can get faults in the handler above.
1377  *
1378  * The fault handler will take care of binding the object into the GTT
1379  * (since it may have been evicted to make room for something), allocating
1380  * a fence register, and mapping the appropriate aperture address into
1381  * userspace.
1382  */
1383 int
1384 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1385                         struct drm_file *file_priv)
1386 {
1387         struct drm_i915_gem_mmap_gtt *args = data;
1388         struct drm_gem_object *obj;
1389         struct drm_i915_gem_object *obj_priv;
1390         int ret;
1391
1392         if (!(dev->driver->driver_features & DRIVER_GEM))
1393                 return -ENODEV;
1394
1395         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1396         if (obj == NULL)
1397                 return -ENOENT;
1398
1399         mutex_lock(&dev->struct_mutex);
1400
1401         obj_priv = to_intel_bo(obj);
1402
1403         if (obj_priv->madv != I915_MADV_WILLNEED) {
1404                 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1405                 drm_gem_object_unreference(obj);
1406                 mutex_unlock(&dev->struct_mutex);
1407                 return -EINVAL;
1408         }
1409
1410
1411         if (!obj_priv->mmap_offset) {
1412                 ret = i915_gem_create_mmap_offset(obj);
1413                 if (ret) {
1414                         drm_gem_object_unreference(obj);
1415                         mutex_unlock(&dev->struct_mutex);
1416                         return ret;
1417                 }
1418         }
1419
1420         args->offset = obj_priv->mmap_offset;
1421
1422         /*
1423          * Pull it into the GTT so that we have a page list (makes the
1424          * initial fault faster and any subsequent flushing possible).
1425          */
1426         if (!obj_priv->agp_mem) {
1427                 ret = i915_gem_object_bind_to_gtt(obj, 0);
1428                 if (ret) {
1429                         drm_gem_object_unreference(obj);
1430                         mutex_unlock(&dev->struct_mutex);
1431                         return ret;
1432                 }
1433         }
1434
1435         drm_gem_object_unreference(obj);
1436         mutex_unlock(&dev->struct_mutex);
1437
1438         return 0;
1439 }
1440
1441 void
1442 i915_gem_object_put_pages(struct drm_gem_object *obj)
1443 {
1444         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1445         int page_count = obj->size / PAGE_SIZE;
1446         int i;
1447
1448         BUG_ON(obj_priv->pages_refcount == 0);
1449         BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
1450
1451         if (--obj_priv->pages_refcount != 0)
1452                 return;
1453
1454         if (obj_priv->tiling_mode != I915_TILING_NONE)
1455                 i915_gem_object_save_bit_17_swizzle(obj);
1456
1457         if (obj_priv->madv == I915_MADV_DONTNEED)
1458                 obj_priv->dirty = 0;
1459
1460         for (i = 0; i < page_count; i++) {
1461                 if (obj_priv->dirty)
1462                         set_page_dirty(obj_priv->pages[i]);
1463
1464                 if (obj_priv->madv == I915_MADV_WILLNEED)
1465                         mark_page_accessed(obj_priv->pages[i]);
1466
1467                 page_cache_release(obj_priv->pages[i]);
1468         }
1469         obj_priv->dirty = 0;
1470
1471         drm_free_large(obj_priv->pages);
1472         obj_priv->pages = NULL;
1473 }
1474
1475 static void
1476 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1477                                struct intel_ring_buffer *ring)
1478 {
1479         struct drm_device *dev = obj->dev;
1480         drm_i915_private_t *dev_priv = dev->dev_private;
1481         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1482         BUG_ON(ring == NULL);
1483         obj_priv->ring = ring;
1484
1485         /* Add a reference if we're newly entering the active list. */
1486         if (!obj_priv->active) {
1487                 drm_gem_object_reference(obj);
1488                 obj_priv->active = 1;
1489         }
1490         /* Move from whatever list we were on to the tail of execution. */
1491         spin_lock(&dev_priv->mm.active_list_lock);
1492         list_move_tail(&obj_priv->list, &ring->active_list);
1493         spin_unlock(&dev_priv->mm.active_list_lock);
1494         obj_priv->last_rendering_seqno = seqno;
1495 }
1496
1497 static void
1498 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1499 {
1500         struct drm_device *dev = obj->dev;
1501         drm_i915_private_t *dev_priv = dev->dev_private;
1502         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1503
1504         BUG_ON(!obj_priv->active);
1505         list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1506         obj_priv->last_rendering_seqno = 0;
1507 }
1508
1509 /* Immediately discard the backing storage */
1510 static void
1511 i915_gem_object_truncate(struct drm_gem_object *obj)
1512 {
1513         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1514         struct inode *inode;
1515
1516         /* Our goal here is to return as much of the memory as
1517          * is possible back to the system as we are called from OOM.
1518          * To do this we must instruct the shmfs to drop all of its
1519          * backing pages, *now*. Here we mirror the actions taken
1520          * when by shmem_delete_inode() to release the backing store.
1521          */
1522         inode = obj->filp->f_path.dentry->d_inode;
1523         truncate_inode_pages(inode->i_mapping, 0);
1524         if (inode->i_op->truncate_range)
1525                 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
1526
1527         obj_priv->madv = __I915_MADV_PURGED;
1528 }
1529
1530 static inline int
1531 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1532 {
1533         return obj_priv->madv == I915_MADV_DONTNEED;
1534 }
1535
1536 static void
1537 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1538 {
1539         struct drm_device *dev = obj->dev;
1540         drm_i915_private_t *dev_priv = dev->dev_private;
1541         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1542
1543         i915_verify_inactive(dev, __FILE__, __LINE__);
1544         if (obj_priv->pin_count != 0)
1545                 list_del_init(&obj_priv->list);
1546         else
1547                 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1548
1549         BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1550
1551         obj_priv->last_rendering_seqno = 0;
1552         obj_priv->ring = NULL;
1553         if (obj_priv->active) {
1554                 obj_priv->active = 0;
1555                 drm_gem_object_unreference(obj);
1556         }
1557         i915_verify_inactive(dev, __FILE__, __LINE__);
1558 }
1559
1560 static void
1561 i915_gem_process_flushing_list(struct drm_device *dev,
1562                                uint32_t flush_domains, uint32_t seqno,
1563                                struct intel_ring_buffer *ring)
1564 {
1565         drm_i915_private_t *dev_priv = dev->dev_private;
1566         struct drm_i915_gem_object *obj_priv, *next;
1567
1568         list_for_each_entry_safe(obj_priv, next,
1569                                  &dev_priv->mm.gpu_write_list,
1570                                  gpu_write_list) {
1571                 struct drm_gem_object *obj = &obj_priv->base;
1572
1573                 if ((obj->write_domain & flush_domains) ==
1574                     obj->write_domain &&
1575                     obj_priv->ring->ring_flag == ring->ring_flag) {
1576                         uint32_t old_write_domain = obj->write_domain;
1577
1578                         obj->write_domain = 0;
1579                         list_del_init(&obj_priv->gpu_write_list);
1580                         i915_gem_object_move_to_active(obj, seqno, ring);
1581
1582                         /* update the fence lru list */
1583                         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1584                                 struct drm_i915_fence_reg *reg =
1585                                         &dev_priv->fence_regs[obj_priv->fence_reg];
1586                                 list_move_tail(&reg->lru_list,
1587                                                 &dev_priv->mm.fence_list);
1588                         }
1589
1590                         trace_i915_gem_object_change_domain(obj,
1591                                                             obj->read_domains,
1592                                                             old_write_domain);
1593                 }
1594         }
1595 }
1596
1597 uint32_t
1598 i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1599                  uint32_t flush_domains, struct intel_ring_buffer *ring)
1600 {
1601         drm_i915_private_t *dev_priv = dev->dev_private;
1602         struct drm_i915_file_private *i915_file_priv = NULL;
1603         struct drm_i915_gem_request *request;
1604         uint32_t seqno;
1605         int was_empty;
1606
1607         if (file_priv != NULL)
1608                 i915_file_priv = file_priv->driver_priv;
1609
1610         request = kzalloc(sizeof(*request), GFP_KERNEL);
1611         if (request == NULL)
1612                 return 0;
1613
1614         seqno = ring->add_request(dev, ring, file_priv, flush_domains);
1615
1616         request->seqno = seqno;
1617         request->ring = ring;
1618         request->emitted_jiffies = jiffies;
1619         was_empty = list_empty(&ring->request_list);
1620         list_add_tail(&request->list, &ring->request_list);
1621
1622         if (i915_file_priv) {
1623                 list_add_tail(&request->client_list,
1624                               &i915_file_priv->mm.request_list);
1625         } else {
1626                 INIT_LIST_HEAD(&request->client_list);
1627         }
1628
1629         /* Associate any objects on the flushing list matching the write
1630          * domain we're flushing with our flush.
1631          */
1632         if (flush_domains != 0)
1633                 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
1634
1635         if (!dev_priv->mm.suspended) {
1636                 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1637                 if (was_empty)
1638                         queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1639         }
1640         return seqno;
1641 }
1642
1643 /**
1644  * Command execution barrier
1645  *
1646  * Ensures that all commands in the ring are finished
1647  * before signalling the CPU
1648  */
1649 static uint32_t
1650 i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
1651 {
1652         uint32_t flush_domains = 0;
1653
1654         /* The sampler always gets flushed on i965 (sigh) */
1655         if (IS_I965G(dev))
1656                 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1657
1658         ring->flush(dev, ring,
1659                         I915_GEM_DOMAIN_COMMAND, flush_domains);
1660         return flush_domains;
1661 }
1662
1663 /**
1664  * Moves buffers associated only with the given active seqno from the active
1665  * to inactive list, potentially freeing them.
1666  */
1667 static void
1668 i915_gem_retire_request(struct drm_device *dev,
1669                         struct drm_i915_gem_request *request)
1670 {
1671         drm_i915_private_t *dev_priv = dev->dev_private;
1672
1673         trace_i915_gem_request_retire(dev, request->seqno);
1674
1675         /* Move any buffers on the active list that are no longer referenced
1676          * by the ringbuffer to the flushing/inactive lists as appropriate.
1677          */
1678         spin_lock(&dev_priv->mm.active_list_lock);
1679         while (!list_empty(&request->ring->active_list)) {
1680                 struct drm_gem_object *obj;
1681                 struct drm_i915_gem_object *obj_priv;
1682
1683                 obj_priv = list_first_entry(&request->ring->active_list,
1684                                             struct drm_i915_gem_object,
1685                                             list);
1686                 obj = &obj_priv->base;
1687
1688                 /* If the seqno being retired doesn't match the oldest in the
1689                  * list, then the oldest in the list must still be newer than
1690                  * this seqno.
1691                  */
1692                 if (obj_priv->last_rendering_seqno != request->seqno)
1693                         goto out;
1694
1695 #if WATCH_LRU
1696                 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1697                          __func__, request->seqno, obj);
1698 #endif
1699
1700                 if (obj->write_domain != 0)
1701                         i915_gem_object_move_to_flushing(obj);
1702                 else {
1703                         /* Take a reference on the object so it won't be
1704                          * freed while the spinlock is held.  The list
1705                          * protection for this spinlock is safe when breaking
1706                          * the lock like this since the next thing we do
1707                          * is just get the head of the list again.
1708                          */
1709                         drm_gem_object_reference(obj);
1710                         i915_gem_object_move_to_inactive(obj);
1711                         spin_unlock(&dev_priv->mm.active_list_lock);
1712                         drm_gem_object_unreference(obj);
1713                         spin_lock(&dev_priv->mm.active_list_lock);
1714                 }
1715         }
1716 out:
1717         spin_unlock(&dev_priv->mm.active_list_lock);
1718 }
1719
1720 /**
1721  * Returns true if seq1 is later than seq2.
1722  */
1723 bool
1724 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1725 {
1726         return (int32_t)(seq1 - seq2) >= 0;
1727 }
1728
1729 uint32_t
1730 i915_get_gem_seqno(struct drm_device *dev,
1731                    struct intel_ring_buffer *ring)
1732 {
1733         return ring->get_gem_seqno(dev, ring);
1734 }
1735
1736 /**
1737  * This function clears the request list as sequence numbers are passed.
1738  */
1739 static void
1740 i915_gem_retire_requests_ring(struct drm_device *dev,
1741                               struct intel_ring_buffer *ring)
1742 {
1743         drm_i915_private_t *dev_priv = dev->dev_private;
1744         uint32_t seqno;
1745
1746         if (!ring->status_page.page_addr
1747                         || list_empty(&ring->request_list))
1748                 return;
1749
1750         seqno = i915_get_gem_seqno(dev, ring);
1751
1752         while (!list_empty(&ring->request_list)) {
1753                 struct drm_i915_gem_request *request;
1754                 uint32_t retiring_seqno;
1755
1756                 request = list_first_entry(&ring->request_list,
1757                                            struct drm_i915_gem_request,
1758                                            list);
1759                 retiring_seqno = request->seqno;
1760
1761                 if (i915_seqno_passed(seqno, retiring_seqno) ||
1762                     atomic_read(&dev_priv->mm.wedged)) {
1763                         i915_gem_retire_request(dev, request);
1764
1765                         list_del(&request->list);
1766                         list_del(&request->client_list);
1767                         kfree(request);
1768                 } else
1769                         break;
1770         }
1771
1772         if (unlikely (dev_priv->trace_irq_seqno &&
1773                       i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1774
1775                 ring->user_irq_put(dev, ring);
1776                 dev_priv->trace_irq_seqno = 0;
1777         }
1778 }
1779
1780 void
1781 i915_gem_retire_requests(struct drm_device *dev)
1782 {
1783         drm_i915_private_t *dev_priv = dev->dev_private;
1784
1785         if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1786             struct drm_i915_gem_object *obj_priv, *tmp;
1787
1788             /* We must be careful that during unbind() we do not
1789              * accidentally infinitely recurse into retire requests.
1790              * Currently:
1791              *   retire -> free -> unbind -> wait -> retire_ring
1792              */
1793             list_for_each_entry_safe(obj_priv, tmp,
1794                                      &dev_priv->mm.deferred_free_list,
1795                                      list)
1796                     i915_gem_free_object_tail(&obj_priv->base);
1797         }
1798
1799         i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1800         if (HAS_BSD(dev))
1801                 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1802 }
1803
1804 void
1805 i915_gem_retire_work_handler(struct work_struct *work)
1806 {
1807         drm_i915_private_t *dev_priv;
1808         struct drm_device *dev;
1809
1810         dev_priv = container_of(work, drm_i915_private_t,
1811                                 mm.retire_work.work);
1812         dev = dev_priv->dev;
1813
1814         mutex_lock(&dev->struct_mutex);
1815         i915_gem_retire_requests(dev);
1816
1817         if (!dev_priv->mm.suspended &&
1818                 (!list_empty(&dev_priv->render_ring.request_list) ||
1819                         (HAS_BSD(dev) &&
1820                          !list_empty(&dev_priv->bsd_ring.request_list))))
1821                 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1822         mutex_unlock(&dev->struct_mutex);
1823 }
1824
1825 int
1826 i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1827                 int interruptible, struct intel_ring_buffer *ring)
1828 {
1829         drm_i915_private_t *dev_priv = dev->dev_private;
1830         u32 ier;
1831         int ret = 0;
1832
1833         BUG_ON(seqno == 0);
1834
1835         if (atomic_read(&dev_priv->mm.wedged))
1836                 return -EIO;
1837
1838         if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
1839                 if (HAS_PCH_SPLIT(dev))
1840                         ier = I915_READ(DEIER) | I915_READ(GTIER);
1841                 else
1842                         ier = I915_READ(IER);
1843                 if (!ier) {
1844                         DRM_ERROR("something (likely vbetool) disabled "
1845                                   "interrupts, re-enabling\n");
1846                         i915_driver_irq_preinstall(dev);
1847                         i915_driver_irq_postinstall(dev);
1848                 }
1849
1850                 trace_i915_gem_request_wait_begin(dev, seqno);
1851
1852                 ring->waiting_gem_seqno = seqno;
1853                 ring->user_irq_get(dev, ring);
1854                 if (interruptible)
1855                         ret = wait_event_interruptible(ring->irq_queue,
1856                                 i915_seqno_passed(
1857                                         ring->get_gem_seqno(dev, ring), seqno)
1858                                 || atomic_read(&dev_priv->mm.wedged));
1859                 else
1860                         wait_event(ring->irq_queue,
1861                                 i915_seqno_passed(
1862                                         ring->get_gem_seqno(dev, ring), seqno)
1863                                 || atomic_read(&dev_priv->mm.wedged));
1864
1865                 ring->user_irq_put(dev, ring);
1866                 ring->waiting_gem_seqno = 0;
1867
1868                 trace_i915_gem_request_wait_end(dev, seqno);
1869         }
1870         if (atomic_read(&dev_priv->mm.wedged))
1871                 ret = -EIO;
1872
1873         if (ret && ret != -ERESTARTSYS)
1874                 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1875                           __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
1876
1877         /* Directly dispatch request retiring.  While we have the work queue
1878          * to handle this, the waiter on a request often wants an associated
1879          * buffer to have made it to the inactive list, and we would need
1880          * a separate wait queue to handle that.
1881          */
1882         if (ret == 0)
1883                 i915_gem_retire_requests_ring(dev, ring);
1884
1885         return ret;
1886 }
1887
1888 /**
1889  * Waits for a sequence number to be signaled, and cleans up the
1890  * request and object lists appropriately for that event.
1891  */
1892 static int
1893 i915_wait_request(struct drm_device *dev, uint32_t seqno,
1894                 struct intel_ring_buffer *ring)
1895 {
1896         return i915_do_wait_request(dev, seqno, 1, ring);
1897 }
1898
1899 static void
1900 i915_gem_flush(struct drm_device *dev,
1901                uint32_t invalidate_domains,
1902                uint32_t flush_domains)
1903 {
1904         drm_i915_private_t *dev_priv = dev->dev_private;
1905         if (flush_domains & I915_GEM_DOMAIN_CPU)
1906                 drm_agp_chipset_flush(dev);
1907         dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1908                         invalidate_domains,
1909                         flush_domains);
1910
1911         if (HAS_BSD(dev))
1912                 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1913                                 invalidate_domains,
1914                                 flush_domains);
1915 }
1916
1917 /**
1918  * Ensures that all rendering to the object has completed and the object is
1919  * safe to unbind from the GTT or access from the CPU.
1920  */
1921 static int
1922 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1923 {
1924         struct drm_device *dev = obj->dev;
1925         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1926         int ret;
1927
1928         /* This function only exists to support waiting for existing rendering,
1929          * not for emitting required flushes.
1930          */
1931         BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
1932
1933         /* If there is rendering queued on the buffer being evicted, wait for
1934          * it.
1935          */
1936         if (obj_priv->active) {
1937 #if WATCH_BUF
1938                 DRM_INFO("%s: object %p wait for seqno %08x\n",
1939                           __func__, obj, obj_priv->last_rendering_seqno);
1940 #endif
1941                 ret = i915_wait_request(dev,
1942                                 obj_priv->last_rendering_seqno, obj_priv->ring);
1943                 if (ret != 0)
1944                         return ret;
1945         }
1946
1947         return 0;
1948 }
1949
1950 /**
1951  * Unbinds an object from the GTT aperture.
1952  */
1953 int
1954 i915_gem_object_unbind(struct drm_gem_object *obj)
1955 {
1956         struct drm_device *dev = obj->dev;
1957         drm_i915_private_t *dev_priv = dev->dev_private;
1958         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1959         int ret = 0;
1960
1961 #if WATCH_BUF
1962         DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1963         DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1964 #endif
1965         if (obj_priv->gtt_space == NULL)
1966                 return 0;
1967
1968         if (obj_priv->pin_count != 0) {
1969                 DRM_ERROR("Attempting to unbind pinned buffer\n");
1970                 return -EINVAL;
1971         }
1972
1973         /* blow away mappings if mapped through GTT */
1974         i915_gem_release_mmap(obj);
1975
1976         /* Move the object to the CPU domain to ensure that
1977          * any possible CPU writes while it's not in the GTT
1978          * are flushed when we go to remap it. This will
1979          * also ensure that all pending GPU writes are finished
1980          * before we unbind.
1981          */
1982         ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1983         if (ret == -ERESTARTSYS)
1984                 return ret;
1985         /* Continue on if we fail due to EIO, the GPU is hung so we
1986          * should be safe and we need to cleanup or else we might
1987          * cause memory corruption through use-after-free.
1988          */
1989
1990         /* release the fence reg _after_ flushing */
1991         if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1992                 i915_gem_clear_fence_reg(obj);
1993
1994         if (obj_priv->agp_mem != NULL) {
1995                 drm_unbind_agp(obj_priv->agp_mem);
1996                 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1997                 obj_priv->agp_mem = NULL;
1998         }
1999
2000         i915_gem_object_put_pages(obj);
2001         BUG_ON(obj_priv->pages_refcount);
2002
2003         if (obj_priv->gtt_space) {
2004                 atomic_dec(&dev->gtt_count);
2005                 atomic_sub(obj->size, &dev->gtt_memory);
2006
2007                 drm_mm_put_block(obj_priv->gtt_space);
2008                 obj_priv->gtt_space = NULL;
2009         }
2010
2011         /* Remove ourselves from the LRU list if present. */
2012         spin_lock(&dev_priv->mm.active_list_lock);
2013         if (!list_empty(&obj_priv->list))
2014                 list_del_init(&obj_priv->list);
2015         spin_unlock(&dev_priv->mm.active_list_lock);
2016
2017         if (i915_gem_object_is_purgeable(obj_priv))
2018                 i915_gem_object_truncate(obj);
2019
2020         trace_i915_gem_object_unbind(obj);
2021
2022         return ret;
2023 }
2024
2025 int
2026 i915_gpu_idle(struct drm_device *dev)
2027 {
2028         drm_i915_private_t *dev_priv = dev->dev_private;
2029         bool lists_empty;
2030         uint32_t seqno1, seqno2;
2031         int ret;
2032
2033         spin_lock(&dev_priv->mm.active_list_lock);
2034         lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2035                        list_empty(&dev_priv->render_ring.active_list) &&
2036                        (!HAS_BSD(dev) ||
2037                         list_empty(&dev_priv->bsd_ring.active_list)));
2038         spin_unlock(&dev_priv->mm.active_list_lock);
2039
2040         if (lists_empty)
2041                 return 0;
2042
2043         /* Flush everything onto the inactive list. */
2044         i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2045         seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2046                         &dev_priv->render_ring);
2047         if (seqno1 == 0)
2048                 return -ENOMEM;
2049         ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2050
2051         if (HAS_BSD(dev)) {
2052                 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2053                                 &dev_priv->bsd_ring);
2054                 if (seqno2 == 0)
2055                         return -ENOMEM;
2056
2057                 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2058                 if (ret)
2059                         return ret;
2060         }
2061
2062
2063         return ret;
2064 }
2065
2066 int
2067 i915_gem_object_get_pages(struct drm_gem_object *obj,
2068                           gfp_t gfpmask)
2069 {
2070         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2071         int page_count, i;
2072         struct address_space *mapping;
2073         struct inode *inode;
2074         struct page *page;
2075
2076         BUG_ON(obj_priv->pages_refcount
2077                         == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2078
2079         if (obj_priv->pages_refcount++ != 0)
2080                 return 0;
2081
2082         /* Get the list of pages out of our struct file.  They'll be pinned
2083          * at this point until we release them.
2084          */
2085         page_count = obj->size / PAGE_SIZE;
2086         BUG_ON(obj_priv->pages != NULL);
2087         obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
2088         if (obj_priv->pages == NULL) {
2089                 obj_priv->pages_refcount--;
2090                 return -ENOMEM;
2091         }
2092
2093         inode = obj->filp->f_path.dentry->d_inode;
2094         mapping = inode->i_mapping;
2095         for (i = 0; i < page_count; i++) {
2096                 page = read_cache_page_gfp(mapping, i,
2097                                            GFP_HIGHUSER |
2098                                            __GFP_COLD |
2099                                            __GFP_RECLAIMABLE |
2100                                            gfpmask);
2101                 if (IS_ERR(page))
2102                         goto err_pages;
2103
2104                 obj_priv->pages[i] = page;
2105         }
2106
2107         if (obj_priv->tiling_mode != I915_TILING_NONE)
2108                 i915_gem_object_do_bit_17_swizzle(obj);
2109
2110         return 0;
2111
2112 err_pages:
2113         while (i--)
2114                 page_cache_release(obj_priv->pages[i]);
2115
2116         drm_free_large(obj_priv->pages);
2117         obj_priv->pages = NULL;
2118         obj_priv->pages_refcount--;
2119         return PTR_ERR(page);
2120 }
2121
2122 static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2123 {
2124         struct drm_gem_object *obj = reg->obj;
2125         struct drm_device *dev = obj->dev;
2126         drm_i915_private_t *dev_priv = dev->dev_private;
2127         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2128         int regnum = obj_priv->fence_reg;
2129         uint64_t val;
2130
2131         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2132                     0xfffff000) << 32;
2133         val |= obj_priv->gtt_offset & 0xfffff000;
2134         val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2135                 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2136
2137         if (obj_priv->tiling_mode == I915_TILING_Y)
2138                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2139         val |= I965_FENCE_REG_VALID;
2140
2141         I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2142 }
2143
2144 static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2145 {
2146         struct drm_gem_object *obj = reg->obj;
2147         struct drm_device *dev = obj->dev;
2148         drm_i915_private_t *dev_priv = dev->dev_private;
2149         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2150         int regnum = obj_priv->fence_reg;
2151         uint64_t val;
2152
2153         val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2154                     0xfffff000) << 32;
2155         val |= obj_priv->gtt_offset & 0xfffff000;
2156         val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2157         if (obj_priv->tiling_mode == I915_TILING_Y)
2158                 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2159         val |= I965_FENCE_REG_VALID;
2160
2161         I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2162 }
2163
2164 static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2165 {
2166         struct drm_gem_object *obj = reg->obj;
2167         struct drm_device *dev = obj->dev;
2168         drm_i915_private_t *dev_priv = dev->dev_private;
2169         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2170         int regnum = obj_priv->fence_reg;
2171         int tile_width;
2172         uint32_t fence_reg, val;
2173         uint32_t pitch_val;
2174
2175         if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2176             (obj_priv->gtt_offset & (obj->size - 1))) {
2177                 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
2178                      __func__, obj_priv->gtt_offset, obj->size);
2179                 return;
2180         }
2181
2182         if (obj_priv->tiling_mode == I915_TILING_Y &&
2183             HAS_128_BYTE_Y_TILING(dev))
2184                 tile_width = 128;
2185         else
2186                 tile_width = 512;
2187
2188         /* Note: pitch better be a power of two tile widths */
2189         pitch_val = obj_priv->stride / tile_width;
2190         pitch_val = ffs(pitch_val) - 1;
2191
2192         if (obj_priv->tiling_mode == I915_TILING_Y &&
2193             HAS_128_BYTE_Y_TILING(dev))
2194                 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2195         else
2196                 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2197
2198         val = obj_priv->gtt_offset;
2199         if (obj_priv->tiling_mode == I915_TILING_Y)
2200                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2201         val |= I915_FENCE_SIZE_BITS(obj->size);
2202         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2203         val |= I830_FENCE_REG_VALID;
2204
2205         if (regnum < 8)
2206                 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2207         else
2208                 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2209         I915_WRITE(fence_reg, val);
2210 }
2211
2212 static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2213 {
2214         struct drm_gem_object *obj = reg->obj;
2215         struct drm_device *dev = obj->dev;
2216         drm_i915_private_t *dev_priv = dev->dev_private;
2217         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2218         int regnum = obj_priv->fence_reg;
2219         uint32_t val;
2220         uint32_t pitch_val;
2221         uint32_t fence_size_bits;
2222
2223         if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
2224             (obj_priv->gtt_offset & (obj->size - 1))) {
2225                 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
2226                      __func__, obj_priv->gtt_offset);
2227                 return;
2228         }
2229
2230         pitch_val = obj_priv->stride / 128;
2231         pitch_val = ffs(pitch_val) - 1;
2232         WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2233
2234         val = obj_priv->gtt_offset;
2235         if (obj_priv->tiling_mode == I915_TILING_Y)
2236                 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2237         fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2238         WARN_ON(fence_size_bits & ~0x00000f00);
2239         val |= fence_size_bits;
2240         val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2241         val |= I830_FENCE_REG_VALID;
2242
2243         I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2244 }
2245
2246 static int i915_find_fence_reg(struct drm_device *dev)
2247 {
2248         struct drm_i915_fence_reg *reg = NULL;
2249         struct drm_i915_gem_object *obj_priv = NULL;
2250         struct drm_i915_private *dev_priv = dev->dev_private;
2251         struct drm_gem_object *obj = NULL;
2252         int i, avail, ret;
2253
2254         /* First try to find a free reg */
2255         avail = 0;
2256         for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2257                 reg = &dev_priv->fence_regs[i];
2258                 if (!reg->obj)
2259                         return i;
2260
2261                 obj_priv = to_intel_bo(reg->obj);
2262                 if (!obj_priv->pin_count)
2263                     avail++;
2264         }
2265
2266         if (avail == 0)
2267                 return -ENOSPC;
2268
2269         /* None available, try to steal one or wait for a user to finish */
2270         i = I915_FENCE_REG_NONE;
2271         list_for_each_entry(reg, &dev_priv->mm.fence_list,
2272                             lru_list) {
2273                 obj = reg->obj;
2274                 obj_priv = to_intel_bo(obj);
2275
2276                 if (obj_priv->pin_count)
2277                         continue;
2278
2279                 /* found one! */
2280                 i = obj_priv->fence_reg;
2281                 break;
2282         }
2283
2284         BUG_ON(i == I915_FENCE_REG_NONE);
2285
2286         /* We only have a reference on obj from the active list. put_fence_reg
2287          * might drop that one, causing a use-after-free in it. So hold a
2288          * private reference to obj like the other callers of put_fence_reg
2289          * (set_tiling ioctl) do. */
2290         drm_gem_object_reference(obj);
2291         ret = i915_gem_object_put_fence_reg(obj);
2292         drm_gem_object_unreference(obj);
2293         if (ret != 0)
2294                 return ret;
2295
2296         return i;
2297 }
2298
2299 /**
2300  * i915_gem_object_get_fence_reg - set up a fence reg for an object
2301  * @obj: object to map through a fence reg
2302  *
2303  * When mapping objects through the GTT, userspace wants to be able to write
2304  * to them without having to worry about swizzling if the object is tiled.
2305  *
2306  * This function walks the fence regs looking for a free one for @obj,
2307  * stealing one if it can't find any.
2308  *
2309  * It then sets up the reg based on the object's properties: address, pitch
2310  * and tiling format.
2311  */
2312 int
2313 i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
2314 {
2315         struct drm_device *dev = obj->dev;
2316         struct drm_i915_private *dev_priv = dev->dev_private;
2317         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2318         struct drm_i915_fence_reg *reg = NULL;
2319         int ret;
2320
2321         /* Just update our place in the LRU if our fence is getting used. */
2322         if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2323                 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2324                 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2325                 return 0;
2326         }
2327
2328         switch (obj_priv->tiling_mode) {
2329         case I915_TILING_NONE:
2330                 WARN(1, "allocating a fence for non-tiled object?\n");
2331                 break;
2332         case I915_TILING_X:
2333                 if (!obj_priv->stride)
2334                         return -EINVAL;
2335                 WARN((obj_priv->stride & (512 - 1)),
2336                      "object 0x%08x is X tiled but has non-512B pitch\n",
2337                      obj_priv->gtt_offset);
2338                 break;
2339         case I915_TILING_Y:
2340                 if (!obj_priv->stride)
2341                         return -EINVAL;
2342                 WARN((obj_priv->stride & (128 - 1)),
2343                      "object 0x%08x is Y tiled but has non-128B pitch\n",
2344                      obj_priv->gtt_offset);
2345                 break;
2346         }
2347
2348         ret = i915_find_fence_reg(dev);
2349         if (ret < 0)
2350                 return ret;
2351
2352         obj_priv->fence_reg = ret;
2353         reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2354         list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
2355
2356         reg->obj = obj;
2357
2358         switch (INTEL_INFO(dev)->gen) {
2359         case 6:
2360                 sandybridge_write_fence_reg(reg);
2361                 break;
2362         case 5:
2363         case 4:
2364                 i965_write_fence_reg(reg);
2365                 break;
2366         case 3:
2367                 i915_write_fence_reg(reg);
2368                 break;
2369         case 2:
2370                 i830_write_fence_reg(reg);
2371                 break;
2372         }
2373
2374         trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2375                         obj_priv->tiling_mode);
2376
2377         return 0;
2378 }
2379
2380 /**
2381  * i915_gem_clear_fence_reg - clear out fence register info
2382  * @obj: object to clear
2383  *
2384  * Zeroes out the fence register itself and clears out the associated
2385  * data structures in dev_priv and obj_priv.
2386  */
2387 static void
2388 i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2389 {
2390         struct drm_device *dev = obj->dev;
2391         drm_i915_private_t *dev_priv = dev->dev_private;
2392         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2393         struct drm_i915_fence_reg *reg =
2394                 &dev_priv->fence_regs[obj_priv->fence_reg];
2395         uint32_t fence_reg;
2396
2397         switch (INTEL_INFO(dev)->gen) {
2398         case 6:
2399                 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2400                              (obj_priv->fence_reg * 8), 0);
2401                 break;
2402         case 5:
2403         case 4:
2404                 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
2405                 break;
2406         case 3:
2407                 if (obj_priv->fence_reg >= 8)
2408                         fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
2409                 else
2410         case 2:
2411                         fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2412
2413                 I915_WRITE(fence_reg, 0);
2414                 break;
2415         }
2416
2417         reg->obj = NULL;
2418         obj_priv->fence_reg = I915_FENCE_REG_NONE;
2419         list_del_init(&reg->lru_list);
2420 }
2421
2422 /**
2423  * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2424  * to the buffer to finish, and then resets the fence register.
2425  * @obj: tiled object holding a fence register.
2426  *
2427  * Zeroes out the fence register itself and clears out the associated
2428  * data structures in dev_priv and obj_priv.
2429  */
2430 int
2431 i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2432 {
2433         struct drm_device *dev = obj->dev;
2434         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2435
2436         if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2437                 return 0;
2438
2439         /* If we've changed tiling, GTT-mappings of the object
2440          * need to re-fault to ensure that the correct fence register
2441          * setup is in place.
2442          */
2443         i915_gem_release_mmap(obj);
2444
2445         /* On the i915, GPU access to tiled buffers is via a fence,
2446          * therefore we must wait for any outstanding access to complete
2447          * before clearing the fence.
2448          */
2449         if (!IS_I965G(dev)) {
2450                 int ret;
2451
2452                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2453                 if (ret != 0)
2454                         return ret;
2455
2456                 ret = i915_gem_object_wait_rendering(obj);
2457                 if (ret != 0)
2458                         return ret;
2459         }
2460
2461         i915_gem_object_flush_gtt_write_domain(obj);
2462         i915_gem_clear_fence_reg (obj);
2463
2464         return 0;
2465 }
2466
2467 /**
2468  * Finds free space in the GTT aperture and binds the object there.
2469  */
2470 static int
2471 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2472 {
2473         struct drm_device *dev = obj->dev;
2474         drm_i915_private_t *dev_priv = dev->dev_private;
2475         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2476         struct drm_mm_node *free_space;
2477         gfp_t gfpmask =  __GFP_NORETRY | __GFP_NOWARN;
2478         int ret;
2479
2480         if (obj_priv->madv != I915_MADV_WILLNEED) {
2481                 DRM_ERROR("Attempting to bind a purgeable object\n");
2482                 return -EINVAL;
2483         }
2484
2485         if (alignment == 0)
2486                 alignment = i915_gem_get_gtt_alignment(obj);
2487         if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
2488                 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2489                 return -EINVAL;
2490         }
2491
2492         /* If the object is bigger than the entire aperture, reject it early
2493          * before evicting everything in a vain attempt to find space.
2494          */
2495         if (obj->size > dev->gtt_total) {
2496                 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2497                 return -E2BIG;
2498         }
2499
2500  search_free:
2501         free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2502                                         obj->size, alignment, 0);
2503         if (free_space != NULL) {
2504                 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2505                                                        alignment);
2506                 if (obj_priv->gtt_space != NULL)
2507                         obj_priv->gtt_offset = obj_priv->gtt_space->start;
2508         }
2509         if (obj_priv->gtt_space == NULL) {
2510                 /* If the gtt is empty and we're still having trouble
2511                  * fitting our object in, we're out of memory.
2512                  */
2513 #if WATCH_LRU
2514                 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2515 #endif
2516                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2517                 if (ret)
2518                         return ret;
2519
2520                 goto search_free;
2521         }
2522
2523 #if WATCH_BUF
2524         DRM_INFO("Binding object of size %zd at 0x%08x\n",
2525                  obj->size, obj_priv->gtt_offset);
2526 #endif
2527         ret = i915_gem_object_get_pages(obj, gfpmask);
2528         if (ret) {
2529                 drm_mm_put_block(obj_priv->gtt_space);
2530                 obj_priv->gtt_space = NULL;
2531
2532                 if (ret == -ENOMEM) {
2533                         /* first try to clear up some space from the GTT */
2534                         ret = i915_gem_evict_something(dev, obj->size,
2535                                                        alignment);
2536                         if (ret) {
2537                                 /* now try to shrink everyone else */
2538                                 if (gfpmask) {
2539                                         gfpmask = 0;
2540                                         goto search_free;
2541                                 }
2542
2543                                 return ret;
2544                         }
2545
2546                         goto search_free;
2547                 }
2548
2549                 return ret;
2550         }
2551
2552         /* Create an AGP memory structure pointing at our pages, and bind it
2553          * into the GTT.
2554          */
2555         obj_priv->agp_mem = drm_agp_bind_pages(dev,
2556                                                obj_priv->pages,
2557                                                obj->size >> PAGE_SHIFT,
2558                                                obj_priv->gtt_offset,
2559                                                obj_priv->agp_type);
2560         if (obj_priv->agp_mem == NULL) {
2561                 i915_gem_object_put_pages(obj);
2562                 drm_mm_put_block(obj_priv->gtt_space);
2563                 obj_priv->gtt_space = NULL;
2564
2565                 ret = i915_gem_evict_something(dev, obj->size, alignment);
2566                 if (ret)
2567                         return ret;
2568
2569                 goto search_free;
2570         }
2571         atomic_inc(&dev->gtt_count);
2572         atomic_add(obj->size, &dev->gtt_memory);
2573
2574         /* keep track of bounds object by adding it to the inactive list */
2575         list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2576
2577         /* Assert that the object is not currently in any GPU domain. As it
2578          * wasn't in the GTT, there shouldn't be any way it could have been in
2579          * a GPU cache
2580          */
2581         BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2582         BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2583
2584         trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2585
2586         return 0;
2587 }
2588
2589 void
2590 i915_gem_clflush_object(struct drm_gem_object *obj)
2591 {
2592         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2593
2594         /* If we don't have a page list set up, then we're not pinned
2595          * to GPU, and we can ignore the cache flush because it'll happen
2596          * again at bind time.
2597          */
2598         if (obj_priv->pages == NULL)
2599                 return;
2600
2601         trace_i915_gem_object_clflush(obj);
2602
2603         drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
2604 }
2605
2606 /** Flushes any GPU write domain for the object if it's dirty. */
2607 static int
2608 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2609 {
2610         struct drm_device *dev = obj->dev;
2611         uint32_t old_write_domain;
2612         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2613
2614         if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2615                 return 0;
2616
2617         /* Queue the GPU write cache flushing we need. */
2618         old_write_domain = obj->write_domain;
2619         i915_gem_flush(dev, 0, obj->write_domain);
2620         if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2621                 return -ENOMEM;
2622
2623         trace_i915_gem_object_change_domain(obj,
2624                                             obj->read_domains,
2625                                             old_write_domain);
2626         return 0;
2627 }
2628
2629 /** Flushes the GTT write domain for the object if it's dirty. */
2630 static void
2631 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2632 {
2633         uint32_t old_write_domain;
2634
2635         if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2636                 return;
2637
2638         /* No actual flushing is required for the GTT write domain.   Writes
2639          * to it immediately go to main memory as far as we know, so there's
2640          * no chipset flush.  It also doesn't land in render cache.
2641          */
2642         old_write_domain = obj->write_domain;
2643         obj->write_domain = 0;
2644
2645         trace_i915_gem_object_change_domain(obj,
2646                                             obj->read_domains,
2647                                             old_write_domain);
2648 }
2649
2650 /** Flushes the CPU write domain for the object if it's dirty. */
2651 static void
2652 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2653 {
2654         struct drm_device *dev = obj->dev;
2655         uint32_t old_write_domain;
2656
2657         if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2658                 return;
2659
2660         i915_gem_clflush_object(obj);
2661         drm_agp_chipset_flush(dev);
2662         old_write_domain = obj->write_domain;
2663         obj->write_domain = 0;
2664
2665         trace_i915_gem_object_change_domain(obj,
2666                                             obj->read_domains,
2667                                             old_write_domain);
2668 }
2669
2670 int
2671 i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2672 {
2673         int ret = 0;
2674
2675         switch (obj->write_domain) {
2676         case I915_GEM_DOMAIN_GTT:
2677                 i915_gem_object_flush_gtt_write_domain(obj);
2678                 break;
2679         case I915_GEM_DOMAIN_CPU:
2680                 i915_gem_object_flush_cpu_write_domain(obj);
2681                 break;
2682         default:
2683                 ret = i915_gem_object_flush_gpu_write_domain(obj);
2684                 break;
2685         }
2686
2687         return ret;
2688 }
2689
2690 /**
2691  * Moves a single object to the GTT read, and possibly write domain.
2692  *
2693  * This function returns when the move is complete, including waiting on
2694  * flushes to occur.
2695  */
2696 int
2697 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2698 {
2699         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2700         uint32_t old_write_domain, old_read_domains;
2701         int ret;
2702
2703         /* Not valid to be called on unbound objects. */
2704         if (obj_priv->gtt_space == NULL)
2705                 return -EINVAL;
2706
2707         ret = i915_gem_object_flush_gpu_write_domain(obj);
2708         if (ret != 0)
2709                 return ret;
2710
2711         /* Wait on any GPU rendering and flushing to occur. */
2712         ret = i915_gem_object_wait_rendering(obj);
2713         if (ret != 0)
2714                 return ret;
2715
2716         old_write_domain = obj->write_domain;
2717         old_read_domains = obj->read_domains;
2718
2719         /* If we're writing through the GTT domain, then CPU and GPU caches
2720          * will need to be invalidated at next use.
2721          */
2722         if (write)
2723                 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2724
2725         i915_gem_object_flush_cpu_write_domain(obj);
2726
2727         /* It should now be out of any other write domains, and we can update
2728          * the domain values for our changes.
2729          */
2730         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2731         obj->read_domains |= I915_GEM_DOMAIN_GTT;
2732         if (write) {
2733                 obj->write_domain = I915_GEM_DOMAIN_GTT;
2734                 obj_priv->dirty = 1;
2735         }
2736
2737         trace_i915_gem_object_change_domain(obj,
2738                                             old_read_domains,
2739                                             old_write_domain);
2740
2741         return 0;
2742 }
2743
2744 /*
2745  * Prepare buffer for display plane. Use uninterruptible for possible flush
2746  * wait, as in modesetting process we're not supposed to be interrupted.
2747  */
2748 int
2749 i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2750 {
2751         struct drm_device *dev = obj->dev;
2752         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2753         uint32_t old_write_domain, old_read_domains;
2754         int ret;
2755
2756         /* Not valid to be called on unbound objects. */
2757         if (obj_priv->gtt_space == NULL)
2758                 return -EINVAL;
2759
2760         ret = i915_gem_object_flush_gpu_write_domain(obj);
2761         if (ret)
2762                 return ret;
2763
2764         /* Wait on any GPU rendering and flushing to occur. */
2765         if (obj_priv->active) {
2766 #if WATCH_BUF
2767                 DRM_INFO("%s: object %p wait for seqno %08x\n",
2768                           __func__, obj, obj_priv->last_rendering_seqno);
2769 #endif
2770                 ret = i915_do_wait_request(dev,
2771                                 obj_priv->last_rendering_seqno,
2772                                 0,
2773                                 obj_priv->ring);
2774                 if (ret != 0)
2775                         return ret;
2776         }
2777
2778         i915_gem_object_flush_cpu_write_domain(obj);
2779
2780         old_write_domain = obj->write_domain;
2781         old_read_domains = obj->read_domains;
2782
2783         /* It should now be out of any other write domains, and we can update
2784          * the domain values for our changes.
2785          */
2786         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2787         obj->read_domains = I915_GEM_DOMAIN_GTT;
2788         obj->write_domain = I915_GEM_DOMAIN_GTT;
2789         obj_priv->dirty = 1;
2790
2791         trace_i915_gem_object_change_domain(obj,
2792                                             old_read_domains,
2793                                             old_write_domain);
2794
2795         return 0;
2796 }
2797
2798 /**
2799  * Moves a single object to the CPU read, and possibly write domain.
2800  *
2801  * This function returns when the move is complete, including waiting on
2802  * flushes to occur.
2803  */
2804 static int
2805 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2806 {
2807         uint32_t old_write_domain, old_read_domains;
2808         int ret;
2809
2810         ret = i915_gem_object_flush_gpu_write_domain(obj);
2811         if (ret)
2812                 return ret;
2813
2814         /* Wait on any GPU rendering and flushing to occur. */
2815         ret = i915_gem_object_wait_rendering(obj);
2816         if (ret != 0)
2817                 return ret;
2818
2819         i915_gem_object_flush_gtt_write_domain(obj);
2820
2821         /* If we have a partially-valid cache of the object in the CPU,
2822          * finish invalidating it and free the per-page flags.
2823          */
2824         i915_gem_object_set_to_full_cpu_read_domain(obj);
2825
2826         old_write_domain = obj->write_domain;
2827         old_read_domains = obj->read_domains;
2828
2829         /* Flush the CPU cache if it's still invalid. */
2830         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2831                 i915_gem_clflush_object(obj);
2832
2833                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2834         }
2835
2836         /* It should now be out of any other write domains, and we can update
2837          * the domain values for our changes.
2838          */
2839         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2840
2841         /* If we're writing through the CPU, then the GPU read domains will
2842          * need to be invalidated at next use.
2843          */
2844         if (write) {
2845                 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2846                 obj->write_domain = I915_GEM_DOMAIN_CPU;
2847         }
2848
2849         trace_i915_gem_object_change_domain(obj,
2850                                             old_read_domains,
2851                                             old_write_domain);
2852
2853         return 0;
2854 }
2855
2856 /*
2857  * Set the next domain for the specified object. This
2858  * may not actually perform the necessary flushing/invaliding though,
2859  * as that may want to be batched with other set_domain operations
2860  *
2861  * This is (we hope) the only really tricky part of gem. The goal
2862  * is fairly simple -- track which caches hold bits of the object
2863  * and make sure they remain coherent. A few concrete examples may
2864  * help to explain how it works. For shorthand, we use the notation
2865  * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2866  * a pair of read and write domain masks.
2867  *
2868  * Case 1: the batch buffer
2869  *
2870  *      1. Allocated
2871  *      2. Written by CPU
2872  *      3. Mapped to GTT
2873  *      4. Read by GPU
2874  *      5. Unmapped from GTT
2875  *      6. Freed
2876  *
2877  *      Let's take these a step at a time
2878  *
2879  *      1. Allocated
2880  *              Pages allocated from the kernel may still have
2881  *              cache contents, so we set them to (CPU, CPU) always.
2882  *      2. Written by CPU (using pwrite)
2883  *              The pwrite function calls set_domain (CPU, CPU) and
2884  *              this function does nothing (as nothing changes)
2885  *      3. Mapped by GTT
2886  *              This function asserts that the object is not
2887  *              currently in any GPU-based read or write domains
2888  *      4. Read by GPU
2889  *              i915_gem_execbuffer calls set_domain (COMMAND, 0).
2890  *              As write_domain is zero, this function adds in the
2891  *              current read domains (CPU+COMMAND, 0).
2892  *              flush_domains is set to CPU.
2893  *              invalidate_domains is set to COMMAND
2894  *              clflush is run to get data out of the CPU caches
2895  *              then i915_dev_set_domain calls i915_gem_flush to
2896  *              emit an MI_FLUSH and drm_agp_chipset_flush
2897  *      5. Unmapped from GTT
2898  *              i915_gem_object_unbind calls set_domain (CPU, CPU)
2899  *              flush_domains and invalidate_domains end up both zero
2900  *              so no flushing/invalidating happens
2901  *      6. Freed
2902  *              yay, done
2903  *
2904  * Case 2: The shared render buffer
2905  *
2906  *      1. Allocated
2907  *      2. Mapped to GTT
2908  *      3. Read/written by GPU
2909  *      4. set_domain to (CPU,CPU)
2910  *      5. Read/written by CPU
2911  *      6. Read/written by GPU
2912  *
2913  *      1. Allocated
2914  *              Same as last example, (CPU, CPU)
2915  *      2. Mapped to GTT
2916  *              Nothing changes (assertions find that it is not in the GPU)
2917  *      3. Read/written by GPU
2918  *              execbuffer calls set_domain (RENDER, RENDER)
2919  *              flush_domains gets CPU
2920  *              invalidate_domains gets GPU
2921  *              clflush (obj)
2922  *              MI_FLUSH and drm_agp_chipset_flush
2923  *      4. set_domain (CPU, CPU)
2924  *              flush_domains gets GPU
2925  *              invalidate_domains gets CPU
2926  *              wait_rendering (obj) to make sure all drawing is complete.
2927  *              This will include an MI_FLUSH to get the data from GPU
2928  *              to memory
2929  *              clflush (obj) to invalidate the CPU cache
2930  *              Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2931  *      5. Read/written by CPU
2932  *              cache lines are loaded and dirtied
2933  *      6. Read written by GPU
2934  *              Same as last GPU access
2935  *
2936  * Case 3: The constant buffer
2937  *
2938  *      1. Allocated
2939  *      2. Written by CPU
2940  *      3. Read by GPU
2941  *      4. Updated (written) by CPU again
2942  *      5. Read by GPU
2943  *
2944  *      1. Allocated
2945  *              (CPU, CPU)
2946  *      2. Written by CPU
2947  *              (CPU, CPU)
2948  *      3. Read by GPU
2949  *              (CPU+RENDER, 0)
2950  *              flush_domains = CPU
2951  *              invalidate_domains = RENDER
2952  *              clflush (obj)
2953  *              MI_FLUSH
2954  *              drm_agp_chipset_flush
2955  *      4. Updated (written) by CPU again
2956  *              (CPU, CPU)
2957  *              flush_domains = 0 (no previous write domain)
2958  *              invalidate_domains = 0 (no new read domains)
2959  *      5. Read by GPU
2960  *              (CPU+RENDER, 0)
2961  *              flush_domains = CPU
2962  *              invalidate_domains = RENDER
2963  *              clflush (obj)
2964  *              MI_FLUSH
2965  *              drm_agp_chipset_flush
2966  */
2967 static void
2968 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
2969 {
2970         struct drm_device               *dev = obj->dev;
2971         drm_i915_private_t              *dev_priv = dev->dev_private;
2972         struct drm_i915_gem_object      *obj_priv = to_intel_bo(obj);
2973         uint32_t                        invalidate_domains = 0;
2974         uint32_t                        flush_domains = 0;
2975         uint32_t                        old_read_domains;
2976
2977         BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2978         BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
2979
2980         intel_mark_busy(dev, obj);
2981
2982 #if WATCH_BUF
2983         DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2984                  __func__, obj,
2985                  obj->read_domains, obj->pending_read_domains,
2986                  obj->write_domain, obj->pending_write_domain);
2987 #endif
2988         /*
2989          * If the object isn't moving to a new write domain,
2990          * let the object stay in multiple read domains
2991          */
2992         if (obj->pending_write_domain == 0)
2993                 obj->pending_read_domains |= obj->read_domains;
2994         else
2995                 obj_priv->dirty = 1;
2996
2997         /*
2998          * Flush the current write domain if
2999          * the new read domains don't match. Invalidate
3000          * any read domains which differ from the old
3001          * write domain
3002          */
3003         if (obj->write_domain &&
3004             obj->write_domain != obj->pending_read_domains) {
3005                 flush_domains |= obj->write_domain;
3006                 invalidate_domains |=
3007                         obj->pending_read_domains & ~obj->write_domain;
3008         }
3009         /*
3010          * Invalidate any read caches which may have
3011          * stale data. That is, any new read domains.
3012          */
3013         invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3014         if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3015 #if WATCH_BUF
3016                 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3017                          __func__, flush_domains, invalidate_domains);
3018 #endif
3019                 i915_gem_clflush_object(obj);
3020         }
3021
3022         old_read_domains = obj->read_domains;
3023
3024         /* The actual obj->write_domain will be updated with
3025          * pending_write_domain after we emit the accumulated flush for all
3026          * of our domain changes in execbuffers (which clears objects'
3027          * write_domains).  So if we have a current write domain that we
3028          * aren't changing, set pending_write_domain to that.
3029          */
3030         if (flush_domains == 0 && obj->pending_write_domain == 0)
3031                 obj->pending_write_domain = obj->write_domain;
3032         obj->read_domains = obj->pending_read_domains;
3033
3034         if (flush_domains & I915_GEM_GPU_DOMAINS) {
3035                 if (obj_priv->ring == &dev_priv->render_ring)
3036                         dev_priv->flush_rings |= FLUSH_RENDER_RING;
3037                 else if (obj_priv->ring == &dev_priv->bsd_ring)
3038                         dev_priv->flush_rings |= FLUSH_BSD_RING;
3039         }
3040
3041         dev->invalidate_domains |= invalidate_domains;
3042         dev->flush_domains |= flush_domains;
3043 #if WATCH_BUF
3044         DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3045                  __func__,
3046                  obj->read_domains, obj->write_domain,
3047                  dev->invalidate_domains, dev->flush_domains);
3048 #endif
3049
3050         trace_i915_gem_object_change_domain(obj,
3051                                             old_read_domains,
3052                                             obj->write_domain);
3053 }
3054
3055 /**
3056  * Moves the object from a partially CPU read to a full one.
3057  *
3058  * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3059  * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3060  */
3061 static void
3062 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3063 {
3064         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3065
3066         if (!obj_priv->page_cpu_valid)
3067                 return;
3068
3069         /* If we're partially in the CPU read domain, finish moving it in.
3070          */
3071         if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3072                 int i;
3073
3074                 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3075                         if (obj_priv->page_cpu_valid[i])
3076                                 continue;
3077                         drm_clflush_pages(obj_priv->pages + i, 1);
3078                 }
3079         }
3080
3081         /* Free the page_cpu_valid mappings which are now stale, whether
3082          * or not we've got I915_GEM_DOMAIN_CPU.
3083          */
3084         kfree(obj_priv->page_cpu_valid);
3085         obj_priv->page_cpu_valid = NULL;
3086 }
3087
3088 /**
3089  * Set the CPU read domain on a range of the object.
3090  *
3091  * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3092  * not entirely valid.  The page_cpu_valid member of the object flags which
3093  * pages have been flushed, and will be respected by
3094  * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3095  * of the whole object.
3096  *
3097  * This function returns when the move is complete, including waiting on
3098  * flushes to occur.
3099  */
3100 static int
3101 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3102                                           uint64_t offset, uint64_t size)
3103 {
3104         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3105         uint32_t old_read_domains;
3106         int i, ret;
3107
3108         if (offset == 0 && size == obj->size)
3109                 return i915_gem_object_set_to_cpu_domain(obj, 0);
3110
3111         ret = i915_gem_object_flush_gpu_write_domain(obj);
3112         if (ret)
3113                 return ret;
3114
3115         /* Wait on any GPU rendering and flushing to occur. */
3116         ret = i915_gem_object_wait_rendering(obj);
3117         if (ret != 0)
3118                 return ret;
3119         i915_gem_object_flush_gtt_write_domain(obj);
3120
3121         /* If we're already fully in the CPU read domain, we're done. */
3122         if (obj_priv->page_cpu_valid == NULL &&
3123             (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3124                 return 0;
3125
3126         /* Otherwise, create/clear the per-page CPU read domain flag if we're
3127          * newly adding I915_GEM_DOMAIN_CPU
3128          */
3129         if (obj_priv->page_cpu_valid == NULL) {
3130                 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3131                                                    GFP_KERNEL);
3132                 if (obj_priv->page_cpu_valid == NULL)
3133                         return -ENOMEM;
3134         } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3135                 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
3136
3137         /* Flush the cache on any pages that are still invalid from the CPU's
3138          * perspective.
3139          */
3140         for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3141              i++) {
3142                 if (obj_priv->page_cpu_valid[i])
3143                         continue;
3144
3145                 drm_clflush_pages(obj_priv->pages + i, 1);
3146
3147                 obj_priv->page_cpu_valid[i] = 1;
3148         }
3149
3150         /* It should now be out of any other write domains, and we can update
3151          * the domain values for our changes.
3152          */
3153         BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3154
3155         old_read_domains = obj->read_domains;
3156         obj->read_domains |= I915_GEM_DOMAIN_CPU;
3157
3158         trace_i915_gem_object_change_domain(obj,
3159                                             old_read_domains,
3160                                             obj->write_domain);
3161
3162         return 0;
3163 }
3164
3165 /**
3166  * Pin an object to the GTT and evaluate the relocations landing in it.
3167  */
3168 static int
3169 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3170                                  struct drm_file *file_priv,
3171                                  struct drm_i915_gem_exec_object2 *entry,
3172                                  struct drm_i915_gem_relocation_entry *relocs)
3173 {
3174         struct drm_device *dev = obj->dev;
3175         drm_i915_private_t *dev_priv = dev->dev_private;
3176         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3177         int i, ret;
3178         void __iomem *reloc_page;
3179         bool need_fence;
3180
3181         need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3182                      obj_priv->tiling_mode != I915_TILING_NONE;
3183
3184         /* Check fence reg constraints and rebind if necessary */
3185         if (need_fence &&
3186             !i915_gem_object_fence_offset_ok(obj,
3187                                              obj_priv->tiling_mode)) {
3188                 ret = i915_gem_object_unbind(obj);
3189                 if (ret)
3190                         return ret;
3191         }
3192
3193         /* Choose the GTT offset for our buffer and put it there. */
3194         ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3195         if (ret)
3196                 return ret;
3197
3198         /*
3199          * Pre-965 chips need a fence register set up in order to
3200          * properly handle blits to/from tiled surfaces.
3201          */
3202         if (need_fence) {
3203                 ret = i915_gem_object_get_fence_reg(obj);
3204                 if (ret != 0) {
3205                         i915_gem_object_unpin(obj);
3206                         return ret;
3207                 }
3208         }
3209
3210         entry->offset = obj_priv->gtt_offset;
3211
3212         /* Apply the relocations, using the GTT aperture to avoid cache
3213          * flushing requirements.
3214          */
3215         for (i = 0; i < entry->relocation_count; i++) {
3216                 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
3217                 struct drm_gem_object *target_obj;
3218                 struct drm_i915_gem_object *target_obj_priv;
3219                 uint32_t reloc_val, reloc_offset;
3220                 uint32_t __iomem *reloc_entry;
3221
3222                 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
3223                                                    reloc->target_handle);
3224                 if (target_obj == NULL) {
3225                         i915_gem_object_unpin(obj);
3226                         return -ENOENT;
3227                 }
3228                 target_obj_priv = to_intel_bo(target_obj);
3229
3230 #if WATCH_RELOC
3231                 DRM_INFO("%s: obj %p offset %08x target %d "
3232                          "read %08x write %08x gtt %08x "
3233                          "presumed %08x delta %08x\n",
3234                          __func__,
3235                          obj,
3236                          (int) reloc->offset,
3237                          (int) reloc->target_handle,
3238                          (int) reloc->read_domains,
3239                          (int) reloc->write_domain,
3240                          (int) target_obj_priv->gtt_offset,
3241                          (int) reloc->presumed_offset,
3242                          reloc->delta);
3243 #endif
3244
3245                 /* The target buffer should have appeared before us in the
3246                  * exec_object list, so it should have a GTT space bound by now.
3247                  */
3248                 if (target_obj_priv->gtt_space == NULL) {
3249                         DRM_ERROR("No GTT space found for object %d\n",
3250                                   reloc->target_handle);
3251                         drm_gem_object_unreference(target_obj);
3252                         i915_gem_object_unpin(obj);
3253                         return -EINVAL;
3254                 }
3255
3256                 /* Validate that the target is in a valid r/w GPU domain */
3257                 if (reloc->write_domain & (reloc->write_domain - 1)) {
3258                         DRM_ERROR("reloc with multiple write domains: "
3259                                   "obj %p target %d offset %d "
3260                                   "read %08x write %08x",
3261                                   obj, reloc->target_handle,
3262                                   (int) reloc->offset,
3263                                   reloc->read_domains,
3264                                   reloc->write_domain);
3265                         drm_gem_object_unreference(target_obj);
3266                         i915_gem_object_unpin(obj);
3267                         return -EINVAL;
3268                 }
3269                 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3270                     reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3271                         DRM_ERROR("reloc with read/write CPU domains: "
3272                                   "obj %p target %d offset %d "
3273                                   "read %08x write %08x",
3274                                   obj, reloc->target_handle,
3275                                   (int) reloc->offset,
3276                                   reloc->read_domains,
3277                                   reloc->write_domain);
3278                         drm_gem_object_unreference(target_obj);
3279                         i915_gem_object_unpin(obj);
3280                         return -EINVAL;
3281                 }
3282                 if (reloc->write_domain && target_obj->pending_write_domain &&
3283                     reloc->write_domain != target_obj->pending_write_domain) {
3284                         DRM_ERROR("Write domain conflict: "
3285                                   "obj %p target %d offset %d "
3286                                   "new %08x old %08x\n",
3287                                   obj, reloc->target_handle,
3288                                   (int) reloc->offset,
3289                                   reloc->write_domain,
3290                                   target_obj->pending_write_domain);
3291                         drm_gem_object_unreference(target_obj);
3292                         i915_gem_object_unpin(obj);
3293                         return -EINVAL;
3294                 }
3295
3296                 target_obj->pending_read_domains |= reloc->read_domains;
3297                 target_obj->pending_write_domain |= reloc->write_domain;
3298
3299                 /* If the relocation already has the right value in it, no
3300                  * more work needs to be done.
3301                  */
3302                 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3303                         drm_gem_object_unreference(target_obj);
3304                         continue;
3305                 }
3306
3307                 /* Check that the relocation address is valid... */
3308                 if (reloc->offset > obj->size - 4) {
3309                         DRM_ERROR("Relocation beyond object bounds: "
3310                                   "obj %p target %d offset %d size %d.\n",
3311                                   obj, reloc->target_handle,
3312                                   (int) reloc->offset, (int) obj->size);
3313                         drm_gem_object_unreference(target_obj);
3314                         i915_gem_object_unpin(obj);
3315                         return -EINVAL;
3316                 }
3317                 if (reloc->offset & 3) {
3318                         DRM_ERROR("Relocation not 4-byte aligned: "
3319                                   "obj %p target %d offset %d.\n",
3320                                   obj, reloc->target_handle,
3321                                   (int) reloc->offset);
3322                         drm_gem_object_unreference(target_obj);
3323                         i915_gem_object_unpin(obj);
3324                         return -EINVAL;
3325                 }
3326
3327                 /* and points to somewhere within the target object. */
3328                 if (reloc->delta >= target_obj->size) {
3329                         DRM_ERROR("Relocation beyond target object bounds: "
3330                                   "obj %p target %d delta %d size %d.\n",
3331                                   obj, reloc->target_handle,
3332                                   (int) reloc->delta, (int) target_obj->size);
3333                         drm_gem_object_unreference(target_obj);
3334                         i915_gem_object_unpin(obj);
3335                         return -EINVAL;
3336                 }
3337
3338                 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3339                 if (ret != 0) {
3340                         drm_gem_object_unreference(target_obj);
3341                         i915_gem_object_unpin(obj);
3342                         return -EINVAL;
3343                 }
3344
3345                 /* Map the page containing the relocation we're going to
3346                  * perform.
3347                  */
3348                 reloc_offset = obj_priv->gtt_offset + reloc->offset;
3349                 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3350                                                       (reloc_offset &
3351                                                        ~(PAGE_SIZE - 1)),
3352                                                       KM_USER0);
3353                 reloc_entry = (uint32_t __iomem *)(reloc_page +
3354                                                    (reloc_offset & (PAGE_SIZE - 1)));
3355                 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
3356
3357 #if WATCH_BUF
3358                 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
3359                           obj, (unsigned int) reloc->offset,
3360                           readl(reloc_entry), reloc_val);
3361 #endif
3362                 writel(reloc_val, reloc_entry);
3363                 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3364
3365                 /* The updated presumed offset for this entry will be
3366                  * copied back out to the user.
3367                  */
3368                 reloc->presumed_offset = target_obj_priv->gtt_offset;
3369
3370                 drm_gem_object_unreference(target_obj);
3371         }
3372
3373 #if WATCH_BUF
3374         if (0)
3375                 i915_gem_dump_object(obj, 128, __func__, ~0);
3376 #endif
3377         return 0;
3378 }
3379
3380 /* Throttle our rendering by waiting until the ring has completed our requests
3381  * emitted over 20 msec ago.
3382  *
3383  * Note that if we were to use the current jiffies each time around the loop,
3384  * we wouldn't escape the function with any frames outstanding if the time to
3385  * render a frame was over 20ms.
3386  *
3387  * This should get us reasonable parallelism between CPU and GPU but also
3388  * relatively low latency when blocking on a particular request to finish.
3389  */
3390 static int
3391 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3392 {
3393         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3394         int ret = 0;
3395         unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3396
3397         mutex_lock(&dev->struct_mutex);
3398         while (!list_empty(&i915_file_priv->mm.request_list)) {
3399                 struct drm_i915_gem_request *request;
3400
3401                 request = list_first_entry(&i915_file_priv->mm.request_list,
3402                                            struct drm_i915_gem_request,
3403                                            client_list);
3404
3405                 if (time_after_eq(request->emitted_jiffies, recent_enough))
3406                         break;
3407
3408                 ret = i915_wait_request(dev, request->seqno, request->ring);
3409                 if (ret != 0)
3410                         break;
3411         }
3412         mutex_unlock(&dev->struct_mutex);
3413
3414         return ret;
3415 }
3416
3417 static int
3418 i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
3419                               uint32_t buffer_count,
3420                               struct drm_i915_gem_relocation_entry **relocs)
3421 {
3422         uint32_t reloc_count = 0, reloc_index = 0, i;
3423         int ret;
3424
3425         *relocs = NULL;
3426         for (i = 0; i < buffer_count; i++) {
3427                 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3428                         return -EINVAL;
3429                 reloc_count += exec_list[i].relocation_count;
3430         }
3431
3432         *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
3433         if (*relocs == NULL) {
3434                 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
3435                 return -ENOMEM;
3436         }
3437
3438         for (i = 0; i < buffer_count; i++) {
3439                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3440
3441                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3442
3443                 ret = copy_from_user(&(*relocs)[reloc_index],
3444                                      user_relocs,
3445                                      exec_list[i].relocation_count *
3446                                      sizeof(**relocs));
3447                 if (ret != 0) {
3448                         drm_free_large(*relocs);
3449                         *relocs = NULL;
3450                         return -EFAULT;
3451                 }
3452
3453                 reloc_index += exec_list[i].relocation_count;
3454         }
3455
3456         return 0;
3457 }
3458
3459 static int
3460 i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
3461                             uint32_t buffer_count,
3462                             struct drm_i915_gem_relocation_entry *relocs)
3463 {
3464         uint32_t reloc_count = 0, i;
3465         int ret = 0;
3466
3467         if (relocs == NULL)
3468             return 0;
3469
3470         for (i = 0; i < buffer_count; i++) {
3471                 struct drm_i915_gem_relocation_entry __user *user_relocs;
3472                 int unwritten;
3473
3474                 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3475
3476                 unwritten = copy_to_user(user_relocs,
3477                                          &relocs[reloc_count],
3478                                          exec_list[i].relocation_count *
3479                                          sizeof(*relocs));
3480
3481                 if (unwritten) {
3482                         ret = -EFAULT;
3483                         goto err;
3484                 }
3485
3486                 reloc_count += exec_list[i].relocation_count;
3487         }
3488
3489 err:
3490         drm_free_large(relocs);
3491
3492         return ret;
3493 }
3494
3495 static int
3496 i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
3497                            uint64_t exec_offset)
3498 {
3499         uint32_t exec_start, exec_len;
3500
3501         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3502         exec_len = (uint32_t) exec->batch_len;
3503
3504         if ((exec_start | exec_len) & 0x7)
3505                 return -EINVAL;
3506
3507         if (!exec_start)
3508                 return -EINVAL;
3509
3510         return 0;
3511 }
3512
3513 static int
3514 i915_gem_wait_for_pending_flip(struct drm_device *dev,
3515                                struct drm_gem_object **object_list,
3516                                int count)
3517 {
3518         drm_i915_private_t *dev_priv = dev->dev_private;
3519         struct drm_i915_gem_object *obj_priv;
3520         DEFINE_WAIT(wait);
3521         int i, ret = 0;
3522
3523         for (;;) {
3524                 prepare_to_wait(&dev_priv->pending_flip_queue,
3525                                 &wait, TASK_INTERRUPTIBLE);
3526                 for (i = 0; i < count; i++) {
3527                         obj_priv = to_intel_bo(object_list[i]);
3528                         if (atomic_read(&obj_priv->pending_flip) > 0)
3529                                 break;
3530                 }
3531                 if (i == count)
3532                         break;
3533
3534                 if (!signal_pending(current)) {
3535                         mutex_unlock(&dev->struct_mutex);
3536                         schedule();
3537                         mutex_lock(&dev->struct_mutex);
3538                         continue;
3539                 }
3540                 ret = -ERESTARTSYS;
3541                 break;
3542         }
3543         finish_wait(&dev_priv->pending_flip_queue, &wait);
3544
3545         return ret;
3546 }
3547
3548
3549 int
3550 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3551                        struct drm_file *file_priv,
3552                        struct drm_i915_gem_execbuffer2 *args,
3553                        struct drm_i915_gem_exec_object2 *exec_list)
3554 {
3555         drm_i915_private_t *dev_priv = dev->dev_private;
3556         struct drm_gem_object **object_list = NULL;
3557         struct drm_gem_object *batch_obj;
3558         struct drm_i915_gem_object *obj_priv;
3559         struct drm_clip_rect *cliprects = NULL;
3560         struct drm_i915_gem_relocation_entry *relocs = NULL;
3561         int ret = 0, ret2, i, pinned = 0;
3562         uint64_t exec_offset;
3563         uint32_t seqno, flush_domains, reloc_index;
3564         int pin_tries, flips;
3565
3566         struct intel_ring_buffer *ring = NULL;
3567
3568 #if WATCH_EXEC
3569         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3570                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3571 #endif
3572         if (args->flags & I915_EXEC_BSD) {
3573                 if (!HAS_BSD(dev)) {
3574                         DRM_ERROR("execbuf with wrong flag\n");
3575                         return -EINVAL;
3576                 }
3577                 ring = &dev_priv->bsd_ring;
3578         } else {
3579                 ring = &dev_priv->render_ring;
3580         }
3581
3582         if (args->buffer_count < 1) {
3583                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3584                 return -EINVAL;
3585         }
3586         object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
3587         if (object_list == NULL) {
3588                 DRM_ERROR("Failed to allocate object list for %d buffers\n",
3589                           args->buffer_count);
3590                 ret = -ENOMEM;
3591                 goto pre_mutex_err;
3592         }
3593
3594         if (args->num_cliprects != 0) {
3595                 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3596                                     GFP_KERNEL);
3597                 if (cliprects == NULL) {
3598                         ret = -ENOMEM;
3599                         goto pre_mutex_err;
3600                 }
3601
3602                 ret = copy_from_user(cliprects,
3603                                      (struct drm_clip_rect __user *)
3604                                      (uintptr_t) args->cliprects_ptr,
3605                                      sizeof(*cliprects) * args->num_cliprects);
3606                 if (ret != 0) {
3607                         DRM_ERROR("copy %d cliprects failed: %d\n",
3608                                   args->num_cliprects, ret);
3609                         ret = -EFAULT;
3610                         goto pre_mutex_err;
3611                 }
3612         }
3613
3614         ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3615                                             &relocs);
3616         if (ret != 0)
3617                 goto pre_mutex_err;
3618
3619         mutex_lock(&dev->struct_mutex);
3620
3621         i915_verify_inactive(dev, __FILE__, __LINE__);
3622
3623         if (atomic_read(&dev_priv->mm.wedged)) {
3624                 mutex_unlock(&dev->struct_mutex);
3625                 ret = -EIO;
3626                 goto pre_mutex_err;
3627         }
3628
3629         if (dev_priv->mm.suspended) {
3630                 mutex_unlock(&dev->struct_mutex);
3631                 ret = -EBUSY;
3632                 goto pre_mutex_err;
3633         }
3634
3635         /* Look up object handles */
3636         flips = 0;
3637         for (i = 0; i < args->buffer_count; i++) {
3638                 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3639                                                        exec_list[i].handle);
3640                 if (object_list[i] == NULL) {
3641                         DRM_ERROR("Invalid object handle %d at index %d\n",
3642                                    exec_list[i].handle, i);
3643                         /* prevent error path from reading uninitialized data */
3644                         args->buffer_count = i + 1;
3645                         ret = -ENOENT;
3646                         goto err;
3647                 }
3648
3649                 obj_priv = to_intel_bo(object_list[i]);
3650                 if (obj_priv->in_execbuffer) {
3651                         DRM_ERROR("Object %p appears more than once in object list\n",
3652                                    object_list[i]);
3653                         /* prevent error path from reading uninitialized data */
3654                         args->buffer_count = i + 1;
3655                         ret = -EINVAL;
3656                         goto err;
3657                 }
3658                 obj_priv->in_execbuffer = true;
3659                 flips += atomic_read(&obj_priv->pending_flip);
3660         }
3661
3662         if (flips > 0) {
3663                 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3664                                                      args->buffer_count);
3665                 if (ret)
3666                         goto err;
3667         }
3668
3669         /* Pin and relocate */
3670         for (pin_tries = 0; ; pin_tries++) {
3671                 ret = 0;
3672                 reloc_index = 0;
3673
3674                 for (i = 0; i < args->buffer_count; i++) {
3675                         object_list[i]->pending_read_domains = 0;
3676                         object_list[i]->pending_write_domain = 0;
3677                         ret = i915_gem_object_pin_and_relocate(object_list[i],
3678                                                                file_priv,
3679                                                                &exec_list[i],
3680                                                                &relocs[reloc_index]);
3681                         if (ret)
3682                                 break;
3683                         pinned = i + 1;
3684                         reloc_index += exec_list[i].relocation_count;
3685                 }
3686                 /* success */
3687                 if (ret == 0)
3688                         break;
3689
3690                 /* error other than GTT full, or we've already tried again */
3691                 if (ret != -ENOSPC || pin_tries >= 1) {
3692                         if (ret != -ERESTARTSYS) {
3693                                 unsigned long long total_size = 0;
3694                                 int num_fences = 0;
3695                                 for (i = 0; i < args->buffer_count; i++) {
3696                                         obj_priv = to_intel_bo(object_list[i]);
3697
3698                                         total_size += object_list[i]->size;
3699                                         num_fences +=
3700                                                 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3701                                                 obj_priv->tiling_mode != I915_TILING_NONE;
3702                                 }
3703                                 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
3704                                           pinned+1, args->buffer_count,
3705                                           total_size, num_fences,
3706                                           ret);
3707                                 DRM_ERROR("%d objects [%d pinned], "
3708                                           "%d object bytes [%d pinned], "
3709                                           "%d/%d gtt bytes\n",
3710                                           atomic_read(&dev->object_count),
3711                                           atomic_read(&dev->pin_count),
3712                                           atomic_read(&dev->object_memory),
3713                                           atomic_read(&dev->pin_memory),
3714                                           atomic_read(&dev->gtt_memory),
3715                                           dev->gtt_total);
3716                         }
3717                         goto err;
3718                 }
3719
3720                 /* unpin all of our buffers */
3721                 for (i = 0; i < pinned; i++)
3722                         i915_gem_object_unpin(object_list[i]);
3723                 pinned = 0;
3724
3725                 /* evict everyone we can from the aperture */
3726                 ret = i915_gem_evict_everything(dev);
3727                 if (ret && ret != -ENOSPC)
3728                         goto err;
3729         }
3730
3731         /* Set the pending read domains for the batch buffer to COMMAND */
3732         batch_obj = object_list[args->buffer_count-1];
3733         if (batch_obj->pending_write_domain) {
3734                 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3735                 ret = -EINVAL;
3736                 goto err;
3737         }
3738         batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
3739
3740         /* Sanity check the batch buffer, prior to moving objects */
3741         exec_offset = exec_list[args->buffer_count - 1].offset;
3742         ret = i915_gem_check_execbuffer (args, exec_offset);
3743         if (ret != 0) {
3744                 DRM_ERROR("execbuf with invalid offset/length\n");
3745                 goto err;
3746         }
3747
3748         i915_verify_inactive(dev, __FILE__, __LINE__);
3749
3750         /* Zero the global flush/invalidate flags. These
3751          * will be modified as new domains are computed
3752          * for each object
3753          */
3754         dev->invalidate_domains = 0;
3755         dev->flush_domains = 0;
3756         dev_priv->flush_rings = 0;
3757
3758         for (i = 0; i < args->buffer_count; i++) {
3759                 struct drm_gem_object *obj = object_list[i];
3760
3761                 /* Compute new gpu domains and update invalidate/flush */
3762                 i915_gem_object_set_to_gpu_domain(obj);
3763         }
3764
3765         i915_verify_inactive(dev, __FILE__, __LINE__);
3766
3767         if (dev->invalidate_domains | dev->flush_domains) {
3768 #if WATCH_EXEC
3769                 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3770                           __func__,
3771                          dev->invalidate_domains,
3772                          dev->flush_domains);
3773 #endif
3774                 i915_gem_flush(dev,
3775                                dev->invalidate_domains,
3776                                dev->flush_domains);
3777                 if (dev_priv->flush_rings & FLUSH_RENDER_RING)
3778                         (void)i915_add_request(dev, file_priv,
3779                                                dev->flush_domains,
3780                                                &dev_priv->render_ring);
3781                 if (dev_priv->flush_rings & FLUSH_BSD_RING)
3782                         (void)i915_add_request(dev, file_priv,
3783                                                dev->flush_domains,
3784                                                &dev_priv->bsd_ring);
3785         }
3786
3787         for (i = 0; i < args->buffer_count; i++) {
3788                 struct drm_gem_object *obj = object_list[i];
3789                 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3790                 uint32_t old_write_domain = obj->write_domain;
3791
3792                 obj->write_domain = obj->pending_write_domain;
3793                 if (obj->write_domain)
3794                         list_move_tail(&obj_priv->gpu_write_list,
3795                                        &dev_priv->mm.gpu_write_list);
3796                 else
3797                         list_del_init(&obj_priv->gpu_write_list);
3798
3799                 trace_i915_gem_object_change_domain(obj,
3800                                                     obj->read_domains,
3801                                                     old_write_domain);
3802         }
3803
3804         i915_verify_inactive(dev, __FILE__, __LINE__);
3805
3806 #if WATCH_COHERENCY
3807         for (i = 0; i < args->buffer_count; i++) {
3808                 i915_gem_object_check_coherency(object_list[i],
3809                                                 exec_list[i].handle);
3810         }
3811 #endif
3812
3813 #if WATCH_EXEC
3814         i915_gem_dump_object(batch_obj,
3815                               args->batch_len,
3816                               __func__,
3817                               ~0);
3818 #endif
3819
3820         /* Exec the batchbuffer */
3821         ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3822                         cliprects, exec_offset);
3823         if (ret) {
3824                 DRM_ERROR("dispatch failed %d\n", ret);
3825                 goto err;
3826         }
3827
3828         /*
3829          * Ensure that the commands in the batch buffer are
3830          * finished before the interrupt fires
3831          */
3832         flush_domains = i915_retire_commands(dev, ring);
3833
3834         i915_verify_inactive(dev, __FILE__, __LINE__);
3835
3836         /*
3837          * Get a seqno representing the execution of the current buffer,
3838          * which we can wait on.  We would like to mitigate these interrupts,
3839          * likely by only creating seqnos occasionally (so that we have
3840          * *some* interrupts representing completion of buffers that we can
3841          * wait on when trying to clear up gtt space).
3842          */
3843         seqno = i915_add_request(dev, file_priv, flush_domains, ring);
3844         BUG_ON(seqno == 0);
3845         for (i = 0; i < args->buffer_count; i++) {
3846                 struct drm_gem_object *obj = object_list[i];
3847                 obj_priv = to_intel_bo(obj);
3848
3849                 i915_gem_object_move_to_active(obj, seqno, ring);
3850 #if WATCH_LRU
3851                 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3852 #endif
3853         }
3854 #if WATCH_LRU
3855         i915_dump_lru(dev, __func__);
3856 #endif
3857
3858         i915_verify_inactive(dev, __FILE__, __LINE__);
3859
3860 err:
3861         for (i = 0; i < pinned; i++)
3862                 i915_gem_object_unpin(object_list[i]);
3863
3864         for (i = 0; i < args->buffer_count; i++) {
3865                 if (object_list[i]) {
3866                         obj_priv = to_intel_bo(object_list[i]);
3867                         obj_priv->in_execbuffer = false;
3868                 }
3869                 drm_gem_object_unreference(object_list[i]);
3870         }
3871
3872         mutex_unlock(&dev->struct_mutex);
3873
3874 pre_mutex_err:
3875         /* Copy the updated relocations out regardless of current error
3876          * state.  Failure to update the relocs would mean that the next
3877          * time userland calls execbuf, it would do so with presumed offset
3878          * state that didn't match the actual object state.
3879          */
3880         ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3881                                            relocs);
3882         if (ret2 != 0) {
3883                 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3884
3885                 if (ret == 0)
3886                         ret = ret2;
3887         }
3888
3889         drm_free_large(object_list);
3890         kfree(cliprects);
3891
3892         return ret;
3893 }
3894
3895 /*
3896  * Legacy execbuffer just creates an exec2 list from the original exec object
3897  * list array and passes it to the real function.
3898  */
3899 int
3900 i915_gem_execbuffer(struct drm_device *dev, void *data,
3901                     struct drm_file *file_priv)
3902 {
3903         struct drm_i915_gem_execbuffer *args = data;
3904         struct drm_i915_gem_execbuffer2 exec2;
3905         struct drm_i915_gem_exec_object *exec_list = NULL;
3906         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3907         int ret, i;
3908
3909 #if WATCH_EXEC
3910         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3911                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3912 #endif
3913
3914         if (args->buffer_count < 1) {
3915                 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3916                 return -EINVAL;
3917         }
3918
3919         /* Copy in the exec list from userland */
3920         exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3921         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3922         if (exec_list == NULL || exec2_list == NULL) {
3923                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3924                           args->buffer_count);
3925                 drm_free_large(exec_list);
3926                 drm_free_large(exec2_list);
3927                 return -ENOMEM;
3928         }
3929         ret = copy_from_user(exec_list,
3930                              (struct drm_i915_relocation_entry __user *)
3931                              (uintptr_t) args->buffers_ptr,
3932                              sizeof(*exec_list) * args->buffer_count);
3933         if (ret != 0) {
3934                 DRM_ERROR("copy %d exec entries failed %d\n",
3935                           args->buffer_count, ret);
3936                 drm_free_large(exec_list);
3937                 drm_free_large(exec2_list);
3938                 return -EFAULT;
3939         }
3940
3941         for (i = 0; i < args->buffer_count; i++) {
3942                 exec2_list[i].handle = exec_list[i].handle;
3943                 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3944                 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3945                 exec2_list[i].alignment = exec_list[i].alignment;
3946                 exec2_list[i].offset = exec_list[i].offset;
3947                 if (!IS_I965G(dev))
3948                         exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3949                 else
3950                         exec2_list[i].flags = 0;
3951         }
3952
3953         exec2.buffers_ptr = args->buffers_ptr;
3954         exec2.buffer_count = args->buffer_count;
3955         exec2.batch_start_offset = args->batch_start_offset;
3956         exec2.batch_len = args->batch_len;
3957         exec2.DR1 = args->DR1;
3958         exec2.DR4 = args->DR4;
3959         exec2.num_cliprects = args->num_cliprects;
3960         exec2.cliprects_ptr = args->cliprects_ptr;
3961         exec2.flags = I915_EXEC_RENDER;
3962
3963         ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3964         if (!ret) {
3965                 /* Copy the new buffer offsets back to the user's exec list. */
3966                 for (i = 0; i < args->buffer_count; i++)
3967                         exec_list[i].offset = exec2_list[i].offset;
3968                 /* ... and back out to userspace */
3969                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3970                                    (uintptr_t) args->buffers_ptr,
3971                                    exec_list,
3972                                    sizeof(*exec_list) * args->buffer_count);
3973                 if (ret) {
3974                         ret = -EFAULT;
3975                         DRM_ERROR("failed to copy %d exec entries "
3976                                   "back to user (%d)\n",
3977                                   args->buffer_count, ret);
3978                 }
3979         }
3980
3981         drm_free_large(exec_list);
3982         drm_free_large(exec2_list);
3983         return ret;
3984 }
3985
3986 int
3987 i915_gem_execbuffer2(struct drm_device *dev, void *data,
3988                      struct drm_file *file_priv)
3989 {
3990         struct drm_i915_gem_execbuffer2 *args = data;
3991         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3992         int ret;
3993
3994 #if WATCH_EXEC
3995         DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3996                   (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3997 #endif
3998
3999         if (args->buffer_count < 1) {
4000                 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4001                 return -EINVAL;
4002         }
4003
4004         exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4005         if (exec2_list == NULL) {
4006                 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4007                           args->buffer_count);
4008                 return -ENOMEM;
4009         }
4010         ret = copy_from_user(exec2_list,
4011                              (struct drm_i915_relocation_entry __user *)
4012                              (uintptr_t) args->buffers_ptr,
4013                              sizeof(*exec2_list) * args->buffer_count);
4014         if (ret != 0) {
4015                 DRM_ERROR("copy %d exec entries failed %d\n",
4016                           args->buffer_count, ret);
4017                 drm_free_large(exec2_list);
4018                 return -EFAULT;
4019         }
4020
4021         ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4022         if (!ret) {
4023                 /* Copy the new buffer offsets back to the user's exec list. */
4024                 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4025                                    (uintptr_t) args->buffers_ptr,
4026                                    exec2_list,
4027                                    sizeof(*exec2_list) * args->buffer_count);
4028                 if (ret) {
4029                         ret = -EFAULT;
4030                         DRM_ERROR("failed to copy %d exec entries "
4031                                   "back to user (%d)\n",
4032                                   args->buffer_count, ret);
4033                 }
4034         }
4035
4036         drm_free_large(exec2_list);
4037         return ret;
4038 }
4039
4040 int
4041 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4042 {
4043         struct drm_device *dev = obj->dev;
4044         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4045         int ret;
4046
4047         BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4048
4049         i915_verify_inactive(dev, __FILE__, __LINE__);
4050
4051         if (obj_priv->gtt_space != NULL) {
4052                 if (alignment == 0)
4053                         alignment = i915_gem_get_gtt_alignment(obj);
4054                 if (obj_priv->gtt_offset & (alignment - 1)) {
4055                         WARN(obj_priv->pin_count,
4056                              "bo is already pinned with incorrect alignment:"
4057                              " offset=%x, req.alignment=%x\n",
4058                              obj_priv->gtt_offset, alignment);
4059                         ret = i915_gem_object_unbind(obj);
4060                         if (ret)
4061                                 return ret;
4062                 }
4063         }
4064
4065         if (obj_priv->gtt_space == NULL) {
4066                 ret = i915_gem_object_bind_to_gtt(obj, alignment);
4067                 if (ret)
4068                         return ret;
4069         }
4070
4071         obj_priv->pin_count++;
4072
4073         /* If the object is not active and not pending a flush,
4074          * remove it from the inactive list
4075          */
4076         if (obj_priv->pin_count == 1) {
4077                 atomic_inc(&dev->pin_count);
4078                 atomic_add(obj->size, &dev->pin_memory);
4079                 if (!obj_priv->active &&
4080                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4081                         list_del_init(&obj_priv->list);
4082         }
4083         i915_verify_inactive(dev, __FILE__, __LINE__);
4084
4085         return 0;
4086 }
4087
4088 void
4089 i915_gem_object_unpin(struct drm_gem_object *obj)
4090 {
4091         struct drm_device *dev = obj->dev;
4092         drm_i915_private_t *dev_priv = dev->dev_private;
4093         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4094
4095         i915_verify_inactive(dev, __FILE__, __LINE__);
4096         obj_priv->pin_count--;
4097         BUG_ON(obj_priv->pin_count < 0);
4098         BUG_ON(obj_priv->gtt_space == NULL);
4099
4100         /* If the object is no longer pinned, and is
4101          * neither active nor being flushed, then stick it on
4102          * the inactive list
4103          */
4104         if (obj_priv->pin_count == 0) {
4105                 if (!obj_priv->active &&
4106                     (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
4107                         list_move_tail(&obj_priv->list,
4108                                        &dev_priv->mm.inactive_list);
4109                 atomic_dec(&dev->pin_count);
4110                 atomic_sub(obj->size, &dev->pin_memory);
4111         }
4112         i915_verify_inactive(dev, __FILE__, __LINE__);
4113 }
4114
4115 int
4116 i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4117                    struct drm_file *file_priv)
4118 {
4119         struct drm_i915_gem_pin *args = data;
4120         struct drm_gem_object *obj;
4121         struct drm_i915_gem_object *obj_priv;
4122         int ret;
4123
4124         mutex_lock(&dev->struct_mutex);
4125
4126         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4127         if (obj == NULL) {
4128                 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4129                           args->handle);
4130                 mutex_unlock(&dev->struct_mutex);
4131                 return -ENOENT;
4132         }
4133         obj_priv = to_intel_bo(obj);
4134
4135         if (obj_priv->madv != I915_MADV_WILLNEED) {
4136                 DRM_ERROR("Attempting to pin a purgeable buffer\n");
4137                 drm_gem_object_unreference(obj);
4138                 mutex_unlock(&dev->struct_mutex);
4139                 return -EINVAL;
4140         }
4141
4142         if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4143                 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4144                           args->handle);
4145                 drm_gem_object_unreference(obj);
4146                 mutex_unlock(&dev->struct_mutex);
4147                 return -EINVAL;
4148         }
4149
4150         obj_priv->user_pin_count++;
4151         obj_priv->pin_filp = file_priv;
4152         if (obj_priv->user_pin_count == 1) {
4153                 ret = i915_gem_object_pin(obj, args->alignment);
4154                 if (ret != 0) {
4155                         drm_gem_object_unreference(obj);
4156                         mutex_unlock(&dev->struct_mutex);
4157                         return ret;
4158                 }
4159         }
4160
4161         /* XXX - flush the CPU caches for pinned objects
4162          * as the X server doesn't manage domains yet
4163          */
4164         i915_gem_object_flush_cpu_write_domain(obj);
4165         args->offset = obj_priv->gtt_offset;
4166         drm_gem_object_unreference(obj);
4167         mutex_unlock(&dev->struct_mutex);
4168
4169         return 0;
4170 }
4171
4172 int
4173 i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4174                      struct drm_file *file_priv)
4175 {
4176         struct drm_i915_gem_pin *args = data;
4177         struct drm_gem_object *obj;
4178         struct drm_i915_gem_object *obj_priv;
4179
4180         mutex_lock(&dev->struct_mutex);
4181
4182         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4183         if (obj == NULL) {
4184                 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4185                           args->handle);
4186                 mutex_unlock(&dev->struct_mutex);
4187                 return -ENOENT;
4188         }
4189
4190         obj_priv = to_intel_bo(obj);
4191         if (obj_priv->pin_filp != file_priv) {
4192                 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4193                           args->handle);
4194                 drm_gem_object_unreference(obj);
4195                 mutex_unlock(&dev->struct_mutex);
4196                 return -EINVAL;
4197         }
4198         obj_priv->user_pin_count--;
4199         if (obj_priv->user_pin_count == 0) {
4200                 obj_priv->pin_filp = NULL;
4201                 i915_gem_object_unpin(obj);
4202         }
4203
4204         drm_gem_object_unreference(obj);
4205         mutex_unlock(&dev->struct_mutex);
4206         return 0;
4207 }
4208
4209 int
4210 i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4211                     struct drm_file *file_priv)
4212 {
4213         struct drm_i915_gem_busy *args = data;
4214         struct drm_gem_object *obj;
4215         struct drm_i915_gem_object *obj_priv;
4216
4217         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4218         if (obj == NULL) {
4219                 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4220                           args->handle);
4221                 return -ENOENT;
4222         }
4223
4224         mutex_lock(&dev->struct_mutex);
4225
4226         /* Count all active objects as busy, even if they are currently not used
4227          * by the gpu. Users of this interface expect objects to eventually
4228          * become non-busy without any further actions, therefore emit any
4229          * necessary flushes here.
4230          */
4231         obj_priv = to_intel_bo(obj);
4232         args->busy = obj_priv->active;
4233         if (args->busy) {
4234                 /* Unconditionally flush objects, even when the gpu still uses this
4235                  * object. Userspace calling this function indicates that it wants to
4236                  * use this buffer rather sooner than later, so issuing the required
4237                  * flush earlier is beneficial.
4238                  */
4239                 if (obj->write_domain) {
4240                         i915_gem_flush(dev, 0, obj->write_domain);
4241                         (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4242                 }
4243
4244                 /* Update the active list for the hardware's current position.
4245                  * Otherwise this only updates on a delayed timer or when irqs
4246                  * are actually unmasked, and our working set ends up being
4247                  * larger than required.
4248                  */
4249                 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4250
4251                 args->busy = obj_priv->active;
4252         }
4253
4254         drm_gem_object_unreference(obj);
4255         mutex_unlock(&dev->struct_mutex);
4256         return 0;
4257 }
4258
4259 int
4260 i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4261                         struct drm_file *file_priv)
4262 {
4263     return i915_gem_ring_throttle(dev, file_priv);
4264 }
4265
4266 int
4267 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4268                        struct drm_file *file_priv)
4269 {
4270         struct drm_i915_gem_madvise *args = data;
4271         struct drm_gem_object *obj;
4272         struct drm_i915_gem_object *obj_priv;
4273
4274         switch (args->madv) {
4275         case I915_MADV_DONTNEED:
4276         case I915_MADV_WILLNEED:
4277             break;
4278         default:
4279             return -EINVAL;
4280         }
4281
4282         obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4283         if (obj == NULL) {
4284                 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4285                           args->handle);
4286                 return -ENOENT;
4287         }
4288
4289         mutex_lock(&dev->struct_mutex);
4290         obj_priv = to_intel_bo(obj);
4291
4292         if (obj_priv->pin_count) {
4293                 drm_gem_object_unreference(obj);
4294                 mutex_unlock(&dev->struct_mutex);
4295
4296                 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4297                 return -EINVAL;
4298         }
4299
4300         if (obj_priv->madv != __I915_MADV_PURGED)
4301                 obj_priv->madv = args->madv;
4302
4303         /* if the object is no longer bound, discard its backing storage */
4304         if (i915_gem_object_is_purgeable(obj_priv) &&
4305             obj_priv->gtt_space == NULL)
4306                 i915_gem_object_truncate(obj);
4307
4308         args->retained = obj_priv->madv != __I915_MADV_PURGED;
4309
4310         drm_gem_object_unreference(obj);
4311         mutex_unlock(&dev->struct_mutex);
4312
4313         return 0;
4314 }
4315
4316 struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4317                                               size_t size)
4318 {
4319         struct drm_i915_gem_object *obj;
4320
4321         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4322         if (obj == NULL)
4323                 return NULL;
4324
4325         if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4326                 kfree(obj);
4327                 return NULL;
4328         }
4329
4330         obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4331         obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4332
4333         obj->agp_type = AGP_USER_MEMORY;
4334         obj->base.driver_private = NULL;
4335         obj->fence_reg = I915_FENCE_REG_NONE;
4336         INIT_LIST_HEAD(&obj->list);
4337         INIT_LIST_HEAD(&obj->gpu_write_list);
4338         obj->madv = I915_MADV_WILLNEED;
4339
4340         trace_i915_gem_object_create(&obj->base);
4341
4342         return &obj->base;
4343 }
4344
4345 int i915_gem_init_object(struct drm_gem_object *obj)
4346 {
4347         BUG();
4348
4349         return 0;
4350 }
4351
4352 static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4353 {
4354         struct drm_device *dev = obj->dev;
4355         drm_i915_private_t *dev_priv = dev->dev_private;
4356         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4357         int ret;
4358
4359         ret = i915_gem_object_unbind(obj);
4360         if (ret == -ERESTARTSYS) {
4361                 list_move(&obj_priv->list,
4362                           &dev_priv->mm.deferred_free_list);
4363                 return;
4364         }
4365
4366         if (obj_priv->mmap_offset)
4367                 i915_gem_free_mmap_offset(obj);
4368
4369         drm_gem_object_release(obj);
4370
4371         kfree(obj_priv->page_cpu_valid);
4372         kfree(obj_priv->bit_17);
4373         kfree(obj_priv);
4374 }
4375
4376 void i915_gem_free_object(struct drm_gem_object *obj)
4377 {
4378         struct drm_device *dev = obj->dev;
4379         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4380
4381         trace_i915_gem_object_destroy(obj);
4382
4383         while (obj_priv->pin_count > 0)
4384                 i915_gem_object_unpin(obj);
4385
4386         if (obj_priv->phys_obj)
4387                 i915_gem_detach_phys_object(dev, obj);
4388
4389         i915_gem_free_object_tail(obj);
4390 }
4391
4392 int
4393 i915_gem_idle(struct drm_device *dev)
4394 {
4395         drm_i915_private_t *dev_priv = dev->dev_private;
4396         int ret;
4397
4398         mutex_lock(&dev->struct_mutex);
4399
4400         if (dev_priv->mm.suspended ||
4401                         (dev_priv->render_ring.gem_object == NULL) ||
4402                         (HAS_BSD(dev) &&
4403                          dev_priv->bsd_ring.gem_object == NULL)) {
4404                 mutex_unlock(&dev->struct_mutex);
4405                 return 0;
4406         }
4407
4408         ret = i915_gpu_idle(dev);
4409         if (ret) {
4410                 mutex_unlock(&dev->struct_mutex);
4411                 return ret;
4412         }
4413
4414         /* Under UMS, be paranoid and evict. */
4415         if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4416                 ret = i915_gem_evict_inactive(dev);
4417                 if (ret) {
4418                         mutex_unlock(&dev->struct_mutex);
4419                         return ret;
4420                 }
4421         }
4422
4423         /* Hack!  Don't let anybody do execbuf while we don't control the chip.
4424          * We need to replace this with a semaphore, or something.
4425          * And not confound mm.suspended!
4426          */
4427         dev_priv->mm.suspended = 1;
4428         del_timer(&dev_priv->hangcheck_timer);
4429
4430         i915_kernel_lost_context(dev);
4431         i915_gem_cleanup_ringbuffer(dev);
4432
4433         mutex_unlock(&dev->struct_mutex);
4434
4435         /* Cancel the retire work handler, which should be idle now. */
4436         cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4437
4438         return 0;
4439 }
4440
4441 /*
4442  * 965+ support PIPE_CONTROL commands, which provide finer grained control
4443  * over cache flushing.
4444  */
4445 static int
4446 i915_gem_init_pipe_control(struct drm_device *dev)
4447 {
4448         drm_i915_private_t *dev_priv = dev->dev_private;
4449         struct drm_gem_object *obj;
4450         struct drm_i915_gem_object *obj_priv;
4451         int ret;
4452
4453         obj = i915_gem_alloc_object(dev, 4096);
4454         if (obj == NULL) {
4455                 DRM_ERROR("Failed to allocate seqno page\n");
4456                 ret = -ENOMEM;
4457                 goto err;
4458         }
4459         obj_priv = to_intel_bo(obj);
4460         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4461
4462         ret = i915_gem_object_pin(obj, 4096);
4463         if (ret)
4464                 goto err_unref;
4465
4466         dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4467         dev_priv->seqno_page =  kmap(obj_priv->pages[0]);
4468         if (dev_priv->seqno_page == NULL)
4469                 goto err_unpin;
4470
4471         dev_priv->seqno_obj = obj;
4472         memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4473
4474         return 0;
4475
4476 err_unpin:
4477         i915_gem_object_unpin(obj);
4478 err_unref:
4479         drm_gem_object_unreference(obj);
4480 err:
4481         return ret;
4482 }
4483
4484
4485 static void
4486 i915_gem_cleanup_pipe_control(struct drm_device *dev)
4487 {
4488         drm_i915_private_t *dev_priv = dev->dev_private;
4489         struct drm_gem_object *obj;
4490         struct drm_i915_gem_object *obj_priv;
4491
4492         obj = dev_priv->seqno_obj;
4493         obj_priv = to_intel_bo(obj);
4494         kunmap(obj_priv->pages[0]);
4495         i915_gem_object_unpin(obj);
4496         drm_gem_object_unreference(obj);
4497         dev_priv->seqno_obj = NULL;
4498
4499         dev_priv->seqno_page = NULL;
4500 }
4501
4502 int
4503 i915_gem_init_ringbuffer(struct drm_device *dev)
4504 {
4505         drm_i915_private_t *dev_priv = dev->dev_private;
4506         int ret;
4507
4508         dev_priv->render_ring = render_ring;
4509
4510         if (!I915_NEED_GFX_HWS(dev)) {
4511                 dev_priv->render_ring.status_page.page_addr
4512                         = dev_priv->status_page_dmah->vaddr;
4513                 memset(dev_priv->render_ring.status_page.page_addr,
4514                                 0, PAGE_SIZE);
4515         }
4516
4517         if (HAS_PIPE_CONTROL(dev)) {
4518                 ret = i915_gem_init_pipe_control(dev);
4519                 if (ret)
4520                         return ret;
4521         }
4522
4523         ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
4524         if (ret)
4525                 goto cleanup_pipe_control;
4526
4527         if (HAS_BSD(dev)) {
4528                 dev_priv->bsd_ring = bsd_ring;
4529                 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
4530                 if (ret)
4531                         goto cleanup_render_ring;
4532         }
4533
4534         dev_priv->next_seqno = 1;
4535
4536         return 0;
4537
4538 cleanup_render_ring:
4539         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4540 cleanup_pipe_control:
4541         if (HAS_PIPE_CONTROL(dev))
4542                 i915_gem_cleanup_pipe_control(dev);
4543         return ret;
4544 }
4545
4546 void
4547 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4548 {
4549         drm_i915_private_t *dev_priv = dev->dev_private;
4550
4551         intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4552         if (HAS_BSD(dev))
4553                 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
4554         if (HAS_PIPE_CONTROL(dev))
4555                 i915_gem_cleanup_pipe_control(dev);
4556 }
4557
4558 int
4559 i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4560                        struct drm_file *file_priv)
4561 {
4562         drm_i915_private_t *dev_priv = dev->dev_private;
4563         int ret;
4564
4565         if (drm_core_check_feature(dev, DRIVER_MODESET))
4566                 return 0;
4567
4568         if (atomic_read(&dev_priv->mm.wedged)) {
4569                 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4570                 atomic_set(&dev_priv->mm.wedged, 0);
4571         }
4572
4573         mutex_lock(&dev->struct_mutex);
4574         dev_priv->mm.suspended = 0;
4575
4576         ret = i915_gem_init_ringbuffer(dev);
4577         if (ret != 0) {
4578                 mutex_unlock(&dev->struct_mutex);
4579                 return ret;
4580         }
4581
4582         spin_lock(&dev_priv->mm.active_list_lock);
4583         BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
4584         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
4585         spin_unlock(&dev_priv->mm.active_list_lock);
4586
4587         BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4588         BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4589         BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
4590         BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
4591         mutex_unlock(&dev->struct_mutex);
4592
4593         ret = drm_irq_install(dev);
4594         if (ret)
4595                 goto cleanup_ringbuffer;
4596
4597         return 0;
4598
4599 cleanup_ringbuffer:
4600         mutex_lock(&dev->struct_mutex);
4601         i915_gem_cleanup_ringbuffer(dev);
4602         dev_priv->mm.suspended = 1;
4603         mutex_unlock(&dev->struct_mutex);
4604
4605         return ret;
4606 }
4607
4608 int
4609 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4610                        struct drm_file *file_priv)
4611 {
4612         if (drm_core_check_feature(dev, DRIVER_MODESET))
4613                 return 0;
4614
4615         drm_irq_uninstall(dev);
4616         return i915_gem_idle(dev);
4617 }
4618
4619 void
4620 i915_gem_lastclose(struct drm_device *dev)
4621 {
4622         int ret;
4623
4624         if (drm_core_check_feature(dev, DRIVER_MODESET))
4625                 return;
4626
4627         ret = i915_gem_idle(dev);
4628         if (ret)
4629                 DRM_ERROR("failed to idle hardware: %d\n", ret);
4630 }
4631
4632 void
4633 i915_gem_load(struct drm_device *dev)
4634 {
4635         int i;
4636         drm_i915_private_t *dev_priv = dev->dev_private;
4637
4638         spin_lock_init(&dev_priv->mm.active_list_lock);
4639         INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4640         INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
4641         INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4642         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
4643         INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
4644         INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4645         INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
4646         if (HAS_BSD(dev)) {
4647                 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4648                 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4649         }
4650         for (i = 0; i < 16; i++)
4651                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
4652         INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4653                           i915_gem_retire_work_handler);
4654         spin_lock(&shrink_list_lock);
4655         list_add(&dev_priv->mm.shrink_list, &shrink_list);
4656         spin_unlock(&shrink_list_lock);
4657
4658         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4659         if (IS_GEN3(dev)) {
4660                 u32 tmp = I915_READ(MI_ARB_STATE);
4661                 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4662                         /* arb state is a masked write, so set bit + bit in mask */
4663                         tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4664                         I915_WRITE(MI_ARB_STATE, tmp);
4665                 }
4666         }
4667
4668         /* Old X drivers will take 0-2 for front, back, depth buffers */
4669         if (!drm_core_check_feature(dev, DRIVER_MODESET))
4670                 dev_priv->fence_reg_start = 3;
4671
4672         if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4673                 dev_priv->num_fence_regs = 16;
4674         else
4675                 dev_priv->num_fence_regs = 8;
4676
4677         /* Initialize fence registers to zero */
4678         if (IS_I965G(dev)) {
4679                 for (i = 0; i < 16; i++)
4680                         I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4681         } else {
4682                 for (i = 0; i < 8; i++)
4683                         I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4684                 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4685                         for (i = 0; i < 8; i++)
4686                                 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4687         }
4688         i915_gem_detect_bit_6_swizzle(dev);
4689         init_waitqueue_head(&dev_priv->pending_flip_queue);
4690 }
4691
4692 /*
4693  * Create a physically contiguous memory object for this object
4694  * e.g. for cursor + overlay regs
4695  */
4696 int i915_gem_init_phys_object(struct drm_device *dev,
4697                               int id, int size, int align)
4698 {
4699         drm_i915_private_t *dev_priv = dev->dev_private;
4700         struct drm_i915_gem_phys_object *phys_obj;
4701         int ret;
4702
4703         if (dev_priv->mm.phys_objs[id - 1] || !size)
4704                 return 0;
4705
4706         phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4707         if (!phys_obj)
4708                 return -ENOMEM;
4709
4710         phys_obj->id = id;
4711
4712         phys_obj->handle = drm_pci_alloc(dev, size, align);
4713         if (!phys_obj->handle) {
4714                 ret = -ENOMEM;
4715                 goto kfree_obj;
4716         }
4717 #ifdef CONFIG_X86
4718         set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4719 #endif
4720
4721         dev_priv->mm.phys_objs[id - 1] = phys_obj;
4722
4723         return 0;
4724 kfree_obj:
4725         kfree(phys_obj);
4726         return ret;
4727 }
4728
4729 void i915_gem_free_phys_object(struct drm_device *dev, int id)
4730 {
4731         drm_i915_private_t *dev_priv = dev->dev_private;
4732         struct drm_i915_gem_phys_object *phys_obj;
4733
4734         if (!dev_priv->mm.phys_objs[id - 1])
4735                 return;
4736
4737         phys_obj = dev_priv->mm.phys_objs[id - 1];
4738         if (phys_obj->cur_obj) {
4739                 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4740         }
4741
4742 #ifdef CONFIG_X86
4743         set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4744 #endif
4745         drm_pci_free(dev, phys_obj->handle);
4746         kfree(phys_obj);
4747         dev_priv->mm.phys_objs[id - 1] = NULL;
4748 }
4749
4750 void i915_gem_free_all_phys_object(struct drm_device *dev)
4751 {
4752         int i;
4753
4754         for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4755                 i915_gem_free_phys_object(dev, i);
4756 }
4757
4758 void i915_gem_detach_phys_object(struct drm_device *dev,
4759                                  struct drm_gem_object *obj)
4760 {
4761         struct drm_i915_gem_object *obj_priv;
4762         int i;
4763         int ret;
4764         int page_count;
4765
4766         obj_priv = to_intel_bo(obj);
4767         if (!obj_priv->phys_obj)
4768                 return;
4769
4770         ret = i915_gem_object_get_pages(obj, 0);
4771         if (ret)
4772                 goto out;
4773
4774         page_count = obj->size / PAGE_SIZE;
4775
4776         for (i = 0; i < page_count; i++) {
4777                 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
4778                 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4779
4780                 memcpy(dst, src, PAGE_SIZE);
4781                 kunmap_atomic(dst, KM_USER0);
4782         }
4783         drm_clflush_pages(obj_priv->pages, page_count);
4784         drm_agp_chipset_flush(dev);
4785
4786         i915_gem_object_put_pages(obj);
4787 out:
4788         obj_priv->phys_obj->cur_obj = NULL;
4789         obj_priv->phys_obj = NULL;
4790 }
4791
4792 int
4793 i915_gem_attach_phys_object(struct drm_device *dev,
4794                             struct drm_gem_object *obj,
4795                             int id,
4796                             int align)
4797 {
4798         drm_i915_private_t *dev_priv = dev->dev_private;
4799         struct drm_i915_gem_object *obj_priv;
4800         int ret = 0;
4801         int page_count;
4802         int i;
4803
4804         if (id > I915_MAX_PHYS_OBJECT)
4805                 return -EINVAL;
4806
4807         obj_priv = to_intel_bo(obj);
4808
4809         if (obj_priv->phys_obj) {
4810                 if (obj_priv->phys_obj->id == id)
4811                         return 0;
4812                 i915_gem_detach_phys_object(dev, obj);
4813         }
4814
4815         /* create a new object */
4816         if (!dev_priv->mm.phys_objs[id - 1]) {
4817                 ret = i915_gem_init_phys_object(dev, id,
4818                                                 obj->size, align);
4819                 if (ret) {
4820                         DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
4821                         goto out;
4822                 }
4823         }
4824
4825         /* bind to the object */
4826         obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4827         obj_priv->phys_obj->cur_obj = obj;
4828
4829         ret = i915_gem_object_get_pages(obj, 0);
4830         if (ret) {
4831                 DRM_ERROR("failed to get page list\n");
4832                 goto out;
4833         }
4834
4835         page_count = obj->size / PAGE_SIZE;
4836
4837         for (i = 0; i < page_count; i++) {
4838                 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
4839                 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4840
4841                 memcpy(dst, src, PAGE_SIZE);
4842                 kunmap_atomic(src, KM_USER0);
4843         }
4844
4845         i915_gem_object_put_pages(obj);
4846
4847         return 0;
4848 out:
4849         return ret;
4850 }
4851
4852 static int
4853 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4854                      struct drm_i915_gem_pwrite *args,
4855                      struct drm_file *file_priv)
4856 {
4857         struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4858         void *obj_addr;
4859         int ret;
4860         char __user *user_data;
4861
4862         user_data = (char __user *) (uintptr_t) args->data_ptr;
4863         obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4864
4865         DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
4866         ret = copy_from_user(obj_addr, user_data, args->size);
4867         if (ret)
4868                 return -EFAULT;
4869
4870         drm_agp_chipset_flush(dev);
4871         return 0;
4872 }
4873
4874 void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4875 {
4876         struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4877
4878         /* Clean up our request list when the client is going away, so that
4879          * later retire_requests won't dereference our soon-to-be-gone
4880          * file_priv.
4881          */
4882         mutex_lock(&dev->struct_mutex);
4883         while (!list_empty(&i915_file_priv->mm.request_list))
4884                 list_del_init(i915_file_priv->mm.request_list.next);
4885         mutex_unlock(&dev->struct_mutex);
4886 }
4887
4888 static int
4889 i915_gpu_is_active(struct drm_device *dev)
4890 {
4891         drm_i915_private_t *dev_priv = dev->dev_private;
4892         int lists_empty;
4893
4894         spin_lock(&dev_priv->mm.active_list_lock);
4895         lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4896                       list_empty(&dev_priv->render_ring.active_list);
4897         if (HAS_BSD(dev))
4898                 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
4899         spin_unlock(&dev_priv->mm.active_list_lock);
4900
4901         return !lists_empty;
4902 }
4903
4904 static int
4905 i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
4906 {
4907         drm_i915_private_t *dev_priv, *next_dev;
4908         struct drm_i915_gem_object *obj_priv, *next_obj;
4909         int cnt = 0;
4910         int would_deadlock = 1;
4911
4912         /* "fast-path" to count number of available objects */
4913         if (nr_to_scan == 0) {
4914                 spin_lock(&shrink_list_lock);
4915                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4916                         struct drm_device *dev = dev_priv->dev;
4917
4918                         if (mutex_trylock(&dev->struct_mutex)) {
4919                                 list_for_each_entry(obj_priv,
4920                                                     &dev_priv->mm.inactive_list,
4921                                                     list)
4922                                         cnt++;
4923                                 mutex_unlock(&dev->struct_mutex);
4924                         }
4925                 }
4926                 spin_unlock(&shrink_list_lock);
4927
4928                 return (cnt / 100) * sysctl_vfs_cache_pressure;
4929         }
4930
4931         spin_lock(&shrink_list_lock);
4932
4933 rescan:
4934         /* first scan for clean buffers */
4935         list_for_each_entry_safe(dev_priv, next_dev,
4936                                  &shrink_list, mm.shrink_list) {
4937                 struct drm_device *dev = dev_priv->dev;
4938
4939                 if (! mutex_trylock(&dev->struct_mutex))
4940                         continue;
4941
4942                 spin_unlock(&shrink_list_lock);
4943                 i915_gem_retire_requests(dev);
4944
4945                 list_for_each_entry_safe(obj_priv, next_obj,
4946                                          &dev_priv->mm.inactive_list,
4947                                          list) {
4948                         if (i915_gem_object_is_purgeable(obj_priv)) {
4949                                 i915_gem_object_unbind(&obj_priv->base);
4950                                 if (--nr_to_scan <= 0)
4951                                         break;
4952                         }
4953                 }
4954
4955                 spin_lock(&shrink_list_lock);
4956                 mutex_unlock(&dev->struct_mutex);
4957
4958                 would_deadlock = 0;
4959
4960                 if (nr_to_scan <= 0)
4961                         break;
4962         }
4963
4964         /* second pass, evict/count anything still on the inactive list */
4965         list_for_each_entry_safe(dev_priv, next_dev,
4966                                  &shrink_list, mm.shrink_list) {
4967                 struct drm_device *dev = dev_priv->dev;
4968
4969                 if (! mutex_trylock(&dev->struct_mutex))
4970                         continue;
4971
4972                 spin_unlock(&shrink_list_lock);
4973
4974                 list_for_each_entry_safe(obj_priv, next_obj,
4975                                          &dev_priv->mm.inactive_list,
4976                                          list) {
4977                         if (nr_to_scan > 0) {
4978                                 i915_gem_object_unbind(&obj_priv->base);
4979                                 nr_to_scan--;
4980                         } else
4981                                 cnt++;
4982                 }
4983
4984                 spin_lock(&shrink_list_lock);
4985                 mutex_unlock(&dev->struct_mutex);
4986
4987                 would_deadlock = 0;
4988         }
4989
4990         if (nr_to_scan) {
4991                 int active = 0;
4992
4993                 /*
4994                  * We are desperate for pages, so as a last resort, wait
4995                  * for the GPU to finish and discard whatever we can.
4996                  * This has a dramatic impact to reduce the number of
4997                  * OOM-killer events whilst running the GPU aggressively.
4998                  */
4999                 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5000                         struct drm_device *dev = dev_priv->dev;
5001
5002                         if (!mutex_trylock(&dev->struct_mutex))
5003                                 continue;
5004
5005                         spin_unlock(&shrink_list_lock);
5006
5007                         if (i915_gpu_is_active(dev)) {
5008                                 i915_gpu_idle(dev);
5009                                 active++;
5010                         }
5011
5012                         spin_lock(&shrink_list_lock);
5013                         mutex_unlock(&dev->struct_mutex);
5014                 }
5015
5016                 if (active)
5017                         goto rescan;
5018         }
5019
5020         spin_unlock(&shrink_list_lock);
5021
5022         if (would_deadlock)
5023                 return -1;
5024         else if (cnt > 0)
5025                 return (cnt / 100) * sysctl_vfs_cache_pressure;
5026         else
5027                 return 0;
5028 }
5029
5030 static struct shrinker shrinker = {
5031         .shrink = i915_gem_shrink,
5032         .seeks = DEFAULT_SEEKS,
5033 };
5034
5035 __init void
5036 i915_gem_shrinker_init(void)
5037 {
5038     register_shrinker(&shrinker);
5039 }
5040
5041 __exit void
5042 i915_gem_shrinker_exit(void)
5043 {
5044     unregister_shrinker(&shrinker);
5045 }