]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/gpu/drm/i915/i915_gem.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Perform relocations in CPU domain [if in CPU domain]
[net-next-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
5a0e3ad6 34#include <linux/slab.h>
673a394b 35#include <linux/swap.h>
79e53945 36#include <linux/pci.h>
f8f235e5 37#include <linux/intel-gtt.h>
673a394b 38
0108a3ed 39static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
ba3d8d74
DV		 40
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
42 bool pipelined);
e47c68e9
EA		 43static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 45static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
2cf34d7b
CW		 51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 bool interruptible);
de151cf6
JB		 53static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 unsigned alignment);
de151cf6 55static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
71acb5eb
DA		 56static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
57 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv);
be72615b 59static void i915_gem_free_object_tail(struct drm_gem_object *obj);
673a394b 60
5cdf5881
CW		 61static int
62i915_gem_object_get_pages(struct drm_gem_object *obj,
63 gfp_t gfpmask);
64
65static void
66i915_gem_object_put_pages(struct drm_gem_object *obj);
67
31169714
CW		 68static LIST_HEAD(shrink_list);
69static DEFINE_SPINLOCK(shrink_list_lock);
70
73aa808f
CW		 71/* some bookkeeping */
72static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
73 size_t size)
74{
75 dev_priv->mm.object_count++;
76 dev_priv->mm.object_memory += size;
77}
78
79static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
80 size_t size)
81{
82 dev_priv->mm.object_count--;
83 dev_priv->mm.object_memory -= size;
84}
85
86static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv,
87 size_t size)
88{
89 dev_priv->mm.gtt_count++;
90 dev_priv->mm.gtt_memory += size;
91}
92
93static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv,
94 size_t size)
95{
96 dev_priv->mm.gtt_count--;
97 dev_priv->mm.gtt_memory -= size;
98}
99
100static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv,
101 size_t size)
102{
103 dev_priv->mm.pin_count++;
104 dev_priv->mm.pin_memory += size;
105}
106
107static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv,
108 size_t size)
109{
110 dev_priv->mm.pin_count--;
111 dev_priv->mm.pin_memory -= size;
112}
113
30dbf0c0
CW		 114int
115i915_gem_check_is_wedged(struct drm_device *dev)
116{
117 struct drm_i915_private *dev_priv = dev->dev_private;
118 struct completion *x = &dev_priv->error_completion;
119 unsigned long flags;
120 int ret;
121
122 if (!atomic_read(&dev_priv->mm.wedged))
123 return 0;
124
125 ret = wait_for_completion_interruptible(x);
126 if (ret)
127 return ret;
128
129 /* Success, we reset the GPU! */
130 if (!atomic_read(&dev_priv->mm.wedged))
131 return 0;
132
133 /* GPU is hung, bump the completion count to account for
134 * the token we just consumed so that we never hit zero and
135 * end up waiting upon a subsequent completion event that
136 * will never happen.
137 */
138 spin_lock_irqsave(&x->wait.lock, flags);
139 x->done++;
140 spin_unlock_irqrestore(&x->wait.lock, flags);
141 return -EIO;
142}
143
76c1dec1
CW		 144static int i915_mutex_lock_interruptible(struct drm_device *dev)
145{
146 struct drm_i915_private *dev_priv = dev->dev_private;
147 int ret;
148
149 ret = i915_gem_check_is_wedged(dev);
150 if (ret)
151 return ret;
152
153 ret = mutex_lock_interruptible(&dev->struct_mutex);
154 if (ret)
155 return ret;
156
157 if (atomic_read(&dev_priv->mm.wedged)) {
158 mutex_unlock(&dev->struct_mutex);
159 return -EAGAIN;
160 }
161
23bc5982 162 WARN_ON(i915_verify_lists(dev));
76c1dec1
CW		 163 return 0;
164}
30dbf0c0 165
7d1c4804
CW		 166static inline bool
167i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
168{
169 return obj_priv->gtt_space &&
170 !obj_priv->active &&
171 obj_priv->pin_count == 0;
172}
173
73aa808f
CW		 174int i915_gem_do_init(struct drm_device *dev,
175 unsigned long start,
79e53945 176 unsigned long end)
673a394b
EA		 177{
178 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 179
79e53945
JB		 180 if (start >= end ||
181 (start & (PAGE_SIZE - 1)) != 0 ||
182 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 183 return -EINVAL;
184 }
185
79e53945
JB		 186 drm_mm_init(&dev_priv->mm.gtt_space, start,
187 end - start);
673a394b 188
73aa808f 189 dev_priv->mm.gtt_total = end - start;
79e53945
JB		 190
191 return 0;
192}
673a394b 193
79e53945
JB		 194int
195i915_gem_init_ioctl(struct drm_device *dev, void *data,
196 struct drm_file *file_priv)
197{
198 struct drm_i915_gem_init *args = data;
199 int ret;
200
201 mutex_lock(&dev->struct_mutex);
202 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 203 mutex_unlock(&dev->struct_mutex);
204
79e53945 205 return ret;
673a394b
EA		 206}
207
5a125c3c
EA		 208int
209i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
210 struct drm_file *file_priv)
211{
73aa808f 212 struct drm_i915_private *dev_priv = dev->dev_private;
5a125c3c 213 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 214
215 if (!(dev->driver->driver_features & DRIVER_GEM))
216 return -ENODEV;
217
73aa808f
CW		 218 mutex_lock(&dev->struct_mutex);
219 args->aper_size = dev_priv->mm.gtt_total;
220 args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory;
221 mutex_unlock(&dev->struct_mutex);
5a125c3c
EA		 222
223 return 0;
224}
225
673a394b
EA		 226
227/**
228 * Creates a new mm object and returns a handle to it.
229 */
230int
231i915_gem_create_ioctl(struct drm_device *dev, void *data,
232 struct drm_file *file_priv)
233{
234 struct drm_i915_gem_create *args = data;
235 struct drm_gem_object *obj;
a1a2d1d3
PP		 236 int ret;
237 u32 handle;
673a394b
EA		 238
239 args->size = roundup(args->size, PAGE_SIZE);
240
241 /* Allocate the new object */
ac52bc56 242 obj = i915_gem_alloc_object(dev, args->size);
673a394b
EA		 243 if (obj == NULL)
244 return -ENOMEM;
245
246 ret = drm_gem_handle_create(file_priv, obj, &handle);
29d08b3e
DA		 247 /* drop reference from allocate - handle holds it now */
248 drm_gem_object_unreference_unlocked(obj);
1dfd9754 249 if (ret) {
673a394b 250 return ret;
1dfd9754 251 }
673a394b 252
1dfd9754 253 args->handle = handle;
673a394b
EA		 254 return 0;
255}
256
eb01459f
EA		 257static inline int
258fast_shmem_read(struct page **pages,
259 loff_t page_base, int page_offset,
260 char __user *data,
261 int length)
262{
263 char __iomem *vaddr;
2bc43b5c 264 int unwritten;
eb01459f
EA		 265
266 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
267 if (vaddr == NULL)
268 return -ENOMEM;
2bc43b5c 269 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 270 kunmap_atomic(vaddr, KM_USER0);
271
2bc43b5c
FM		 272 if (unwritten)
273 return -EFAULT;
274
275 return 0;
eb01459f
EA		 276}
277
280b713b
EA		 278static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
279{
280 drm_i915_private_t *dev_priv = obj->dev->dev_private;
23010e43 281 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
280b713b
EA		 282
283 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
284 obj_priv->tiling_mode != I915_TILING_NONE;
285}
286
99a03df5 287static inline void
40123c1f
EA		 288slow_shmem_copy(struct page *dst_page,
289 int dst_offset,
290 struct page *src_page,
291 int src_offset,
292 int length)
293{
294 char *dst_vaddr, *src_vaddr;
295
99a03df5
CW		 296 dst_vaddr = kmap(dst_page);
297 src_vaddr = kmap(src_page);
40123c1f
EA		 298
299 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
300
99a03df5
CW		 301 kunmap(src_page);
302 kunmap(dst_page);
40123c1f
EA		 303}
304
99a03df5 305static inline void
280b713b
EA		 306slow_shmem_bit17_copy(struct page *gpu_page,
307 int gpu_offset,
308 struct page *cpu_page,
309 int cpu_offset,
310 int length,
311 int is_read)
312{
313 char *gpu_vaddr, *cpu_vaddr;
314
315 /* Use the unswizzled path if this page isn't affected. */
316 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
317 if (is_read)
318 return slow_shmem_copy(cpu_page, cpu_offset,
319 gpu_page, gpu_offset, length);
320 else
321 return slow_shmem_copy(gpu_page, gpu_offset,
322 cpu_page, cpu_offset, length);
323 }
324
99a03df5
CW		 325 gpu_vaddr = kmap(gpu_page);
326 cpu_vaddr = kmap(cpu_page);
280b713b
EA		 327
328 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
329 * XORing with the other bits (A9 for Y, A9 and A10 for X)
330 */
331 while (length > 0) {
332 int cacheline_end = ALIGN(gpu_offset + 1, 64);
333 int this_length = min(cacheline_end - gpu_offset, length);
334 int swizzled_gpu_offset = gpu_offset ^ 64;
335
336 if (is_read) {
337 memcpy(cpu_vaddr + cpu_offset,
338 gpu_vaddr + swizzled_gpu_offset,
339 this_length);
340 } else {
341 memcpy(gpu_vaddr + swizzled_gpu_offset,
342 cpu_vaddr + cpu_offset,
343 this_length);
344 }
345 cpu_offset += this_length;
346 gpu_offset += this_length;
347 length -= this_length;
348 }
349
99a03df5
CW		 350 kunmap(cpu_page);
351 kunmap(gpu_page);
280b713b
EA		 352}
353
eb01459f
EA		 354/**
355 * This is the fast shmem pread path, which attempts to copy_from_user directly
356 * from the backing pages of the object to the user's address space. On a
357 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
358 */
359static int
360i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
361 struct drm_i915_gem_pread *args,
362 struct drm_file *file_priv)
363{
23010e43 364 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 365 ssize_t remain;
366 loff_t offset, page_base;
367 char __user *user_data;
368 int page_offset, page_length;
369 int ret;
370
371 user_data = (char __user *) (uintptr_t) args->data_ptr;
372 remain = args->size;
373
76c1dec1
CW		 374 ret = i915_mutex_lock_interruptible(dev);
375 if (ret)
376 return ret;
eb01459f 377
4bdadb97 378 ret = i915_gem_object_get_pages(obj, 0);
eb01459f
EA		 379 if (ret != 0)
380 goto fail_unlock;
381
382 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
383 args->size);
384 if (ret != 0)
385 goto fail_put_pages;
386
23010e43 387 obj_priv = to_intel_bo(obj);
eb01459f
EA		 388 offset = args->offset;
389
390 while (remain > 0) {
391 /* Operation in this page
392 *
393 * page_base = page offset within aperture
394 * page_offset = offset within page
395 * page_length = bytes to copy for this page
396 */
397 page_base = (offset & ~(PAGE_SIZE-1));
398 page_offset = offset & (PAGE_SIZE-1);
399 page_length = remain;
400 if ((page_offset + remain) > PAGE_SIZE)
401 page_length = PAGE_SIZE - page_offset;
402
403 ret = fast_shmem_read(obj_priv->pages,
404 page_base, page_offset,
405 user_data, page_length);
406 if (ret)
407 goto fail_put_pages;
408
409 remain -= page_length;
410 user_data += page_length;
411 offset += page_length;
412 }
413
414fail_put_pages:
415 i915_gem_object_put_pages(obj);
416fail_unlock:
417 mutex_unlock(&dev->struct_mutex);
418
419 return ret;
420}
421
07f73f69
CW		 422static int
423i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
424{
425 int ret;
426
4bdadb97 427 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
07f73f69
CW		 428
429 /* If we've insufficient memory to map in the pages, attempt
430 * to make some space by throwing out some old buffers.
431 */
432 if (ret == -ENOMEM) {
433 struct drm_device *dev = obj->dev;
07f73f69 434
0108a3ed
DV		 435 ret = i915_gem_evict_something(dev, obj->size,
436 i915_gem_get_gtt_alignment(obj));
07f73f69
CW		 437 if (ret)
438 return ret;
439
4bdadb97 440 ret = i915_gem_object_get_pages(obj, 0);
07f73f69
CW		 441 }
442
443 return ret;
444}
445
eb01459f
EA		 446/**
447 * This is the fallback shmem pread path, which allocates temporary storage
448 * in kernel space to copy_to_user into outside of the struct_mutex, so we
449 * can copy out of the object's backing pages while holding the struct mutex
450 * and not take page faults.
451 */
452static int
453i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
454 struct drm_i915_gem_pread *args,
455 struct drm_file *file_priv)
456{
23010e43 457 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
eb01459f
EA		 458 struct mm_struct *mm = current->mm;
459 struct page **user_pages;
460 ssize_t remain;
461 loff_t offset, pinned_pages, i;
462 loff_t first_data_page, last_data_page, num_pages;
463 int shmem_page_index, shmem_page_offset;
464 int data_page_index, data_page_offset;
465 int page_length;
466 int ret;
467 uint64_t data_ptr = args->data_ptr;
280b713b 468 int do_bit17_swizzling;
eb01459f
EA		 469
470 remain = args->size;
471
472 /* Pin the user pages containing the data. We can't fault while
473 * holding the struct mutex, yet we want to hold it while
474 * dereferencing the user data.
475 */
476 first_data_page = data_ptr / PAGE_SIZE;
477 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
478 num_pages = last_data_page - first_data_page + 1;
479
8e7d2b2c 480 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 481 if (user_pages == NULL)
482 return -ENOMEM;
483
484 down_read(&mm->mmap_sem);
485 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 486 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 487 up_read(&mm->mmap_sem);
488 if (pinned_pages < num_pages) {
489 ret = -EFAULT;
490 goto fail_put_user_pages;
491 }
492
280b713b
EA		 493 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
494
76c1dec1
CW		 495 ret = i915_mutex_lock_interruptible(dev);
496 if (ret)
497 goto fail_put_user_pages;
eb01459f 498
07f73f69
CW		 499 ret = i915_gem_object_get_pages_or_evict(obj);
500 if (ret)
eb01459f
EA		 501 goto fail_unlock;
502
503 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
504 args->size);
505 if (ret != 0)
506 goto fail_put_pages;
507
23010e43 508 obj_priv = to_intel_bo(obj);
eb01459f
EA		 509 offset = args->offset;
510
511 while (remain > 0) {
512 /* Operation in this page
513 *
514 * shmem_page_index = page number within shmem file
515 * shmem_page_offset = offset within page in shmem file
516 * data_page_index = page number in get_user_pages return
517 * data_page_offset = offset with data_page_index page.
518 * page_length = bytes to copy for this page
519 */
520 shmem_page_index = offset / PAGE_SIZE;
521 shmem_page_offset = offset & ~PAGE_MASK;
522 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
523 data_page_offset = data_ptr & ~PAGE_MASK;
524
525 page_length = remain;
526 if ((shmem_page_offset + page_length) > PAGE_SIZE)
527 page_length = PAGE_SIZE - shmem_page_offset;
528 if ((data_page_offset + page_length) > PAGE_SIZE)
529 page_length = PAGE_SIZE - data_page_offset;
530
280b713b 531 if (do_bit17_swizzling) {
99a03df5 532 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b 533 shmem_page_offset,
99a03df5
CW		 534 user_pages[data_page_index],
535 data_page_offset,
536 page_length,
537 1);
538 } else {
539 slow_shmem_copy(user_pages[data_page_index],
540 data_page_offset,
541 obj_priv->pages[shmem_page_index],
542 shmem_page_offset,
543 page_length);
280b713b 544 }
eb01459f
EA		 545
546 remain -= page_length;
547 data_ptr += page_length;
548 offset += page_length;
549 }
550
551fail_put_pages:
552 i915_gem_object_put_pages(obj);
553fail_unlock:
554 mutex_unlock(&dev->struct_mutex);
555fail_put_user_pages:
556 for (i = 0; i < pinned_pages; i++) {
557 SetPageDirty(user_pages[i]);
558 page_cache_release(user_pages[i]);
559 }
8e7d2b2c 560 drm_free_large(user_pages);
eb01459f
EA		 561
562 return ret;
563}
564
673a394b
EA		 565/**
566 * Reads data from the object referenced by handle.
567 *
568 * On error, the contents of *data are undefined.
569 */
570int
571i915_gem_pread_ioctl(struct drm_device *dev, void *data,
572 struct drm_file *file_priv)
573{
574 struct drm_i915_gem_pread *args = data;
575 struct drm_gem_object *obj;
576 struct drm_i915_gem_object *obj_priv;
35b62a89 577 int ret = 0;
673a394b
EA		 578
579 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
580 if (obj == NULL)
bf79cb91 581 return -ENOENT;
23010e43 582 obj_priv = to_intel_bo(obj);
673a394b 583
7dcd2499
CW		 584 /* Bounds check source. */
585 if (args->offset > obj->size || args->size > obj->size - args->offset) {
ce9d419d 586 ret = -EINVAL;
35b62a89 587 goto out;
ce9d419d
CW		 588 }
589
35b62a89
CW		 590 if (args->size == 0)
591 goto out;
592
ce9d419d
CW		 593 if (!access_ok(VERIFY_WRITE,
594 (char __user *)(uintptr_t)args->data_ptr,
595 args->size)) {
596 ret = -EFAULT;
35b62a89 597 goto out;
673a394b
EA		 598 }
599
280b713b 600 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 601 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 602 } else {
603 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
604 if (ret != 0)
605 ret = i915_gem_shmem_pread_slow(dev, obj, args,
606 file_priv);
607 }
673a394b 608
35b62a89 609out:
bc9025bd 610 drm_gem_object_unreference_unlocked(obj);
eb01459f 611 return ret;
673a394b
EA		 612}
613
0839ccb8
KP		 614/* This is the fast write path which cannot handle
615 * page faults in the source data
9b7530cc 616 */
0839ccb8
KP		 617
618static inline int
619fast_user_write(struct io_mapping *mapping,
620 loff_t page_base, int page_offset,
621 char __user *user_data,
622 int length)
9b7530cc 623{
9b7530cc 624 char *vaddr_atomic;
0839ccb8 625 unsigned long unwritten;
9b7530cc 626
fca3ec01 627 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
0839ccb8
KP		 628 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
629 user_data, length);
fca3ec01 630 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
0839ccb8
KP		 631 if (unwritten)
632 return -EFAULT;
633 return 0;
634}
635
636/* Here's the write path which can sleep for
637 * page faults
638 */
639
ab34c226 640static inline void
3de09aa3
EA		 641slow_kernel_write(struct io_mapping *mapping,
642 loff_t gtt_base, int gtt_offset,
643 struct page *user_page, int user_offset,
644 int length)
0839ccb8 645{
ab34c226
CW		 646 char __iomem *dst_vaddr;
647 char *src_vaddr;
0839ccb8 648
ab34c226
CW		 649 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
650 src_vaddr = kmap(user_page);
651
652 memcpy_toio(dst_vaddr + gtt_offset,
653 src_vaddr + user_offset,
654 length);
655
656 kunmap(user_page);
657 io_mapping_unmap(dst_vaddr);
9b7530cc
LT		 658}
659
40123c1f
EA		 660static inline int
661fast_shmem_write(struct page **pages,
662 loff_t page_base, int page_offset,
663 char __user *data,
664 int length)
665{
666 char __iomem *vaddr;
d0088775 667 unsigned long unwritten;
40123c1f
EA		 668
669 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
670 if (vaddr == NULL)
671 return -ENOMEM;
d0088775 672 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 673 kunmap_atomic(vaddr, KM_USER0);
674
d0088775
DA		 675 if (unwritten)
676 return -EFAULT;
40123c1f
EA		 677 return 0;
678}
679
3de09aa3
EA		 680/**
681 * This is the fast pwrite path, where we copy the data directly from the
682 * user into the GTT, uncached.
683 */
673a394b 684static int
3de09aa3
EA		 685i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
686 struct drm_i915_gem_pwrite *args,
687 struct drm_file *file_priv)
673a394b 688{
23010e43 689 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
0839ccb8 690 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 691 ssize_t remain;
0839ccb8 692 loff_t offset, page_base;
673a394b 693 char __user *user_data;
0839ccb8
KP		 694 int page_offset, page_length;
695 int ret;
673a394b
EA		 696
697 user_data = (char __user *) (uintptr_t) args->data_ptr;
698 remain = args->size;
673a394b 699
76c1dec1
CW		 700 ret = i915_mutex_lock_interruptible(dev);
701 if (ret)
702 return ret;
673a394b 703
673a394b
EA		 704 ret = i915_gem_object_pin(obj, 0);
705 if (ret) {
706 mutex_unlock(&dev->struct_mutex);
707 return ret;
708 }
2ef7eeaa 709 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 710 if (ret)
711 goto fail;
712
23010e43 713 obj_priv = to_intel_bo(obj);
673a394b 714 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 715
716 while (remain > 0) {
717 /* Operation in this page
718 *
0839ccb8
KP		 719 * page_base = page offset within aperture
720 * page_offset = offset within page
721 * page_length = bytes to copy for this page
673a394b 722 */
0839ccb8
KP		 723 page_base = (offset & ~(PAGE_SIZE-1));
724 page_offset = offset & (PAGE_SIZE-1);
725 page_length = remain;
726 if ((page_offset + remain) > PAGE_SIZE)
727 page_length = PAGE_SIZE - page_offset;
728
729 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
730 page_offset, user_data, page_length);
731
732 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 733 * source page isn't available. Return the error and we'll
734 * retry in the slow path.
0839ccb8 735 */
3de09aa3
EA		 736 if (ret)
737 goto fail;
673a394b 738
0839ccb8
KP		 739 remain -= page_length;
740 user_data += page_length;
741 offset += page_length;
673a394b 742 }
673a394b
EA		 743
744fail:
745 i915_gem_object_unpin(obj);
746 mutex_unlock(&dev->struct_mutex);
747
748 return ret;
749}
750
3de09aa3
EA		 751/**
752 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
753 * the memory and maps it using kmap_atomic for copying.
754 *
755 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
756 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
757 */
3043c60c 758static int
3de09aa3
EA		 759i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
760 struct drm_i915_gem_pwrite *args,
761 struct drm_file *file_priv)
673a394b 762{
23010e43 763 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
3de09aa3
EA		 764 drm_i915_private_t *dev_priv = dev->dev_private;
765 ssize_t remain;
766 loff_t gtt_page_base, offset;
767 loff_t first_data_page, last_data_page, num_pages;
768 loff_t pinned_pages, i;
769 struct page **user_pages;
770 struct mm_struct *mm = current->mm;
771 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 772 int ret;
3de09aa3
EA		 773 uint64_t data_ptr = args->data_ptr;
774
775 remain = args->size;
776
777 /* Pin the user pages containing the data. We can't fault while
778 * holding the struct mutex, and all of the pwrite implementations
779 * want to hold it while dereferencing the user data.
780 */
781 first_data_page = data_ptr / PAGE_SIZE;
782 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
783 num_pages = last_data_page - first_data_page + 1;
784
8e7d2b2c 785 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 786 if (user_pages == NULL)
787 return -ENOMEM;
788
789 down_read(&mm->mmap_sem);
790 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
791 num_pages, 0, 0, user_pages, NULL);
792 up_read(&mm->mmap_sem);
793 if (pinned_pages < num_pages) {
794 ret = -EFAULT;
795 goto out_unpin_pages;
796 }
673a394b 797
76c1dec1
CW		 798 ret = i915_mutex_lock_interruptible(dev);
799 if (ret)
800 goto out_unpin_pages;
801
3de09aa3
EA		 802 ret = i915_gem_object_pin(obj, 0);
803 if (ret)
804 goto out_unlock;
805
806 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
807 if (ret)
808 goto out_unpin_object;
809
23010e43 810 obj_priv = to_intel_bo(obj);
3de09aa3
EA		 811 offset = obj_priv->gtt_offset + args->offset;
812
813 while (remain > 0) {
814 /* Operation in this page
815 *
816 * gtt_page_base = page offset within aperture
817 * gtt_page_offset = offset within page in aperture
818 * data_page_index = page number in get_user_pages return
819 * data_page_offset = offset with data_page_index page.
820 * page_length = bytes to copy for this page
821 */
822 gtt_page_base = offset & PAGE_MASK;
823 gtt_page_offset = offset & ~PAGE_MASK;
824 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
825 data_page_offset = data_ptr & ~PAGE_MASK;
826
827 page_length = remain;
828 if ((gtt_page_offset + page_length) > PAGE_SIZE)
829 page_length = PAGE_SIZE - gtt_page_offset;
830 if ((data_page_offset + page_length) > PAGE_SIZE)
831 page_length = PAGE_SIZE - data_page_offset;
832
ab34c226
CW		 833 slow_kernel_write(dev_priv->mm.gtt_mapping,
834 gtt_page_base, gtt_page_offset,
835 user_pages[data_page_index],
836 data_page_offset,
837 page_length);
3de09aa3
EA		 838
839 remain -= page_length;
840 offset += page_length;
841 data_ptr += page_length;
842 }
843
844out_unpin_object:
845 i915_gem_object_unpin(obj);
846out_unlock:
847 mutex_unlock(&dev->struct_mutex);
848out_unpin_pages:
849 for (i = 0; i < pinned_pages; i++)
850 page_cache_release(user_pages[i]);
8e7d2b2c 851 drm_free_large(user_pages);
3de09aa3
EA		 852
853 return ret;
854}
855
40123c1f
EA		 856/**
857 * This is the fast shmem pwrite path, which attempts to directly
858 * copy_from_user into the kmapped pages backing the object.
859 */
3043c60c 860static int
40123c1f
EA		 861i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
862 struct drm_i915_gem_pwrite *args,
863 struct drm_file *file_priv)
673a394b 864{
23010e43 865 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 866 ssize_t remain;
867 loff_t offset, page_base;
868 char __user *user_data;
869 int page_offset, page_length;
673a394b 870 int ret;
40123c1f
EA		 871
872 user_data = (char __user *) (uintptr_t) args->data_ptr;
873 remain = args->size;
673a394b 874
76c1dec1
CW		 875 ret = i915_mutex_lock_interruptible(dev);
876 if (ret)
877 return ret;
673a394b 878
4bdadb97 879 ret = i915_gem_object_get_pages(obj, 0);
40123c1f
EA		 880 if (ret != 0)
881 goto fail_unlock;
673a394b 882
e47c68e9 883 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 884 if (ret != 0)
885 goto fail_put_pages;
886
23010e43 887 obj_priv = to_intel_bo(obj);
40123c1f
EA		 888 offset = args->offset;
889 obj_priv->dirty = 1;
890
891 while (remain > 0) {
892 /* Operation in this page
893 *
894 * page_base = page offset within aperture
895 * page_offset = offset within page
896 * page_length = bytes to copy for this page
897 */
898 page_base = (offset & ~(PAGE_SIZE-1));
899 page_offset = offset & (PAGE_SIZE-1);
900 page_length = remain;
901 if ((page_offset + remain) > PAGE_SIZE)
902 page_length = PAGE_SIZE - page_offset;
903
904 ret = fast_shmem_write(obj_priv->pages,
905 page_base, page_offset,
906 user_data, page_length);
907 if (ret)
908 goto fail_put_pages;
909
910 remain -= page_length;
911 user_data += page_length;
912 offset += page_length;
913 }
914
915fail_put_pages:
916 i915_gem_object_put_pages(obj);
917fail_unlock:
918 mutex_unlock(&dev->struct_mutex);
919
920 return ret;
921}
922
923/**
924 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
925 * the memory and maps it using kmap_atomic for copying.
926 *
927 * This avoids taking mmap_sem for faulting on the user's address while the
928 * struct_mutex is held.
929 */
930static int
931i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
932 struct drm_i915_gem_pwrite *args,
933 struct drm_file *file_priv)
934{
23010e43 935 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
40123c1f
EA		 936 struct mm_struct *mm = current->mm;
937 struct page **user_pages;
938 ssize_t remain;
939 loff_t offset, pinned_pages, i;
940 loff_t first_data_page, last_data_page, num_pages;
941 int shmem_page_index, shmem_page_offset;
942 int data_page_index, data_page_offset;
943 int page_length;
944 int ret;
945 uint64_t data_ptr = args->data_ptr;
280b713b 946 int do_bit17_swizzling;
40123c1f
EA		 947
948 remain = args->size;
949
950 /* Pin the user pages containing the data. We can't fault while
951 * holding the struct mutex, and all of the pwrite implementations
952 * want to hold it while dereferencing the user data.
953 */
954 first_data_page = data_ptr / PAGE_SIZE;
955 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
956 num_pages = last_data_page - first_data_page + 1;
957
8e7d2b2c 958 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 959 if (user_pages == NULL)
960 return -ENOMEM;
961
962 down_read(&mm->mmap_sem);
963 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
964 num_pages, 0, 0, user_pages, NULL);
965 up_read(&mm->mmap_sem);
966 if (pinned_pages < num_pages) {
967 ret = -EFAULT;
968 goto fail_put_user_pages;
673a394b
EA		 969 }
970
280b713b
EA		 971 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
972
76c1dec1
CW		 973 ret = i915_mutex_lock_interruptible(dev);
974 if (ret)
975 goto fail_put_user_pages;
40123c1f 976
07f73f69
CW		 977 ret = i915_gem_object_get_pages_or_evict(obj);
978 if (ret)
40123c1f
EA		 979 goto fail_unlock;
980
981 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
982 if (ret != 0)
983 goto fail_put_pages;
984
23010e43 985 obj_priv = to_intel_bo(obj);
673a394b 986 offset = args->offset;
40123c1f 987 obj_priv->dirty = 1;
673a394b 988
40123c1f
EA		 989 while (remain > 0) {
990 /* Operation in this page
991 *
992 * shmem_page_index = page number within shmem file
993 * shmem_page_offset = offset within page in shmem file
994 * data_page_index = page number in get_user_pages return
995 * data_page_offset = offset with data_page_index page.
996 * page_length = bytes to copy for this page
997 */
998 shmem_page_index = offset / PAGE_SIZE;
999 shmem_page_offset = offset & ~PAGE_MASK;
1000 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
1001 data_page_offset = data_ptr & ~PAGE_MASK;
1002
1003 page_length = remain;
1004 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1005 page_length = PAGE_SIZE - shmem_page_offset;
1006 if ((data_page_offset + page_length) > PAGE_SIZE)
1007 page_length = PAGE_SIZE - data_page_offset;
1008
280b713b 1009 if (do_bit17_swizzling) {
99a03df5 1010 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
280b713b
EA		 1011 shmem_page_offset,
1012 user_pages[data_page_index],
1013 data_page_offset,
99a03df5
CW		 1014 page_length,
1015 0);
1016 } else {
1017 slow_shmem_copy(obj_priv->pages[shmem_page_index],
1018 shmem_page_offset,
1019 user_pages[data_page_index],
1020 data_page_offset,
1021 page_length);
280b713b 1022 }
40123c1f
EA		 1023
1024 remain -= page_length;
1025 data_ptr += page_length;
1026 offset += page_length;
673a394b
EA		 1027 }
1028
40123c1f
EA		 1029fail_put_pages:
1030 i915_gem_object_put_pages(obj);
1031fail_unlock:
673a394b 1032 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 1033fail_put_user_pages:
1034 for (i = 0; i < pinned_pages; i++)
1035 page_cache_release(user_pages[i]);
8e7d2b2c 1036 drm_free_large(user_pages);
673a394b 1037
40123c1f 1038 return ret;
673a394b
EA		 1039}
1040
1041/**
1042 * Writes data to the object referenced by handle.
1043 *
1044 * On error, the contents of the buffer that were to be modified are undefined.
1045 */
1046int
1047i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1048 struct drm_file *file_priv)
1049{
1050 struct drm_i915_gem_pwrite *args = data;
1051 struct drm_gem_object *obj;
1052 struct drm_i915_gem_object *obj_priv;
1053 int ret = 0;
1054
1055 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1056 if (obj == NULL)
bf79cb91 1057 return -ENOENT;
23010e43 1058 obj_priv = to_intel_bo(obj);
673a394b 1059
7dcd2499
CW		 1060 /* Bounds check destination. */
1061 if (args->offset > obj->size || args->size > obj->size - args->offset) {
ce9d419d 1062 ret = -EINVAL;
35b62a89 1063 goto out;
ce9d419d
CW		 1064 }
1065
35b62a89
CW		 1066 if (args->size == 0)
1067 goto out;
1068
ce9d419d
CW		 1069 if (!access_ok(VERIFY_READ,
1070 (char __user *)(uintptr_t)args->data_ptr,
1071 args->size)) {
1072 ret = -EFAULT;
35b62a89 1073 goto out;
673a394b
EA		 1074 }
1075
1076 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1077 * it would end up going through the fenced access, and we'll get
1078 * different detiling behavior between reading and writing.
1079 * pread/pwrite currently are reading and writing from the CPU
1080 * perspective, requiring manual detiling by the client.
1081 */
71acb5eb
DA		 1082 if (obj_priv->phys_obj)
1083 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
1084 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
5cdf5881 1085 obj_priv->gtt_space &&
9b8c4a0b 1086 obj->write_domain != I915_GEM_DOMAIN_CPU) {
3de09aa3
EA		 1087 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
1088 if (ret == -EFAULT) {
1089 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
1090 file_priv);
1091 }
280b713b
EA		 1092 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1093 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 1094 } else {
1095 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1096 if (ret == -EFAULT) {
1097 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1098 file_priv);
1099 }
1100 }
673a394b
EA		 1101
1102#if WATCH_PWRITE
1103 if (ret)
1104 DRM_INFO("pwrite failed %d\n", ret);
1105#endif
1106
35b62a89 1107out:
bc9025bd 1108 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1109 return ret;
1110}
1111
1112/**
2ef7eeaa
EA		 1113 * Called when user space prepares to use an object with the CPU, either
1114 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1115 */
1116int
1117i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1118 struct drm_file *file_priv)
1119{
a09ba7fa 1120 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1121 struct drm_i915_gem_set_domain *args = data;
1122 struct drm_gem_object *obj;
652c393a 1123 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1124 uint32_t read_domains = args->read_domains;
1125 uint32_t write_domain = args->write_domain;
673a394b
EA		 1126 int ret;
1127
1128 if (!(dev->driver->driver_features & DRIVER_GEM))
1129 return -ENODEV;
1130
2ef7eeaa 1131 /* Only handle setting domains to types used by the CPU. */
21d509e3 1132 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1133 return -EINVAL;
1134
21d509e3 1135 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1136 return -EINVAL;
1137
1138 /* Having something in the write domain implies it's in the read
1139 * domain, and only that read domain. Enforce that in the request.
1140 */
1141 if (write_domain != 0 && read_domains != write_domain)
1142 return -EINVAL;
1143
673a394b
EA		 1144 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1145 if (obj == NULL)
bf79cb91 1146 return -ENOENT;
23010e43 1147 obj_priv = to_intel_bo(obj);
673a394b 1148
76c1dec1
CW		 1149 ret = i915_mutex_lock_interruptible(dev);
1150 if (ret) {
1151 drm_gem_object_unreference_unlocked(obj);
1152 return ret;
1153 }
652c393a
JB		 1154
1155 intel_mark_busy(dev, obj);
1156
2ef7eeaa
EA		 1157 if (read_domains & I915_GEM_DOMAIN_GTT) {
1158 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1159
a09ba7fa
EA		 1160 /* Update the LRU on the fence for the CPU access that's
1161 * about to occur.
1162 */
1163 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 1164 struct drm_i915_fence_reg *reg =
1165 &dev_priv->fence_regs[obj_priv->fence_reg];
1166 list_move_tail(&reg->lru_list,
a09ba7fa
EA		 1167 &dev_priv->mm.fence_list);
1168 }
1169
02354392
EA		 1170 /* Silently promote "you're not bound, there was nothing to do"
1171 * to success, since the client was just asking us to
1172 * make sure everything was done.
1173 */
1174 if (ret == -EINVAL)
1175 ret = 0;
2ef7eeaa 1176 } else {
e47c68e9 1177 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1178 }
1179
7d1c4804
CW		 1180 /* Maintain LRU order of "inactive" objects */
1181 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1182 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1183
673a394b
EA		 1184 drm_gem_object_unreference(obj);
1185 mutex_unlock(&dev->struct_mutex);
1186 return ret;
1187}
1188
1189/**
1190 * Called when user space has done writes to this buffer
1191 */
1192int
1193i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1194 struct drm_file *file_priv)
1195{
1196 struct drm_i915_gem_sw_finish *args = data;
1197 struct drm_gem_object *obj;
673a394b
EA		 1198 int ret = 0;
1199
1200 if (!(dev->driver->driver_features & DRIVER_GEM))
1201 return -ENODEV;
1202
673a394b 1203 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
76c1dec1 1204 if (obj == NULL)
bf79cb91 1205 return -ENOENT;
76c1dec1
CW		 1206
1207 ret = i915_mutex_lock_interruptible(dev);
1208 if (ret) {
1209 drm_gem_object_unreference_unlocked(obj);
1210 return ret;
673a394b
EA		 1211 }
1212
673a394b 1213 /* Pinned buffers may be scanout, so flush the cache */
3d2a812a 1214 if (to_intel_bo(obj)->pin_count)
e47c68e9
EA		 1215 i915_gem_object_flush_cpu_write_domain(obj);
1216
673a394b
EA		 1217 drm_gem_object_unreference(obj);
1218 mutex_unlock(&dev->struct_mutex);
1219 return ret;
1220}
1221
1222/**
1223 * Maps the contents of an object, returning the address it is mapped
1224 * into.
1225 *
1226 * While the mapping holds a reference on the contents of the object, it doesn't
1227 * imply a ref on the object itself.
1228 */
1229int
1230i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1231 struct drm_file *file_priv)
1232{
1233 struct drm_i915_gem_mmap *args = data;
1234 struct drm_gem_object *obj;
1235 loff_t offset;
1236 unsigned long addr;
1237
1238 if (!(dev->driver->driver_features & DRIVER_GEM))
1239 return -ENODEV;
1240
1241 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1242 if (obj == NULL)
bf79cb91 1243 return -ENOENT;
673a394b
EA		 1244
1245 offset = args->offset;
1246
1247 down_write(&current->mm->mmap_sem);
1248 addr = do_mmap(obj->filp, 0, args->size,
1249 PROT_READ | PROT_WRITE, MAP_SHARED,
1250 args->offset);
1251 up_write(&current->mm->mmap_sem);
bc9025bd 1252 drm_gem_object_unreference_unlocked(obj);
673a394b
EA		 1253 if (IS_ERR((void *)addr))
1254 return addr;
1255
1256 args->addr_ptr = (uint64_t) addr;
1257
1258 return 0;
1259}
1260
de151cf6
JB		 1261/**
1262 * i915_gem_fault - fault a page into the GTT
1263 * vma: VMA in question
1264 * vmf: fault info
1265 *
1266 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1267 * from userspace. The fault handler takes care of binding the object to
1268 * the GTT (if needed), allocating and programming a fence register (again,
1269 * only if needed based on whether the old reg is still valid or the object
1270 * is tiled) and inserting a new PTE into the faulting process.
1271 *
1272 * Note that the faulting process may involve evicting existing objects
1273 * from the GTT and/or fence registers to make room. So performance may
1274 * suffer if the GTT working set is large or there are few fence registers
1275 * left.
1276 */
1277int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1278{
1279 struct drm_gem_object *obj = vma->vm_private_data;
1280 struct drm_device *dev = obj->dev;
7d1c4804 1281 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1282 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1283 pgoff_t page_offset;
1284 unsigned long pfn;
1285 int ret = 0;
0f973f27 1286 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1287
1288 /* We don't use vmf->pgoff since that has the fake offset */
1289 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1290 PAGE_SHIFT;
1291
1292 /* Now bind it into the GTT if needed */
1293 mutex_lock(&dev->struct_mutex);
1294 if (!obj_priv->gtt_space) {
e67b8ce1 1295 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1296 if (ret)
1297 goto unlock;
07f4f3e8 1298
07f4f3e8 1299 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1300 if (ret)
1301 goto unlock;
de151cf6
JB		 1302 }
1303
1304 /* Need a new fence register? */
a09ba7fa 1305 if (obj_priv->tiling_mode != I915_TILING_NONE) {
2cf34d7b 1306 ret = i915_gem_object_get_fence_reg(obj, true);
c715089f
CW		 1307 if (ret)
1308 goto unlock;
d9ddcb96 1309 }
de151cf6 1310
7d1c4804
CW		 1311 if (i915_gem_object_is_inactive(obj_priv))
1312 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1313
de151cf6
JB		 1314 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1315 page_offset;
1316
1317 /* Finally, remap it using the new GTT offset */
1318 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1319unlock:
de151cf6
JB		 1320 mutex_unlock(&dev->struct_mutex);
1321
1322 switch (ret) {
c715089f
CW		 1323 case 0:
1324 case -ERESTARTSYS:
1325 return VM_FAULT_NOPAGE;
de151cf6
JB		 1326 case -ENOMEM:
1327 case -EAGAIN:
1328 return VM_FAULT_OOM;
de151cf6 1329 default:
c715089f 1330 return VM_FAULT_SIGBUS;
de151cf6
JB		 1331 }
1332}
1333
1334/**
1335 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1336 * @obj: obj in question
1337 *
1338 * GEM memory mapping works by handing back to userspace a fake mmap offset
1339 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1340 * up the object based on the offset and sets up the various memory mapping
1341 * structures.
1342 *
1343 * This routine allocates and attaches a fake offset for @obj.
1344 */
1345static int
1346i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1347{
1348 struct drm_device *dev = obj->dev;
1349 struct drm_gem_mm *mm = dev->mm_private;
23010e43 1350 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 1351 struct drm_map_list *list;
f77d390c 1352 struct drm_local_map *map;
de151cf6
JB		 1353 int ret = 0;
1354
1355 /* Set the object up for mmap'ing */
1356 list = &obj->map_list;
9a298b2a 1357 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1358 if (!list->map)
1359 return -ENOMEM;
1360
1361 map = list->map;
1362 map->type = _DRM_GEM;
1363 map->size = obj->size;
1364 map->handle = obj;
1365
1366 /* Get a DRM GEM mmap offset allocated... */
1367 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1368 obj->size / PAGE_SIZE, 0, 0);
1369 if (!list->file_offset_node) {
1370 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
9e0ae534 1371 ret = -ENOSPC;
de151cf6
JB		 1372 goto out_free_list;
1373 }
1374
1375 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1376 obj->size / PAGE_SIZE, 0);
1377 if (!list->file_offset_node) {
1378 ret = -ENOMEM;
1379 goto out_free_list;
1380 }
1381
1382 list->hash.key = list->file_offset_node->start;
9e0ae534
CW		 1383 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1384 if (ret) {
de151cf6
JB		 1385 DRM_ERROR("failed to add to map hash\n");
1386 goto out_free_mm;
1387 }
1388
1389 /* By now we should be all set, any drm_mmap request on the offset
1390 * below will get to our mmap & fault handler */
1391 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1392
1393 return 0;
1394
1395out_free_mm:
1396 drm_mm_put_block(list->file_offset_node);
1397out_free_list:
9a298b2a 1398 kfree(list->map);
de151cf6
JB		 1399
1400 return ret;
1401}
1402
901782b2
CW		 1403/**
1404 * i915_gem_release_mmap - remove physical page mappings
1405 * @obj: obj in question
1406 *
af901ca1 1407 * Preserve the reservation of the mmapping with the DRM core code, but
901782b2
CW		 1408 * relinquish ownership of the pages back to the system.
1409 *
1410 * It is vital that we remove the page mapping if we have mapped a tiled
1411 * object through the GTT and then lose the fence register due to
1412 * resource pressure. Similarly if the object has been moved out of the
1413 * aperture, than pages mapped into userspace must be revoked. Removing the
1414 * mapping will then trigger a page fault on the next user access, allowing
1415 * fixup by i915_gem_fault().
1416 */
d05ca301 1417void
901782b2
CW		 1418i915_gem_release_mmap(struct drm_gem_object *obj)
1419{
1420 struct drm_device *dev = obj->dev;
23010e43 1421 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
901782b2
CW		 1422
1423 if (dev->dev_mapping)
1424 unmap_mapping_range(dev->dev_mapping,
1425 obj_priv->mmap_offset, obj->size, 1);
1426}
1427
ab00b3e5
JB		 1428static void
1429i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1430{
1431 struct drm_device *dev = obj->dev;
23010e43 1432 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ab00b3e5
JB		 1433 struct drm_gem_mm *mm = dev->mm_private;
1434 struct drm_map_list *list;
1435
1436 list = &obj->map_list;
1437 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1438
1439 if (list->file_offset_node) {
1440 drm_mm_put_block(list->file_offset_node);
1441 list->file_offset_node = NULL;
1442 }
1443
1444 if (list->map) {
9a298b2a 1445 kfree(list->map);
ab00b3e5
JB		 1446 list->map = NULL;
1447 }
1448
1449 obj_priv->mmap_offset = 0;
1450}
1451
de151cf6
JB		 1452/**
1453 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1454 * @obj: object to check
1455 *
1456 * Return the required GTT alignment for an object, taking into account
1457 * potential fence register mapping if needed.
1458 */
1459static uint32_t
1460i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1461{
1462 struct drm_device *dev = obj->dev;
23010e43 1463 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 1464 int start, i;
1465
1466 /*
1467 * Minimum alignment is 4k (GTT page size), but might be greater
1468 * if a fence register is needed for the object.
1469 */
a6c45cf0 1470 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
de151cf6
JB		 1471 return 4096;
1472
1473 /*
1474 * Previous chips need to be aligned to the size of the smallest
1475 * fence register that can contain the object.
1476 */
a6c45cf0 1477 if (INTEL_INFO(dev)->gen == 3)
de151cf6
JB		 1478 start = 1024*1024;
1479 else
1480 start = 512*1024;
1481
1482 for (i = start; i < obj->size; i <<= 1)
1483 ;
1484
1485 return i;
1486}
1487
1488/**
1489 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1490 * @dev: DRM device
1491 * @data: GTT mapping ioctl data
1492 * @file_priv: GEM object info
1493 *
1494 * Simply returns the fake offset to userspace so it can mmap it.
1495 * The mmap call will end up in drm_gem_mmap(), which will set things
1496 * up so we can get faults in the handler above.
1497 *
1498 * The fault handler will take care of binding the object into the GTT
1499 * (since it may have been evicted to make room for something), allocating
1500 * a fence register, and mapping the appropriate aperture address into
1501 * userspace.
1502 */
1503int
1504i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1505 struct drm_file *file_priv)
1506{
1507 struct drm_i915_gem_mmap_gtt *args = data;
de151cf6
JB		 1508 struct drm_gem_object *obj;
1509 struct drm_i915_gem_object *obj_priv;
1510 int ret;
1511
1512 if (!(dev->driver->driver_features & DRIVER_GEM))
1513 return -ENODEV;
1514
1515 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1516 if (obj == NULL)
bf79cb91 1517 return -ENOENT;
de151cf6 1518
76c1dec1
CW		 1519 ret = i915_mutex_lock_interruptible(dev);
1520 if (ret) {
1521 drm_gem_object_unreference_unlocked(obj);
1522 return ret;
1523 }
de151cf6 1524
23010e43 1525 obj_priv = to_intel_bo(obj);
de151cf6 1526
ab18282d
CW		 1527 if (obj_priv->madv != I915_MADV_WILLNEED) {
1528 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1529 drm_gem_object_unreference(obj);
1530 mutex_unlock(&dev->struct_mutex);
1531 return -EINVAL;
1532 }
1533
1534
de151cf6
JB		 1535 if (!obj_priv->mmap_offset) {
1536 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1537 if (ret) {
1538 drm_gem_object_unreference(obj);
1539 mutex_unlock(&dev->struct_mutex);
de151cf6 1540 return ret;
13af1062 1541 }
de151cf6
JB		 1542 }
1543
1544 args->offset = obj_priv->mmap_offset;
1545
de151cf6
JB		 1546 /*
1547 * Pull it into the GTT so that we have a page list (makes the
1548 * initial fault faster and any subsequent flushing possible).
1549 */
1550 if (!obj_priv->agp_mem) {
e67b8ce1 1551 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1552 if (ret) {
1553 drm_gem_object_unreference(obj);
1554 mutex_unlock(&dev->struct_mutex);
1555 return ret;
1556 }
de151cf6
JB		 1557 }
1558
1559 drm_gem_object_unreference(obj);
1560 mutex_unlock(&dev->struct_mutex);
1561
1562 return 0;
1563}
1564
5cdf5881 1565static void
856fa198 1566i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b 1567{
23010e43 1568 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 1569 int page_count = obj->size / PAGE_SIZE;
1570 int i;
1571
856fa198 1572 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1573 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1574
856fa198
EA		 1575 if (--obj_priv->pages_refcount != 0)
1576 return;
673a394b 1577
280b713b
EA		 1578 if (obj_priv->tiling_mode != I915_TILING_NONE)
1579 i915_gem_object_save_bit_17_swizzle(obj);
1580
3ef94daa 1581 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1582 obj_priv->dirty = 0;
3ef94daa
CW		 1583
1584 for (i = 0; i < page_count; i++) {
3ef94daa
CW		 1585 if (obj_priv->dirty)
1586 set_page_dirty(obj_priv->pages[i]);
1587
1588 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1589 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1590
1591 page_cache_release(obj_priv->pages[i]);
1592 }
673a394b
EA		 1593 obj_priv->dirty = 0;
1594
8e7d2b2c 1595 drm_free_large(obj_priv->pages);
856fa198 1596 obj_priv->pages = NULL;
673a394b
EA		 1597}
1598
a56ba56c
CW		 1599static uint32_t
1600i915_gem_next_request_seqno(struct drm_device *dev,
1601 struct intel_ring_buffer *ring)
1602{
1603 drm_i915_private_t *dev_priv = dev->dev_private;
1604
1605 ring->outstanding_lazy_request = true;
1606 return dev_priv->next_seqno;
1607}
1608
673a394b 1609static void
617dbe27 1610i915_gem_object_move_to_active(struct drm_gem_object *obj,
852835f3 1611 struct intel_ring_buffer *ring)
673a394b 1612{
a56ba56c 1613 struct drm_device *dev = obj->dev;
23010e43 1614 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
a56ba56c 1615 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
617dbe27 1616
852835f3
ZN		 1617 BUG_ON(ring == NULL);
1618 obj_priv->ring = ring;
673a394b
EA		 1619
1620 /* Add a reference if we're newly entering the active list. */
1621 if (!obj_priv->active) {
1622 drm_gem_object_reference(obj);
1623 obj_priv->active = 1;
1624 }
e35a41de 1625
673a394b 1626 /* Move from whatever list we were on to the tail of execution. */
852835f3 1627 list_move_tail(&obj_priv->list, &ring->active_list);
a56ba56c 1628 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1629}
1630
ce44b0ea
EA		 1631static void
1632i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1633{
1634 struct drm_device *dev = obj->dev;
1635 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1636 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ce44b0ea
EA		 1637
1638 BUG_ON(!obj_priv->active);
1639 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1640 obj_priv->last_rendering_seqno = 0;
1641}
673a394b 1642
963b4836
CW		 1643/* Immediately discard the backing storage */
1644static void
1645i915_gem_object_truncate(struct drm_gem_object *obj)
1646{
23010e43 1647 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
bb6baf76 1648 struct inode *inode;
963b4836 1649
ae9fed6b
CW		 1650 /* Our goal here is to return as much of the memory as
1651 * is possible back to the system as we are called from OOM.
1652 * To do this we must instruct the shmfs to drop all of its
1653 * backing pages, *now*. Here we mirror the actions taken
1654 * when by shmem_delete_inode() to release the backing store.
1655 */
bb6baf76 1656 inode = obj->filp->f_path.dentry->d_inode;
ae9fed6b
CW		 1657 truncate_inode_pages(inode->i_mapping, 0);
1658 if (inode->i_op->truncate_range)
1659 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
bb6baf76
CW		 1660
1661 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1662}
1663
1664static inline int
1665i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1666{
1667 return obj_priv->madv == I915_MADV_DONTNEED;
1668}
1669
673a394b
EA		 1670static void
1671i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1672{
1673 struct drm_device *dev = obj->dev;
1674 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 1675 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 1676
673a394b 1677 if (obj_priv->pin_count != 0)
f13d3f73 1678 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
673a394b
EA		 1679 else
1680 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1681
99fcb766
DV		 1682 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1683
ce44b0ea 1684 obj_priv->last_rendering_seqno = 0;
852835f3 1685 obj_priv->ring = NULL;
673a394b
EA		 1686 if (obj_priv->active) {
1687 obj_priv->active = 0;
1688 drm_gem_object_unreference(obj);
1689 }
23bc5982 1690 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1691}
1692
9220434a 1693static void
63560396 1694i915_gem_process_flushing_list(struct drm_device *dev,
8a1a49f9 1695 uint32_t flush_domains,
852835f3 1696 struct intel_ring_buffer *ring)
63560396
DV		 1697{
1698 drm_i915_private_t *dev_priv = dev->dev_private;
1699 struct drm_i915_gem_object *obj_priv, *next;
1700
1701 list_for_each_entry_safe(obj_priv, next,
1702 &dev_priv->mm.gpu_write_list,
1703 gpu_write_list) {
a8089e84 1704 struct drm_gem_object *obj = &obj_priv->base;
63560396 1705
2b6efaa4
CW		 1706 if (obj->write_domain & flush_domains &&
1707 obj_priv->ring == ring) {
63560396
DV		 1708 uint32_t old_write_domain = obj->write_domain;
1709
1710 obj->write_domain = 0;
1711 list_del_init(&obj_priv->gpu_write_list);
617dbe27 1712 i915_gem_object_move_to_active(obj, ring);
63560396
DV		 1713
1714 /* update the fence lru list */
007cc8ac
DV		 1715 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1716 struct drm_i915_fence_reg *reg =
1717 &dev_priv->fence_regs[obj_priv->fence_reg];
1718 list_move_tail(&reg->lru_list,
63560396 1719 &dev_priv->mm.fence_list);
007cc8ac 1720 }
63560396
DV		 1721
1722 trace_i915_gem_object_change_domain(obj,
1723 obj->read_domains,
1724 old_write_domain);
1725 }
1726 }
1727}
8187a2b7 1728
5a5a0c64 1729uint32_t
8a1a49f9 1730i915_add_request(struct drm_device *dev,
f787a5f5 1731 struct drm_file *file,
8dc5d147 1732 struct drm_i915_gem_request *request,
8a1a49f9 1733 struct intel_ring_buffer *ring)
673a394b
EA		 1734{
1735 drm_i915_private_t *dev_priv = dev->dev_private;
f787a5f5 1736 struct drm_i915_file_private *file_priv = NULL;
673a394b
EA		 1737 uint32_t seqno;
1738 int was_empty;
673a394b 1739
f787a5f5
CW		 1740 if (file != NULL)
1741 file_priv = file->driver_priv;
b962442e 1742
8dc5d147
CW		 1743 if (request == NULL) {
1744 request = kzalloc(sizeof(*request), GFP_KERNEL);
1745 if (request == NULL)
1746 return 0;
1747 }
673a394b 1748
f787a5f5 1749 seqno = ring->add_request(dev, ring, 0);
a56ba56c 1750 ring->outstanding_lazy_request = false;
673a394b
EA		 1751
1752 request->seqno = seqno;
852835f3 1753 request->ring = ring;
673a394b 1754 request->emitted_jiffies = jiffies;
852835f3
ZN		 1755 was_empty = list_empty(&ring->request_list);
1756 list_add_tail(&request->list, &ring->request_list);
1757
f787a5f5 1758 if (file_priv) {
1c25595f 1759 spin_lock(&file_priv->mm.lock);
f787a5f5 1760 request->file_priv = file_priv;
b962442e 1761 list_add_tail(&request->client_list,
f787a5f5 1762 &file_priv->mm.request_list);
1c25595f 1763 spin_unlock(&file_priv->mm.lock);
b962442e 1764 }
673a394b 1765
f65d9421 1766 if (!dev_priv->mm.suspended) {
b3b079db
CW		 1767 mod_timer(&dev_priv->hangcheck_timer,
1768 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
f65d9421 1769 if (was_empty)
b3b079db
CW		 1770 queue_delayed_work(dev_priv->wq,
1771 &dev_priv->mm.retire_work, HZ);
f65d9421 1772 }
673a394b
EA		 1773 return seqno;
1774}
1775
1776/**
1777 * Command execution barrier
1778 *
1779 * Ensures that all commands in the ring are finished
1780 * before signalling the CPU
1781 */
8a1a49f9 1782static void
852835f3 1783i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
673a394b 1784{
673a394b 1785 uint32_t flush_domains = 0;
673a394b
EA		 1786
1787 /* The sampler always gets flushed on i965 (sigh) */
a6c45cf0 1788 if (INTEL_INFO(dev)->gen >= 4)
673a394b 1789 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
852835f3
ZN		 1790
1791 ring->flush(dev, ring,
1792 I915_GEM_DOMAIN_COMMAND, flush_domains);
673a394b
EA		 1793}
1794
f787a5f5
CW		 1795static inline void
1796i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
673a394b 1797{
1c25595f
CW		 1798 struct drm_i915_file_private *file_priv = request->file_priv;
1799
1800 if (!file_priv)
1801 return;
1802
1803 spin_lock(&file_priv->mm.lock);
1804 list_del(&request->client_list);
1805 request->file_priv = NULL;
1806 spin_unlock(&file_priv->mm.lock);
673a394b
EA		 1807}
1808
dfaae392
CW		 1809static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1810 struct intel_ring_buffer *ring)
9375e446 1811{
dfaae392
CW		 1812 while (!list_empty(&ring->request_list)) {
1813 struct drm_i915_gem_request *request;
9375e446 1814
dfaae392
CW		 1815 request = list_first_entry(&ring->request_list,
1816 struct drm_i915_gem_request,
1817 list);
1818
1819 list_del(&request->list);
f787a5f5 1820 i915_gem_request_remove_from_client(request);
dfaae392
CW		 1821 kfree(request);
1822 }
1823
1824 while (!list_empty(&ring->active_list)) {
9375e446
CW		 1825 struct drm_i915_gem_object *obj_priv;
1826
dfaae392 1827 obj_priv = list_first_entry(&ring->active_list,
9375e446
CW		 1828 struct drm_i915_gem_object,
1829 list);
1830
1831 obj_priv->base.write_domain = 0;
dfaae392 1832 list_del_init(&obj_priv->gpu_write_list);
9375e446
CW		 1833 i915_gem_object_move_to_inactive(&obj_priv->base);
1834 }
1835}
1836
069efc1d 1837void i915_gem_reset(struct drm_device *dev)
77f01230
CW		 1838{
1839 struct drm_i915_private *dev_priv = dev->dev_private;
1840 struct drm_i915_gem_object *obj_priv;
069efc1d 1841 int i;
77f01230 1842
dfaae392
CW		 1843 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1844 if (HAS_BSD(dev))
1845 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1846
1847 /* Remove anything from the flushing lists. The GPU cache is likely
1848 * to be lost on reset along with the data, so simply move the
1849 * lost bo to the inactive list.
1850 */
1851 while (!list_empty(&dev_priv->mm.flushing_list)) {
1852 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1853 struct drm_i915_gem_object,
1854 list);
1855
1856 obj_priv->base.write_domain = 0;
1857 list_del_init(&obj_priv->gpu_write_list);
1858 i915_gem_object_move_to_inactive(&obj_priv->base);
1859 }
1860
1861 /* Move everything out of the GPU domains to ensure we do any
1862 * necessary invalidation upon reuse.
1863 */
77f01230
CW		 1864 list_for_each_entry(obj_priv,
1865 &dev_priv->mm.inactive_list,
1866 list)
1867 {
1868 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1869 }
069efc1d
CW		 1870
1871 /* The fence registers are invalidated so clear them out */
1872 for (i = 0; i < 16; i++) {
1873 struct drm_i915_fence_reg *reg;
1874
1875 reg = &dev_priv->fence_regs[i];
1876 if (!reg->obj)
1877 continue;
1878
1879 i915_gem_clear_fence_reg(reg->obj);
1880 }
77f01230
CW		 1881}
1882
673a394b
EA		 1883/**
1884 * This function clears the request list as sequence numbers are passed.
1885 */
b09a1fec
CW		 1886static void
1887i915_gem_retire_requests_ring(struct drm_device *dev,
1888 struct intel_ring_buffer *ring)
673a394b
EA		 1889{
1890 drm_i915_private_t *dev_priv = dev->dev_private;
1891 uint32_t seqno;
1892
b84d5f0c
CW		 1893 if (!ring->status_page.page_addr ||
1894 list_empty(&ring->request_list))
6c0594a3
KW		 1895 return;
1896
23bc5982
CW		 1897 WARN_ON(i915_verify_lists(dev));
1898
f787a5f5 1899 seqno = ring->get_seqno(dev, ring);
852835f3 1900 while (!list_empty(&ring->request_list)) {
673a394b 1901 struct drm_i915_gem_request *request;
673a394b 1902
852835f3 1903 request = list_first_entry(&ring->request_list,
673a394b
EA		 1904 struct drm_i915_gem_request,
1905 list);
673a394b 1906
dfaae392 1907 if (!i915_seqno_passed(seqno, request->seqno))
b84d5f0c
CW		 1908 break;
1909
1910 trace_i915_gem_request_retire(dev, request->seqno);
1911
1912 list_del(&request->list);
f787a5f5 1913 i915_gem_request_remove_from_client(request);
b84d5f0c
CW		 1914 kfree(request);
1915 }
1916
1917 /* Move any buffers on the active list that are no longer referenced
1918 * by the ringbuffer to the flushing/inactive lists as appropriate.
1919 */
1920 while (!list_empty(&ring->active_list)) {
1921 struct drm_gem_object *obj;
1922 struct drm_i915_gem_object *obj_priv;
1923
1924 obj_priv = list_first_entry(&ring->active_list,
1925 struct drm_i915_gem_object,
1926 list);
673a394b 1927
dfaae392 1928 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
673a394b 1929 break;
b84d5f0c
CW		 1930
1931 obj = &obj_priv->base;
b84d5f0c
CW		 1932 if (obj->write_domain != 0)
1933 i915_gem_object_move_to_flushing(obj);
1934 else
1935 i915_gem_object_move_to_inactive(obj);
673a394b 1936 }
9d34e5db
CW		 1937
1938 if (unlikely (dev_priv->trace_irq_seqno &&
1939 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
8187a2b7 1940 ring->user_irq_put(dev, ring);
9d34e5db
CW		 1941 dev_priv->trace_irq_seqno = 0;
1942 }
23bc5982
CW		 1943
1944 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 1945}
1946
b09a1fec
CW		 1947void
1948i915_gem_retire_requests(struct drm_device *dev)
1949{
1950 drm_i915_private_t *dev_priv = dev->dev_private;
1951
be72615b
CW		 1952 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1953 struct drm_i915_gem_object *obj_priv, *tmp;
1954
1955 /* We must be careful that during unbind() we do not
1956 * accidentally infinitely recurse into retire requests.
1957 * Currently:
1958 * retire -> free -> unbind -> wait -> retire_ring
1959 */
1960 list_for_each_entry_safe(obj_priv, tmp,
1961 &dev_priv->mm.deferred_free_list,
1962 list)
1963 i915_gem_free_object_tail(&obj_priv->base);
1964 }
1965
b09a1fec
CW		 1966 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1967 if (HAS_BSD(dev))
1968 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1969}
1970
75ef9da2 1971static void
673a394b
EA		 1972i915_gem_retire_work_handler(struct work_struct *work)
1973{
1974 drm_i915_private_t *dev_priv;
1975 struct drm_device *dev;
1976
1977 dev_priv = container_of(work, drm_i915_private_t,
1978 mm.retire_work.work);
1979 dev = dev_priv->dev;
1980
891b48cf
CW		 1981 /* Come back later if the device is busy... */
1982 if (!mutex_trylock(&dev->struct_mutex)) {
1983 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1984 return;
1985 }
1986
b09a1fec 1987 i915_gem_retire_requests(dev);
d1b851fc 1988
6dbe2772 1989 if (!dev_priv->mm.suspended &&
d1b851fc
ZN		 1990 (!list_empty(&dev_priv->render_ring.request_list) ||
1991 (HAS_BSD(dev) &&
1992 !list_empty(&dev_priv->bsd_ring.request_list))))
9c9fe1f8 1993 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1994 mutex_unlock(&dev->struct_mutex);
1995}
1996
5a5a0c64 1997int
852835f3 1998i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
8a1a49f9 1999 bool interruptible, struct intel_ring_buffer *ring)
673a394b
EA		 2000{
2001 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 2002 u32 ier;
673a394b
EA		 2003 int ret = 0;
2004
2005 BUG_ON(seqno == 0);
2006
30dbf0c0
CW		 2007 if (atomic_read(&dev_priv->mm.wedged))
2008 return -EAGAIN;
2009
a56ba56c 2010 if (ring->outstanding_lazy_request) {
8dc5d147 2011 seqno = i915_add_request(dev, NULL, NULL, ring);
e35a41de
DV		 2012 if (seqno == 0)
2013 return -ENOMEM;
2014 }
a56ba56c 2015 BUG_ON(seqno == dev_priv->next_seqno);
e35a41de 2016
f787a5f5 2017 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
bad720ff 2018 if (HAS_PCH_SPLIT(dev))
036a4a7d
ZW		 2019 ier = I915_READ(DEIER) | I915_READ(GTIER);
2020 else
2021 ier = I915_READ(IER);
802c7eb6
JB		 2022 if (!ier) {
2023 DRM_ERROR("something (likely vbetool) disabled "
2024 "interrupts, re-enabling\n");
2025 i915_driver_irq_preinstall(dev);
2026 i915_driver_irq_postinstall(dev);
2027 }
2028
1c5d22f7
CW		 2029 trace_i915_gem_request_wait_begin(dev, seqno);
2030
852835f3 2031 ring->waiting_gem_seqno = seqno;
8187a2b7 2032 ring->user_irq_get(dev, ring);
48764bf4 2033 if (interruptible)
852835f3
ZN		 2034 ret = wait_event_interruptible(ring->irq_queue,
2035 i915_seqno_passed(
f787a5f5 2036 ring->get_seqno(dev, ring), seqno)
852835f3 2037 || atomic_read(&dev_priv->mm.wedged));
48764bf4 2038 else
852835f3
ZN		 2039 wait_event(ring->irq_queue,
2040 i915_seqno_passed(
f787a5f5 2041 ring->get_seqno(dev, ring), seqno)
852835f3 2042 || atomic_read(&dev_priv->mm.wedged));
48764bf4 2043
8187a2b7 2044 ring->user_irq_put(dev, ring);
852835f3 2045 ring->waiting_gem_seqno = 0;
1c5d22f7
CW		 2046
2047 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 2048 }
ba1234d1 2049 if (atomic_read(&dev_priv->mm.wedged))
30dbf0c0 2050 ret = -EAGAIN;
673a394b
EA		 2051
2052 if (ret && ret != -ERESTARTSYS)
8bff917c 2053 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
f787a5f5 2054 __func__, ret, seqno, ring->get_seqno(dev, ring),
8bff917c 2055 dev_priv->next_seqno);
673a394b
EA		 2056
2057 /* Directly dispatch request retiring. While we have the work queue
2058 * to handle this, the waiter on a request often wants an associated
2059 * buffer to have made it to the inactive list, and we would need
2060 * a separate wait queue to handle that.
2061 */
2062 if (ret == 0)
b09a1fec 2063 i915_gem_retire_requests_ring(dev, ring);
673a394b
EA		 2064
2065 return ret;
2066}
2067
48764bf4
DV		 2068/**
2069 * Waits for a sequence number to be signaled, and cleans up the
2070 * request and object lists appropriately for that event.
2071 */
2072static int
852835f3 2073i915_wait_request(struct drm_device *dev, uint32_t seqno,
a56ba56c 2074 struct intel_ring_buffer *ring)
48764bf4 2075{
852835f3 2076 return i915_do_wait_request(dev, seqno, 1, ring);
48764bf4
DV		 2077}
2078
20f0cd55 2079static void
9220434a 2080i915_gem_flush_ring(struct drm_device *dev,
c78ec30b 2081 struct drm_file *file_priv,
9220434a
CW		 2082 struct intel_ring_buffer *ring,
2083 uint32_t invalidate_domains,
2084 uint32_t flush_domains)
2085{
2086 ring->flush(dev, ring, invalidate_domains, flush_domains);
2087 i915_gem_process_flushing_list(dev, flush_domains, ring);
2088}
2089
8187a2b7
ZN		 2090static void
2091i915_gem_flush(struct drm_device *dev,
c78ec30b 2092 struct drm_file *file_priv,
8187a2b7 2093 uint32_t invalidate_domains,
9220434a
CW		 2094 uint32_t flush_domains,
2095 uint32_t flush_rings)
8187a2b7
ZN		 2096{
2097 drm_i915_private_t *dev_priv = dev->dev_private;
8bff917c 2098
8187a2b7
ZN		 2099 if (flush_domains & I915_GEM_DOMAIN_CPU)
2100 drm_agp_chipset_flush(dev);
8bff917c 2101
9220434a
CW		 2102 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
2103 if (flush_rings & RING_RENDER)
c78ec30b 2104 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2105 &dev_priv->render_ring,
2106 invalidate_domains, flush_domains);
2107 if (flush_rings & RING_BSD)
c78ec30b 2108 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 2109 &dev_priv->bsd_ring,
2110 invalidate_domains, flush_domains);
2111 }
8187a2b7
ZN		 2112}
2113
673a394b
EA		 2114/**
2115 * Ensures that all rendering to the object has completed and the object is
2116 * safe to unbind from the GTT or access from the CPU.
2117 */
2118static int
2cf34d7b
CW		 2119i915_gem_object_wait_rendering(struct drm_gem_object *obj,
2120 bool interruptible)
673a394b
EA		 2121{
2122 struct drm_device *dev = obj->dev;
23010e43 2123 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2124 int ret;
2125
e47c68e9
EA		 2126 /* This function only exists to support waiting for existing rendering,
2127 * not for emitting required flushes.
673a394b 2128 */
e47c68e9 2129 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 2130
2131 /* If there is rendering queued on the buffer being evicted, wait for
2132 * it.
2133 */
2134 if (obj_priv->active) {
2cf34d7b
CW		 2135 ret = i915_do_wait_request(dev,
2136 obj_priv->last_rendering_seqno,
2137 interruptible,
2138 obj_priv->ring);
2139 if (ret)
673a394b
EA		 2140 return ret;
2141 }
2142
2143 return 0;
2144}
2145
2146/**
2147 * Unbinds an object from the GTT aperture.
2148 */
0f973f27 2149int
673a394b
EA		 2150i915_gem_object_unbind(struct drm_gem_object *obj)
2151{
2152 struct drm_device *dev = obj->dev;
73aa808f 2153 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2154 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2155 int ret = 0;
2156
673a394b
EA		 2157 if (obj_priv->gtt_space == NULL)
2158 return 0;
2159
2160 if (obj_priv->pin_count != 0) {
2161 DRM_ERROR("Attempting to unbind pinned buffer\n");
2162 return -EINVAL;
2163 }
2164
5323fd04
EA		 2165 /* blow away mappings if mapped through GTT */
2166 i915_gem_release_mmap(obj);
2167
673a394b
EA		 2168 /* Move the object to the CPU domain to ensure that
2169 * any possible CPU writes while it's not in the GTT
2170 * are flushed when we go to remap it. This will
2171 * also ensure that all pending GPU writes are finished
2172 * before we unbind.
2173 */
e47c68e9 2174 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
8dc1775d 2175 if (ret == -ERESTARTSYS)
673a394b 2176 return ret;
8dc1775d
CW		 2177 /* Continue on if we fail due to EIO, the GPU is hung so we
2178 * should be safe and we need to cleanup or else we might
2179 * cause memory corruption through use-after-free.
2180 */
812ed492
CW		 2181 if (ret) {
2182 i915_gem_clflush_object(obj);
2183 obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU;
2184 }
673a394b 2185
96b47b65
DV		 2186 /* release the fence reg _after_ flushing */
2187 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2188 i915_gem_clear_fence_reg(obj);
2189
73aa808f
CW		 2190 drm_unbind_agp(obj_priv->agp_mem);
2191 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
673a394b 2192
856fa198 2193 i915_gem_object_put_pages(obj);
a32808c0 2194 BUG_ON(obj_priv->pages_refcount);
673a394b 2195
73aa808f 2196 i915_gem_info_remove_gtt(dev_priv, obj->size);
f13d3f73 2197 list_del_init(&obj_priv->list);
673a394b 2198
73aa808f
CW		 2199 drm_mm_put_block(obj_priv->gtt_space);
2200 obj_priv->gtt_space = NULL;
2201
963b4836
CW		 2202 if (i915_gem_object_is_purgeable(obj_priv))
2203 i915_gem_object_truncate(obj);
2204
1c5d22f7
CW		 2205 trace_i915_gem_object_unbind(obj);
2206
8dc1775d 2207 return ret;
673a394b
EA		 2208}
2209
a56ba56c
CW		 2210static int i915_ring_idle(struct drm_device *dev,
2211 struct intel_ring_buffer *ring)
2212{
2213 i915_gem_flush_ring(dev, NULL, ring,
2214 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2215 return i915_wait_request(dev,
2216 i915_gem_next_request_seqno(dev, ring),
2217 ring);
2218}
2219
b47eb4a2 2220int
4df2faf4
DV		 2221i915_gpu_idle(struct drm_device *dev)
2222{
2223 drm_i915_private_t *dev_priv = dev->dev_private;
2224 bool lists_empty;
852835f3 2225 int ret;
4df2faf4 2226
d1b851fc
ZN		 2227 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2228 list_empty(&dev_priv->render_ring.active_list) &&
2229 (!HAS_BSD(dev) ||
2230 list_empty(&dev_priv->bsd_ring.active_list)));
4df2faf4
DV		 2231 if (lists_empty)
2232 return 0;
2233
2234 /* Flush everything onto the inactive list. */
a56ba56c 2235 ret = i915_ring_idle(dev, &dev_priv->render_ring);
8a1a49f9
DV		 2236 if (ret)
2237 return ret;
d1b851fc
ZN		 2238
2239 if (HAS_BSD(dev)) {
a56ba56c 2240 ret = i915_ring_idle(dev, &dev_priv->bsd_ring);
d1b851fc
ZN		 2241 if (ret)
2242 return ret;
2243 }
2244
8a1a49f9 2245 return 0;
4df2faf4
DV		 2246}
2247
5cdf5881 2248static int
4bdadb97
CW		 2249i915_gem_object_get_pages(struct drm_gem_object *obj,
2250 gfp_t gfpmask)
673a394b 2251{
23010e43 2252 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2253 int page_count, i;
2254 struct address_space *mapping;
2255 struct inode *inode;
2256 struct page *page;
673a394b 2257
778c3544
DV		 2258 BUG_ON(obj_priv->pages_refcount
2259 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2260
856fa198 2261 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2262 return 0;
2263
2264 /* Get the list of pages out of our struct file. They'll be pinned
2265 * at this point until we release them.
2266 */
2267 page_count = obj->size / PAGE_SIZE;
856fa198 2268 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2269 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2270 if (obj_priv->pages == NULL) {
856fa198 2271 obj_priv->pages_refcount--;
673a394b
EA		 2272 return -ENOMEM;
2273 }
2274
2275 inode = obj->filp->f_path.dentry->d_inode;
2276 mapping = inode->i_mapping;
2277 for (i = 0; i < page_count; i++) {
4bdadb97 2278 page = read_cache_page_gfp(mapping, i,
985b823b 2279 GFP_HIGHUSER |
4bdadb97 2280 __GFP_COLD |
cd9f040d 2281 __GFP_RECLAIMABLE |
4bdadb97 2282 gfpmask);
1f2b1013
CW		 2283 if (IS_ERR(page))
2284 goto err_pages;
2285
856fa198 2286 obj_priv->pages[i] = page;
673a394b 2287 }
280b713b
EA		 2288
2289 if (obj_priv->tiling_mode != I915_TILING_NONE)
2290 i915_gem_object_do_bit_17_swizzle(obj);
2291
673a394b 2292 return 0;
1f2b1013
CW		 2293
2294err_pages:
2295 while (i--)
2296 page_cache_release(obj_priv->pages[i]);
2297
2298 drm_free_large(obj_priv->pages);
2299 obj_priv->pages = NULL;
2300 obj_priv->pages_refcount--;
2301 return PTR_ERR(page);
673a394b
EA		 2302}
2303
4e901fdc
EA		 2304static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2305{
2306 struct drm_gem_object *obj = reg->obj;
2307 struct drm_device *dev = obj->dev;
2308 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2309 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4e901fdc
EA		 2310 int regnum = obj_priv->fence_reg;
2311 uint64_t val;
2312
2313 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2314 0xfffff000) << 32;
2315 val |= obj_priv->gtt_offset & 0xfffff000;
2316 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2317 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2318
2319 if (obj_priv->tiling_mode == I915_TILING_Y)
2320 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2321 val |= I965_FENCE_REG_VALID;
2322
2323 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2324}
2325
de151cf6
JB		 2326static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2327{
2328 struct drm_gem_object *obj = reg->obj;
2329 struct drm_device *dev = obj->dev;
2330 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2331 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2332 int regnum = obj_priv->fence_reg;
2333 uint64_t val;
2334
2335 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2336 0xfffff000) << 32;
2337 val |= obj_priv->gtt_offset & 0xfffff000;
2338 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2339 if (obj_priv->tiling_mode == I915_TILING_Y)
2340 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2341 val |= I965_FENCE_REG_VALID;
2342
2343 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2344}
2345
2346static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2347{
2348 struct drm_gem_object *obj = reg->obj;
2349 struct drm_device *dev = obj->dev;
2350 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2351 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2352 int regnum = obj_priv->fence_reg;
0f973f27 2353 int tile_width;
dc529a4f 2354 uint32_t fence_reg, val;
de151cf6
JB		 2355 uint32_t pitch_val;
2356
2357 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2358 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2359 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2360 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2361 return;
2362 }
2363
0f973f27
JB		 2364 if (obj_priv->tiling_mode == I915_TILING_Y &&
2365 HAS_128_BYTE_Y_TILING(dev))
2366 tile_width = 128;
de151cf6 2367 else
0f973f27
JB		 2368 tile_width = 512;
2369
2370 /* Note: pitch better be a power of two tile widths */
2371 pitch_val = obj_priv->stride / tile_width;
2372 pitch_val = ffs(pitch_val) - 1;
de151cf6 2373
c36a2a6d
DV		 2374 if (obj_priv->tiling_mode == I915_TILING_Y &&
2375 HAS_128_BYTE_Y_TILING(dev))
2376 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2377 else
2378 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2379
de151cf6
JB		 2380 val = obj_priv->gtt_offset;
2381 if (obj_priv->tiling_mode == I915_TILING_Y)
2382 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2383 val |= I915_FENCE_SIZE_BITS(obj->size);
2384 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2385 val |= I830_FENCE_REG_VALID;
2386
dc529a4f
EA		 2387 if (regnum < 8)
2388 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2389 else
2390 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2391 I915_WRITE(fence_reg, val);
de151cf6
JB		 2392}
2393
2394static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2395{
2396 struct drm_gem_object *obj = reg->obj;
2397 struct drm_device *dev = obj->dev;
2398 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2399 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6
JB		 2400 int regnum = obj_priv->fence_reg;
2401 uint32_t val;
2402 uint32_t pitch_val;
8d7773a3 2403 uint32_t fence_size_bits;
de151cf6 2404
8d7773a3 2405 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2406 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2407 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2408 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2409 return;
2410 }
2411
e76a16de
EA		 2412 pitch_val = obj_priv->stride / 128;
2413 pitch_val = ffs(pitch_val) - 1;
2414 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2415
de151cf6
JB		 2416 val = obj_priv->gtt_offset;
2417 if (obj_priv->tiling_mode == I915_TILING_Y)
2418 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2419 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2420 WARN_ON(fence_size_bits & ~0x00000f00);
2421 val |= fence_size_bits;
de151cf6
JB		 2422 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2423 val |= I830_FENCE_REG_VALID;
2424
2425 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2426}
2427
2cf34d7b
CW		 2428static int i915_find_fence_reg(struct drm_device *dev,
2429 bool interruptible)
ae3db24a
DV		 2430{
2431 struct drm_i915_fence_reg *reg = NULL;
2432 struct drm_i915_gem_object *obj_priv = NULL;
2433 struct drm_i915_private *dev_priv = dev->dev_private;
2434 struct drm_gem_object *obj = NULL;
2435 int i, avail, ret;
2436
2437 /* First try to find a free reg */
2438 avail = 0;
2439 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2440 reg = &dev_priv->fence_regs[i];
2441 if (!reg->obj)
2442 return i;
2443
23010e43 2444 obj_priv = to_intel_bo(reg->obj);
ae3db24a
DV		 2445 if (!obj_priv->pin_count)
2446 avail++;
2447 }
2448
2449 if (avail == 0)
2450 return -ENOSPC;
2451
2452 /* None available, try to steal one or wait for a user to finish */
2453 i = I915_FENCE_REG_NONE;
007cc8ac
DV		 2454 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2455 lru_list) {
2456 obj = reg->obj;
2457 obj_priv = to_intel_bo(obj);
ae3db24a
DV		 2458
2459 if (obj_priv->pin_count)
2460 continue;
2461
2462 /* found one! */
2463 i = obj_priv->fence_reg;
2464 break;
2465 }
2466
2467 BUG_ON(i == I915_FENCE_REG_NONE);
2468
2469 /* We only have a reference on obj from the active list. put_fence_reg
2470 * might drop that one, causing a use-after-free in it. So hold a
2471 * private reference to obj like the other callers of put_fence_reg
2472 * (set_tiling ioctl) do. */
2473 drm_gem_object_reference(obj);
2cf34d7b 2474 ret = i915_gem_object_put_fence_reg(obj, interruptible);
ae3db24a
DV		 2475 drm_gem_object_unreference(obj);
2476 if (ret != 0)
2477 return ret;
2478
2479 return i;
2480}
2481
de151cf6
JB		 2482/**
2483 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2484 * @obj: object to map through a fence reg
2485 *
2486 * When mapping objects through the GTT, userspace wants to be able to write
2487 * to them without having to worry about swizzling if the object is tiled.
2488 *
2489 * This function walks the fence regs looking for a free one for @obj,
2490 * stealing one if it can't find any.
2491 *
2492 * It then sets up the reg based on the object's properties: address, pitch
2493 * and tiling format.
2494 */
8c4b8c3f 2495int
2cf34d7b
CW		 2496i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2497 bool interruptible)
de151cf6
JB		 2498{
2499 struct drm_device *dev = obj->dev;
79e53945 2500 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2501 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
de151cf6 2502 struct drm_i915_fence_reg *reg = NULL;
ae3db24a 2503 int ret;
de151cf6 2504
a09ba7fa
EA		 2505 /* Just update our place in the LRU if our fence is getting used. */
2506 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
007cc8ac
DV		 2507 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2508 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa
EA		 2509 return 0;
2510 }
2511
de151cf6
JB		 2512 switch (obj_priv->tiling_mode) {
2513 case I915_TILING_NONE:
2514 WARN(1, "allocating a fence for non-tiled object?\n");
2515 break;
2516 case I915_TILING_X:
0f973f27
JB		 2517 if (!obj_priv->stride)
2518 return -EINVAL;
2519 WARN((obj_priv->stride & (512 - 1)),
2520 "object 0x%08x is X tiled but has non-512B pitch\n",
2521 obj_priv->gtt_offset);
de151cf6
JB		 2522 break;
2523 case I915_TILING_Y:
0f973f27
JB		 2524 if (!obj_priv->stride)
2525 return -EINVAL;
2526 WARN((obj_priv->stride & (128 - 1)),
2527 "object 0x%08x is Y tiled but has non-128B pitch\n",
2528 obj_priv->gtt_offset);
de151cf6
JB		 2529 break;
2530 }
2531
2cf34d7b 2532 ret = i915_find_fence_reg(dev, interruptible);
ae3db24a
DV		 2533 if (ret < 0)
2534 return ret;
de151cf6 2535
ae3db24a
DV		 2536 obj_priv->fence_reg = ret;
2537 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
007cc8ac 2538 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
a09ba7fa 2539
de151cf6
JB		 2540 reg->obj = obj;
2541
e259befd
CW		 2542 switch (INTEL_INFO(dev)->gen) {
2543 case 6:
4e901fdc 2544 sandybridge_write_fence_reg(reg);
e259befd
CW		 2545 break;
2546 case 5:
2547 case 4:
de151cf6 2548 i965_write_fence_reg(reg);
e259befd
CW		 2549 break;
2550 case 3:
de151cf6 2551 i915_write_fence_reg(reg);
e259befd
CW		 2552 break;
2553 case 2:
de151cf6 2554 i830_write_fence_reg(reg);
e259befd
CW		 2555 break;
2556 }
d9ddcb96 2557
ae3db24a
DV		 2558 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2559 obj_priv->tiling_mode);
1c5d22f7 2560
d9ddcb96 2561 return 0;
de151cf6
JB		 2562}
2563
2564/**
2565 * i915_gem_clear_fence_reg - clear out fence register info
2566 * @obj: object to clear
2567 *
2568 * Zeroes out the fence register itself and clears out the associated
2569 * data structures in dev_priv and obj_priv.
2570 */
2571static void
2572i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2573{
2574 struct drm_device *dev = obj->dev;
79e53945 2575 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2576 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
007cc8ac
DV		 2577 struct drm_i915_fence_reg *reg =
2578 &dev_priv->fence_regs[obj_priv->fence_reg];
e259befd 2579 uint32_t fence_reg;
de151cf6 2580
e259befd
CW		 2581 switch (INTEL_INFO(dev)->gen) {
2582 case 6:
4e901fdc
EA		 2583 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2584 (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2585 break;
2586 case 5:
2587 case 4:
de151cf6 2588 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
e259befd
CW		 2589 break;
2590 case 3:
9b74f734 2591 if (obj_priv->fence_reg >= 8)
e259befd 2592 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
dc529a4f 2593 else
e259befd
CW		 2594 case 2:
2595 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
dc529a4f
EA		 2596
2597 I915_WRITE(fence_reg, 0);
e259befd 2598 break;
dc529a4f 2599 }
de151cf6 2600
007cc8ac 2601 reg->obj = NULL;
de151cf6 2602 obj_priv->fence_reg = I915_FENCE_REG_NONE;
007cc8ac 2603 list_del_init(&reg->lru_list);
de151cf6
JB		 2604}
2605
52dc7d32
CW		 2606/**
2607 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2608 * to the buffer to finish, and then resets the fence register.
2609 * @obj: tiled object holding a fence register.
2cf34d7b 2610 * @bool: whether the wait upon the fence is interruptible
52dc7d32
CW		 2611 *
2612 * Zeroes out the fence register itself and clears out the associated
2613 * data structures in dev_priv and obj_priv.
2614 */
2615int
2cf34d7b
CW		 2616i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2617 bool interruptible)
52dc7d32
CW		 2618{
2619 struct drm_device *dev = obj->dev;
53640e1d 2620 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 2621 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
53640e1d 2622 struct drm_i915_fence_reg *reg;
52dc7d32
CW		 2623
2624 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2625 return 0;
2626
10ae9bd2
DV		 2627 /* If we've changed tiling, GTT-mappings of the object
2628 * need to re-fault to ensure that the correct fence register
2629 * setup is in place.
2630 */
2631 i915_gem_release_mmap(obj);
2632
52dc7d32
CW		 2633 /* On the i915, GPU access to tiled buffers is via a fence,
2634 * therefore we must wait for any outstanding access to complete
2635 * before clearing the fence.
2636 */
53640e1d
CW		 2637 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2638 if (reg->gpu) {
52dc7d32
CW		 2639 int ret;
2640
2cf34d7b 2641 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
0bc23aad
CW		 2642 if (ret)
2643 return ret;
2644
2cf34d7b 2645 ret = i915_gem_object_wait_rendering(obj, interruptible);
0bc23aad 2646 if (ret)
52dc7d32 2647 return ret;
53640e1d
CW		 2648
2649 reg->gpu = false;
52dc7d32
CW		 2650 }
2651
4a726612 2652 i915_gem_object_flush_gtt_write_domain(obj);
0bc23aad 2653 i915_gem_clear_fence_reg(obj);
52dc7d32
CW		 2654
2655 return 0;
2656}
2657
673a394b
EA		 2658/**
2659 * Finds free space in the GTT aperture and binds the object there.
2660 */
2661static int
2662i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2663{
2664 struct drm_device *dev = obj->dev;
2665 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 2666 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 2667 struct drm_mm_node *free_space;
4bdadb97 2668 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
07f73f69 2669 int ret;
673a394b 2670
bb6baf76 2671 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2672 DRM_ERROR("Attempting to bind a purgeable object\n");
2673 return -EINVAL;
2674 }
2675
673a394b 2676 if (alignment == 0)
0f973f27 2677 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2678 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2679 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2680 return -EINVAL;
2681 }
2682
654fc607
CW		 2683 /* If the object is bigger than the entire aperture, reject it early
2684 * before evicting everything in a vain attempt to find space.
2685 */
73aa808f 2686 if (obj->size > dev_priv->mm.gtt_total) {
654fc607
CW		 2687 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2688 return -E2BIG;
2689 }
2690
673a394b
EA		 2691 search_free:
2692 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2693 obj->size, alignment, 0);
2694 if (free_space != NULL) {
2695 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2696 alignment);
db3307a9 2697 if (obj_priv->gtt_space != NULL)
673a394b 2698 obj_priv->gtt_offset = obj_priv->gtt_space->start;
673a394b
EA		 2699 }
2700 if (obj_priv->gtt_space == NULL) {
2701 /* If the gtt is empty and we're still having trouble
2702 * fitting our object in, we're out of memory.
2703 */
0108a3ed 2704 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2705 if (ret)
673a394b 2706 return ret;
9731129c 2707
673a394b
EA		 2708 goto search_free;
2709 }
2710
4bdadb97 2711 ret = i915_gem_object_get_pages(obj, gfpmask);
673a394b
EA		 2712 if (ret) {
2713 drm_mm_put_block(obj_priv->gtt_space);
2714 obj_priv->gtt_space = NULL;
07f73f69
CW		 2715
2716 if (ret == -ENOMEM) {
2717 /* first try to clear up some space from the GTT */
0108a3ed
DV		 2718 ret = i915_gem_evict_something(dev, obj->size,
2719 alignment);
07f73f69 2720 if (ret) {
07f73f69 2721 /* now try to shrink everyone else */
4bdadb97
CW		 2722 if (gfpmask) {
2723 gfpmask = 0;
2724 goto search_free;
07f73f69
CW		 2725 }
2726
2727 return ret;
2728 }
2729
2730 goto search_free;
2731 }
2732
673a394b
EA		 2733 return ret;
2734 }
2735
673a394b
EA		 2736 /* Create an AGP memory structure pointing at our pages, and bind it
2737 * into the GTT.
2738 */
2739 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2740 obj_priv->pages,
07f73f69 2741 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2742 obj_priv->gtt_offset,
2743 obj_priv->agp_type);
673a394b 2744 if (obj_priv->agp_mem == NULL) {
856fa198 2745 i915_gem_object_put_pages(obj);
673a394b
EA		 2746 drm_mm_put_block(obj_priv->gtt_space);
2747 obj_priv->gtt_space = NULL;
07f73f69 2748
0108a3ed 2749 ret = i915_gem_evict_something(dev, obj->size, alignment);
9731129c 2750 if (ret)
07f73f69 2751 return ret;
07f73f69
CW		 2752
2753 goto search_free;
673a394b 2754 }
673a394b 2755
bf1a1092
CW		 2756 /* keep track of bounds object by adding it to the inactive list */
2757 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
73aa808f 2758 i915_gem_info_add_gtt(dev_priv, obj->size);
bf1a1092 2759
673a394b
EA		 2760 /* Assert that the object is not currently in any GPU domain. As it
2761 * wasn't in the GTT, there shouldn't be any way it could have been in
2762 * a GPU cache
2763 */
21d509e3
CW		 2764 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2765 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2766
1c5d22f7
CW		 2767 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2768
673a394b
EA		 2769 return 0;
2770}
2771
2772void
2773i915_gem_clflush_object(struct drm_gem_object *obj)
2774{
23010e43 2775 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 2776
2777 /* If we don't have a page list set up, then we're not pinned
2778 * to GPU, and we can ignore the cache flush because it'll happen
2779 * again at bind time.
2780 */
856fa198 2781 if (obj_priv->pages == NULL)
673a394b
EA		 2782 return;
2783
1c5d22f7 2784 trace_i915_gem_object_clflush(obj);
cfa16a0d 2785
856fa198 2786 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2787}
2788
e47c68e9 2789/** Flushes any GPU write domain for the object if it's dirty. */
2dafb1e0 2790static int
ba3d8d74
DV		 2791i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2792 bool pipelined)
e47c68e9
EA		 2793{
2794 struct drm_device *dev = obj->dev;
1c5d22f7 2795 uint32_t old_write_domain;
e47c68e9
EA		 2796
2797 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2dafb1e0 2798 return 0;
e47c68e9
EA		 2799
2800 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2801 old_write_domain = obj->write_domain;
c78ec30b 2802 i915_gem_flush_ring(dev, NULL,
9220434a
CW		 2803 to_intel_bo(obj)->ring,
2804 0, obj->write_domain);
48b956c5 2805 BUG_ON(obj->write_domain);
1c5d22f7
CW		 2806
2807 trace_i915_gem_object_change_domain(obj,
2808 obj->read_domains,
2809 old_write_domain);
ba3d8d74
DV		 2810
2811 if (pipelined)
2812 return 0;
2813
2cf34d7b 2814 return i915_gem_object_wait_rendering(obj, true);
e47c68e9
EA		 2815}
2816
2817/** Flushes the GTT write domain for the object if it's dirty. */
2818static void
2819i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2820{
1c5d22f7
CW		 2821 uint32_t old_write_domain;
2822
e47c68e9
EA		 2823 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2824 return;
2825
2826 /* No actual flushing is required for the GTT write domain. Writes
2827 * to it immediately go to main memory as far as we know, so there's
2828 * no chipset flush. It also doesn't land in render cache.
2829 */
1c5d22f7 2830 old_write_domain = obj->write_domain;
e47c68e9 2831 obj->write_domain = 0;
1c5d22f7
CW		 2832
2833 trace_i915_gem_object_change_domain(obj,
2834 obj->read_domains,
2835 old_write_domain);
e47c68e9
EA		 2836}
2837
2838/** Flushes the CPU write domain for the object if it's dirty. */
2839static void
2840i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2841{
2842 struct drm_device *dev = obj->dev;
1c5d22f7 2843 uint32_t old_write_domain;
e47c68e9
EA		 2844
2845 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2846 return;
2847
2848 i915_gem_clflush_object(obj);
2849 drm_agp_chipset_flush(dev);
1c5d22f7 2850 old_write_domain = obj->write_domain;
e47c68e9 2851 obj->write_domain = 0;
1c5d22f7
CW		 2852
2853 trace_i915_gem_object_change_domain(obj,
2854 obj->read_domains,
2855 old_write_domain);
e47c68e9
EA		 2856}
2857
2ef7eeaa
EA		 2858/**
2859 * Moves a single object to the GTT read, and possibly write domain.
2860 *
2861 * This function returns when the move is complete, including waiting on
2862 * flushes to occur.
2863 */
79e53945 2864int
2ef7eeaa
EA		 2865i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2866{
23010e43 2867 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 2868 uint32_t old_write_domain, old_read_domains;
e47c68e9 2869 int ret;
2ef7eeaa 2870
02354392
EA		 2871 /* Not valid to be called on unbound objects. */
2872 if (obj_priv->gtt_space == NULL)
2873 return -EINVAL;
2874
ba3d8d74 2875 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2876 if (ret != 0)
2877 return ret;
2878
7213342d 2879 i915_gem_object_flush_cpu_write_domain(obj);
1c5d22f7 2880
ba3d8d74 2881 if (write) {
2cf34d7b 2882 ret = i915_gem_object_wait_rendering(obj, true);
ba3d8d74
DV		 2883 if (ret)
2884 return ret;
ba3d8d74 2885 }
2ef7eeaa 2886
7213342d
CW		 2887 old_write_domain = obj->write_domain;
2888 old_read_domains = obj->read_domains;
2ef7eeaa 2889
e47c68e9
EA		 2890 /* It should now be out of any other write domains, and we can update
2891 * the domain values for our changes.
2892 */
2893 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2894 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2895 if (write) {
7213342d 2896 obj->read_domains = I915_GEM_DOMAIN_GTT;
e47c68e9
EA		 2897 obj->write_domain = I915_GEM_DOMAIN_GTT;
2898 obj_priv->dirty = 1;
2ef7eeaa
EA		 2899 }
2900
1c5d22f7
CW		 2901 trace_i915_gem_object_change_domain(obj,
2902 old_read_domains,
2903 old_write_domain);
2904
e47c68e9
EA		 2905 return 0;
2906}
2907
b9241ea3
ZW		 2908/*
2909 * Prepare buffer for display plane. Use uninterruptible for possible flush
2910 * wait, as in modesetting process we're not supposed to be interrupted.
2911 */
2912int
48b956c5
CW		 2913i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2914 bool pipelined)
b9241ea3 2915{
23010e43 2916 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
ba3d8d74 2917 uint32_t old_read_domains;
b9241ea3
ZW		 2918 int ret;
2919
2920 /* Not valid to be called on unbound objects. */
2921 if (obj_priv->gtt_space == NULL)
2922 return -EINVAL;
2923
ced270fa 2924 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
48b956c5 2925 if (ret)
e35a41de 2926 return ret;
b9241ea3 2927
ced270fa
CW		 2928 /* Currently, we are always called from an non-interruptible context. */
2929 if (!pipelined) {
2930 ret = i915_gem_object_wait_rendering(obj, false);
2931 if (ret)
2932 return ret;
2933 }
2934
b118c1e3
CW		 2935 i915_gem_object_flush_cpu_write_domain(obj);
2936
b9241ea3 2937 old_read_domains = obj->read_domains;
c78ec30b 2938 obj->read_domains |= I915_GEM_DOMAIN_GTT;
b9241ea3
ZW		 2939
2940 trace_i915_gem_object_change_domain(obj,
2941 old_read_domains,
ba3d8d74 2942 obj->write_domain);
b9241ea3
ZW		 2943
2944 return 0;
2945}
2946
e47c68e9
EA		 2947/**
2948 * Moves a single object to the CPU read, and possibly write domain.
2949 *
2950 * This function returns when the move is complete, including waiting on
2951 * flushes to occur.
2952 */
2953static int
2954i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2955{
1c5d22f7 2956 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2957 int ret;
2958
ba3d8d74 2959 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9
EA		 2960 if (ret != 0)
2961 return ret;
2ef7eeaa 2962
e47c68e9 2963 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2964
e47c68e9
EA		 2965 /* If we have a partially-valid cache of the object in the CPU,
2966 * finish invalidating it and free the per-page flags.
2ef7eeaa 2967 */
e47c68e9 2968 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2969
7213342d 2970 if (write) {
2cf34d7b 2971 ret = i915_gem_object_wait_rendering(obj, true);
7213342d
CW		 2972 if (ret)
2973 return ret;
2974 }
2975
1c5d22f7
CW		 2976 old_write_domain = obj->write_domain;
2977 old_read_domains = obj->read_domains;
2978
e47c68e9
EA		 2979 /* Flush the CPU cache if it's still invalid. */
2980 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2981 i915_gem_clflush_object(obj);
2ef7eeaa 2982
e47c68e9 2983 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2984 }
2985
2986 /* It should now be out of any other write domains, and we can update
2987 * the domain values for our changes.
2988 */
e47c68e9
EA		 2989 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2990
2991 /* If we're writing through the CPU, then the GPU read domains will
2992 * need to be invalidated at next use.
2993 */
2994 if (write) {
c78ec30b 2995 obj->read_domains = I915_GEM_DOMAIN_CPU;
e47c68e9
EA		 2996 obj->write_domain = I915_GEM_DOMAIN_CPU;
2997 }
2ef7eeaa 2998
1c5d22f7
CW		 2999 trace_i915_gem_object_change_domain(obj,
3000 old_read_domains,
3001 old_write_domain);
3002
2ef7eeaa
EA		 3003 return 0;
3004}
3005
673a394b
EA		 3006/*
3007 * Set the next domain for the specified object. This
3008 * may not actually perform the necessary flushing/invaliding though,
3009 * as that may want to be batched with other set_domain operations
3010 *
3011 * This is (we hope) the only really tricky part of gem. The goal
3012 * is fairly simple -- track which caches hold bits of the object
3013 * and make sure they remain coherent. A few concrete examples may
3014 * help to explain how it works. For shorthand, we use the notation
3015 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
3016 * a pair of read and write domain masks.
3017 *
3018 * Case 1: the batch buffer
3019 *
3020 * 1. Allocated
3021 * 2. Written by CPU
3022 * 3. Mapped to GTT
3023 * 4. Read by GPU
3024 * 5. Unmapped from GTT
3025 * 6. Freed
3026 *
3027 * Let's take these a step at a time
3028 *
3029 * 1. Allocated
3030 * Pages allocated from the kernel may still have
3031 * cache contents, so we set them to (CPU, CPU) always.
3032 * 2. Written by CPU (using pwrite)
3033 * The pwrite function calls set_domain (CPU, CPU) and
3034 * this function does nothing (as nothing changes)
3035 * 3. Mapped by GTT
3036 * This function asserts that the object is not
3037 * currently in any GPU-based read or write domains
3038 * 4. Read by GPU
3039 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
3040 * As write_domain is zero, this function adds in the
3041 * current read domains (CPU+COMMAND, 0).
3042 * flush_domains is set to CPU.
3043 * invalidate_domains is set to COMMAND
3044 * clflush is run to get data out of the CPU caches
3045 * then i915_dev_set_domain calls i915_gem_flush to
3046 * emit an MI_FLUSH and drm_agp_chipset_flush
3047 * 5. Unmapped from GTT
3048 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3049 * flush_domains and invalidate_domains end up both zero
3050 * so no flushing/invalidating happens
3051 * 6. Freed
3052 * yay, done
3053 *
3054 * Case 2: The shared render buffer
3055 *
3056 * 1. Allocated
3057 * 2. Mapped to GTT
3058 * 3. Read/written by GPU
3059 * 4. set_domain to (CPU,CPU)
3060 * 5. Read/written by CPU
3061 * 6. Read/written by GPU
3062 *
3063 * 1. Allocated
3064 * Same as last example, (CPU, CPU)
3065 * 2. Mapped to GTT
3066 * Nothing changes (assertions find that it is not in the GPU)
3067 * 3. Read/written by GPU
3068 * execbuffer calls set_domain (RENDER, RENDER)
3069 * flush_domains gets CPU
3070 * invalidate_domains gets GPU
3071 * clflush (obj)
3072 * MI_FLUSH and drm_agp_chipset_flush
3073 * 4. set_domain (CPU, CPU)
3074 * flush_domains gets GPU
3075 * invalidate_domains gets CPU
3076 * wait_rendering (obj) to make sure all drawing is complete.
3077 * This will include an MI_FLUSH to get the data from GPU
3078 * to memory
3079 * clflush (obj) to invalidate the CPU cache
3080 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3081 * 5. Read/written by CPU
3082 * cache lines are loaded and dirtied
3083 * 6. Read written by GPU
3084 * Same as last GPU access
3085 *
3086 * Case 3: The constant buffer
3087 *
3088 * 1. Allocated
3089 * 2. Written by CPU
3090 * 3. Read by GPU
3091 * 4. Updated (written) by CPU again
3092 * 5. Read by GPU
3093 *
3094 * 1. Allocated
3095 * (CPU, CPU)
3096 * 2. Written by CPU
3097 * (CPU, CPU)
3098 * 3. Read by GPU
3099 * (CPU+RENDER, 0)
3100 * flush_domains = CPU
3101 * invalidate_domains = RENDER
3102 * clflush (obj)
3103 * MI_FLUSH
3104 * drm_agp_chipset_flush
3105 * 4. Updated (written) by CPU again
3106 * (CPU, CPU)
3107 * flush_domains = 0 (no previous write domain)
3108 * invalidate_domains = 0 (no new read domains)
3109 * 5. Read by GPU
3110 * (CPU+RENDER, 0)
3111 * flush_domains = CPU
3112 * invalidate_domains = RENDER
3113 * clflush (obj)
3114 * MI_FLUSH
3115 * drm_agp_chipset_flush
3116 */
c0d90829 3117static void
8b0e378a 3118i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3119{
3120 struct drm_device *dev = obj->dev;
9220434a 3121 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 3122 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 3123 uint32_t invalidate_domains = 0;
3124 uint32_t flush_domains = 0;
1c5d22f7 3125 uint32_t old_read_domains;
e47c68e9 3126
652c393a
JB		 3127 intel_mark_busy(dev, obj);
3128
673a394b
EA		 3129 /*
3130 * If the object isn't moving to a new write domain,
3131 * let the object stay in multiple read domains
3132 */
8b0e378a
EA		 3133 if (obj->pending_write_domain == 0)
3134 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3135 else
3136 obj_priv->dirty = 1;
3137
3138 /*
3139 * Flush the current write domain if
3140 * the new read domains don't match. Invalidate
3141 * any read domains which differ from the old
3142 * write domain
3143 */
8b0e378a
EA		 3144 if (obj->write_domain &&
3145 obj->write_domain != obj->pending_read_domains) {
673a394b 3146 flush_domains |= obj->write_domain;
8b0e378a
EA		 3147 invalidate_domains |=
3148 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3149 }
3150 /*
3151 * Invalidate any read caches which may have
3152 * stale data. That is, any new read domains.
3153 */
8b0e378a 3154 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
3d2a812a 3155 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU)
673a394b 3156 i915_gem_clflush_object(obj);
673a394b 3157
1c5d22f7
CW		 3158 old_read_domains = obj->read_domains;
3159
efbeed96
EA		 3160 /* The actual obj->write_domain will be updated with
3161 * pending_write_domain after we emit the accumulated flush for all
3162 * of our domain changes in execbuffers (which clears objects'
3163 * write_domains). So if we have a current write domain that we
3164 * aren't changing, set pending_write_domain to that.
3165 */
3166 if (flush_domains == 0 && obj->pending_write_domain == 0)
3167 obj->pending_write_domain = obj->write_domain;
8b0e378a 3168 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3169
3170 dev->invalidate_domains |= invalidate_domains;
3171 dev->flush_domains |= flush_domains;
9220434a
CW		 3172 if (obj_priv->ring)
3173 dev_priv->mm.flush_rings |= obj_priv->ring->id;
1c5d22f7
CW		 3174
3175 trace_i915_gem_object_change_domain(obj,
3176 old_read_domains,
3177 obj->write_domain);
673a394b
EA		 3178}
3179
3180/**
e47c68e9 3181 * Moves the object from a partially CPU read to a full one.
673a394b 3182 *
e47c68e9
EA		 3183 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3184 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3185 */
e47c68e9
EA		 3186static void
3187i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b 3188{
23010e43 3189 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 3190
e47c68e9
EA		 3191 if (!obj_priv->page_cpu_valid)
3192 return;
3193
3194 /* If we're partially in the CPU read domain, finish moving it in.
3195 */
3196 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3197 int i;
3198
3199 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3200 if (obj_priv->page_cpu_valid[i])
3201 continue;
856fa198 3202 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3203 }
e47c68e9
EA		 3204 }
3205
3206 /* Free the page_cpu_valid mappings which are now stale, whether
3207 * or not we've got I915_GEM_DOMAIN_CPU.
3208 */
9a298b2a 3209 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3210 obj_priv->page_cpu_valid = NULL;
3211}
3212
3213/**
3214 * Set the CPU read domain on a range of the object.
3215 *
3216 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3217 * not entirely valid. The page_cpu_valid member of the object flags which
3218 * pages have been flushed, and will be respected by
3219 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3220 * of the whole object.
3221 *
3222 * This function returns when the move is complete, including waiting on
3223 * flushes to occur.
3224 */
3225static int
3226i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3227 uint64_t offset, uint64_t size)
3228{
23010e43 3229 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3230 uint32_t old_read_domains;
e47c68e9 3231 int i, ret;
673a394b 3232
e47c68e9
EA		 3233 if (offset == 0 && size == obj->size)
3234 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3235
ba3d8d74 3236 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
e47c68e9 3237 if (ret != 0)
6a47baa6 3238 return ret;
e47c68e9
EA		 3239 i915_gem_object_flush_gtt_write_domain(obj);
3240
3241 /* If we're already fully in the CPU read domain, we're done. */
3242 if (obj_priv->page_cpu_valid == NULL &&
3243 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3244 return 0;
673a394b 3245
e47c68e9
EA		 3246 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3247 * newly adding I915_GEM_DOMAIN_CPU
3248 */
673a394b 3249 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3250 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3251 GFP_KERNEL);
e47c68e9
EA		 3252 if (obj_priv->page_cpu_valid == NULL)
3253 return -ENOMEM;
3254 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3255 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3256
3257 /* Flush the cache on any pages that are still invalid from the CPU's
3258 * perspective.
3259 */
e47c68e9
EA		 3260 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3261 i++) {
673a394b
EA		 3262 if (obj_priv->page_cpu_valid[i])
3263 continue;
3264
856fa198 3265 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3266
3267 obj_priv->page_cpu_valid[i] = 1;
3268 }
3269
e47c68e9
EA		 3270 /* It should now be out of any other write domains, and we can update
3271 * the domain values for our changes.
3272 */
3273 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3274
1c5d22f7 3275 old_read_domains = obj->read_domains;
e47c68e9
EA		 3276 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3277
1c5d22f7
CW		 3278 trace_i915_gem_object_change_domain(obj,
3279 old_read_domains,
3280 obj->write_domain);
3281
673a394b
EA		 3282 return 0;
3283}
3284
673a394b
EA		 3285/**
3286 * Pin an object to the GTT and evaluate the relocations landing in it.
3287 */
3288static int
3289i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3290 struct drm_file *file_priv,
2549d6c2 3291 struct drm_i915_gem_exec_object2 *entry)
673a394b
EA		 3292{
3293 struct drm_device *dev = obj->dev;
0839ccb8 3294 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 3295 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
2549d6c2 3296 struct drm_i915_gem_relocation_entry __user *user_relocs;
673a394b 3297 int i, ret;
76446cac
JB		 3298 bool need_fence;
3299
3300 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3301 obj_priv->tiling_mode != I915_TILING_NONE;
3302
3303 /* Check fence reg constraints and rebind if necessary */
808b24d6
CW		 3304 if (need_fence &&
3305 !i915_gem_object_fence_offset_ok(obj,
3306 obj_priv->tiling_mode)) {
3307 ret = i915_gem_object_unbind(obj);
3308 if (ret)
3309 return ret;
3310 }
673a394b
EA		 3311
3312 /* Choose the GTT offset for our buffer and put it there. */
3313 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3314 if (ret)
3315 return ret;
3316
76446cac
JB		 3317 /*
3318 * Pre-965 chips need a fence register set up in order to
3319 * properly handle blits to/from tiled surfaces.
3320 */
3321 if (need_fence) {
53640e1d 3322 ret = i915_gem_object_get_fence_reg(obj, true);
76446cac 3323 if (ret != 0) {
76446cac
JB		 3324 i915_gem_object_unpin(obj);
3325 return ret;
3326 }
53640e1d
CW		 3327
3328 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
76446cac
JB		 3329 }
3330
673a394b
EA		 3331 entry->offset = obj_priv->gtt_offset;
3332
673a394b
EA		 3333 /* Apply the relocations, using the GTT aperture to avoid cache
3334 * flushing requirements.
3335 */
2549d6c2 3336 user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr;
673a394b 3337 for (i = 0; i < entry->relocation_count; i++) {
2549d6c2 3338 struct drm_i915_gem_relocation_entry reloc;
673a394b
EA		 3339 struct drm_gem_object *target_obj;
3340 struct drm_i915_gem_object *target_obj_priv;
673a394b 3341
2549d6c2
CW		 3342 ret = __copy_from_user_inatomic(&reloc,
3343 user_relocs+i,
3344 sizeof(reloc));
3345 if (ret) {
3346 i915_gem_object_unpin(obj);
3347 return -EFAULT;
3348 }
3349
673a394b 3350 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
2549d6c2 3351 reloc.target_handle);
673a394b
EA		 3352 if (target_obj == NULL) {
3353 i915_gem_object_unpin(obj);
bf79cb91 3354 return -ENOENT;
673a394b 3355 }
23010e43 3356 target_obj_priv = to_intel_bo(target_obj);
673a394b 3357
8542a0bb
CW		 3358#if WATCH_RELOC
3359 DRM_INFO("%s: obj %p offset %08x target %d "
3360 "read %08x write %08x gtt %08x "
3361 "presumed %08x delta %08x\n",
3362 __func__,
3363 obj,
2549d6c2
CW		 3364 (int) reloc.offset,
3365 (int) reloc.target_handle,
3366 (int) reloc.read_domains,
3367 (int) reloc.write_domain,
8542a0bb 3368 (int) target_obj_priv->gtt_offset,
2549d6c2
CW		 3369 (int) reloc.presumed_offset,
3370 reloc.delta);
8542a0bb
CW		 3371#endif
3372
673a394b
EA		 3373 /* The target buffer should have appeared before us in the
3374 * exec_object list, so it should have a GTT space bound by now.
3375 */
3376 if (target_obj_priv->gtt_space == NULL) {
3377 DRM_ERROR("No GTT space found for object %d\n",
2549d6c2 3378 reloc.target_handle);
673a394b
EA		 3379 drm_gem_object_unreference(target_obj);
3380 i915_gem_object_unpin(obj);
3381 return -EINVAL;
3382 }
3383
8542a0bb 3384 /* Validate that the target is in a valid r/w GPU domain */
2549d6c2 3385 if (reloc.write_domain & (reloc.write_domain - 1)) {
16edd550
DV		 3386 DRM_ERROR("reloc with multiple write domains: "
3387 "obj %p target %d offset %d "
3388 "read %08x write %08x",
2549d6c2
CW		 3389 obj, reloc.target_handle,
3390 (int) reloc.offset,
3391 reloc.read_domains,
3392 reloc.write_domain);
929f49bf
JL		 3393 drm_gem_object_unreference(target_obj);
3394 i915_gem_object_unpin(obj);
16edd550
DV		 3395 return -EINVAL;
3396 }
2549d6c2
CW		 3397 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
3398 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3399 DRM_ERROR("reloc with read/write CPU domains: "
3400 "obj %p target %d offset %d "
3401 "read %08x write %08x",
2549d6c2
CW		 3402 obj, reloc.target_handle,
3403 (int) reloc.offset,
3404 reloc.read_domains,
3405 reloc.write_domain);
491152b8
CW		 3406 drm_gem_object_unreference(target_obj);
3407 i915_gem_object_unpin(obj);
e47c68e9
EA		 3408 return -EINVAL;
3409 }
2549d6c2
CW		 3410 if (reloc.write_domain && target_obj->pending_write_domain &&
3411 reloc.write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3412 DRM_ERROR("Write domain conflict: "
3413 "obj %p target %d offset %d "
3414 "new %08x old %08x\n",
2549d6c2
CW		 3415 obj, reloc.target_handle,
3416 (int) reloc.offset,
3417 reloc.write_domain,
673a394b
EA		 3418 target_obj->pending_write_domain);
3419 drm_gem_object_unreference(target_obj);
3420 i915_gem_object_unpin(obj);
3421 return -EINVAL;
3422 }
3423
2549d6c2
CW		 3424 target_obj->pending_read_domains |= reloc.read_domains;
3425 target_obj->pending_write_domain |= reloc.write_domain;
673a394b
EA		 3426
3427 /* If the relocation already has the right value in it, no
3428 * more work needs to be done.
3429 */
2549d6c2 3430 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
673a394b
EA		 3431 drm_gem_object_unreference(target_obj);
3432 continue;
3433 }
3434
8542a0bb 3435 /* Check that the relocation address is valid... */
2549d6c2 3436 if (reloc.offset > obj->size - 4) {
8542a0bb
CW		 3437 DRM_ERROR("Relocation beyond object bounds: "
3438 "obj %p target %d offset %d size %d.\n",
2549d6c2
CW		 3439 obj, reloc.target_handle,
3440 (int) reloc.offset, (int) obj->size);
8542a0bb
CW		 3441 drm_gem_object_unreference(target_obj);
3442 i915_gem_object_unpin(obj);
3443 return -EINVAL;
3444 }
2549d6c2 3445 if (reloc.offset & 3) {
8542a0bb
CW		 3446 DRM_ERROR("Relocation not 4-byte aligned: "
3447 "obj %p target %d offset %d.\n",
2549d6c2
CW		 3448 obj, reloc.target_handle,
3449 (int) reloc.offset);
8542a0bb
CW		 3450 drm_gem_object_unreference(target_obj);
3451 i915_gem_object_unpin(obj);
3452 return -EINVAL;
3453 }
3454
3455 /* and points to somewhere within the target object. */
2549d6c2 3456 if (reloc.delta >= target_obj->size) {
8542a0bb
CW		 3457 DRM_ERROR("Relocation beyond target object bounds: "
3458 "obj %p target %d delta %d size %d.\n",
2549d6c2
CW		 3459 obj, reloc.target_handle,
3460 (int) reloc.delta, (int) target_obj->size);
8542a0bb
CW		 3461 drm_gem_object_unreference(target_obj);
3462 i915_gem_object_unpin(obj);
3463 return -EINVAL;
3464 }
3465
f0c43d9b
CW		 3466 reloc.delta += target_obj_priv->gtt_offset;
3467 if (obj->write_domain == I915_GEM_DOMAIN_CPU) {
3468 uint32_t page_offset = reloc.offset & ~PAGE_MASK;
3469 char *vaddr;
673a394b 3470
f0c43d9b
CW		 3471 vaddr = kmap_atomic(obj_priv->pages[reloc.offset >> PAGE_SHIFT], KM_USER0);
3472 *(uint32_t *)(vaddr + page_offset) = reloc.delta;
3473 kunmap_atomic(vaddr, KM_USER0);
3474 } else {
3475 uint32_t __iomem *reloc_entry;
3476 void __iomem *reloc_page;
3477 int ret;
3478
3479 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3480 if (ret) {
3481 drm_gem_object_unreference(target_obj);
3482 i915_gem_object_unpin(obj);
3483 return ret;
3484 }
3485
3486 /* Map the page containing the relocation we're going to perform. */
3487 reloc.offset += obj_priv->gtt_offset;
3488 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3489 reloc.offset & PAGE_MASK,
3490 KM_USER0);
3491 reloc_entry = (uint32_t __iomem *)
3492 (reloc_page + (reloc.offset & ~PAGE_MASK));
3493 iowrite32(reloc.delta, reloc_entry);
3494 io_mapping_unmap_atomic(reloc_page, KM_USER0);
3495 }
673a394b 3496
673a394b
EA		 3497 drm_gem_object_unreference(target_obj);
3498 }
3499
673a394b
EA		 3500 return 0;
3501}
3502
673a394b
EA		 3503/* Throttle our rendering by waiting until the ring has completed our requests
3504 * emitted over 20 msec ago.
3505 *
b962442e
EA		 3506 * Note that if we were to use the current jiffies each time around the loop,
3507 * we wouldn't escape the function with any frames outstanding if the time to
3508 * render a frame was over 20ms.
3509 *
673a394b
EA		 3510 * This should get us reasonable parallelism between CPU and GPU but also
3511 * relatively low latency when blocking on a particular request to finish.
3512 */
3513static int
f787a5f5 3514i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
673a394b 3515{
f787a5f5
CW		 3516 struct drm_i915_private *dev_priv = dev->dev_private;
3517 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e 3518 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
f787a5f5
CW		 3519 struct drm_i915_gem_request *request;
3520 struct intel_ring_buffer *ring = NULL;
3521 u32 seqno = 0;
3522 int ret;
673a394b 3523
1c25595f 3524 spin_lock(&file_priv->mm.lock);
f787a5f5 3525 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
b962442e
EA		 3526 if (time_after_eq(request->emitted_jiffies, recent_enough))
3527 break;
3528
f787a5f5
CW		 3529 ring = request->ring;
3530 seqno = request->seqno;
b962442e 3531 }
1c25595f 3532 spin_unlock(&file_priv->mm.lock);
f787a5f5
CW		 3533
3534 if (seqno == 0)
3535 return 0;
3536
3537 ret = 0;
3538 if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) {
3539 /* And wait for the seqno passing without holding any locks and
3540 * causing extra latency for others. This is safe as the irq
3541 * generation is designed to be run atomically and so is
3542 * lockless.
3543 */
3544 ring->user_irq_get(dev, ring);
3545 ret = wait_event_interruptible(ring->irq_queue,
3546 i915_seqno_passed(ring->get_seqno(dev, ring), seqno)
3547 || atomic_read(&dev_priv->mm.wedged));
3548 ring->user_irq_put(dev, ring);
3549
3550 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3551 ret = -EIO;
3552 }
3553
3554 if (ret == 0)
3555 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
b962442e 3556
673a394b
EA		 3557 return ret;
3558}
3559
40a5f0de 3560static int
2549d6c2
CW		 3561i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec,
3562 uint64_t exec_offset)
40a5f0de 3563{
2549d6c2 3564 uint32_t exec_start, exec_len;
40a5f0de 3565
2549d6c2
CW		 3566 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3567 exec_len = (uint32_t) exec->batch_len;
40a5f0de 3568
2549d6c2
CW		 3569 if ((exec_start | exec_len) & 0x7)
3570 return -EINVAL;
40a5f0de 3571
2549d6c2
CW		 3572 if (!exec_start)
3573 return -EINVAL;
40a5f0de 3574
2bc43b5c 3575 return 0;
40a5f0de
EA		 3576}
3577
3578static int
2549d6c2
CW		 3579validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
3580 int count)
40a5f0de 3581{
2549d6c2 3582 int i;
40a5f0de 3583
2549d6c2
CW		 3584 for (i = 0; i < count; i++) {
3585 char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr;
3586 size_t length = exec[i].relocation_count * sizeof(struct drm_i915_gem_relocation_entry);
2bc43b5c 3587
2549d6c2
CW		 3588 if (!access_ok(VERIFY_READ, ptr, length))
3589 return -EFAULT;
40a5f0de 3590
2549d6c2
CW		 3591 if (fault_in_pages_readable(ptr, length))
3592 return -EFAULT;
40a5f0de
EA		 3593 }
3594
83d60795
CW		 3595 return 0;
3596}
3597
8dc5d147 3598static int
76446cac
JB		 3599i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3600 struct drm_file *file_priv,
3601 struct drm_i915_gem_execbuffer2 *args,
3602 struct drm_i915_gem_exec_object2 *exec_list)
673a394b
EA		 3603{
3604 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3605 struct drm_gem_object **object_list = NULL;
3606 struct drm_gem_object *batch_obj;
b70d11da 3607 struct drm_i915_gem_object *obj_priv;
201361a5 3608 struct drm_clip_rect *cliprects = NULL;
8dc5d147 3609 struct drm_i915_gem_request *request = NULL;
2549d6c2 3610 int ret, i, pinned = 0;
673a394b 3611 uint64_t exec_offset;
6b95a207 3612 int pin_tries, flips;
673a394b 3613
852835f3
ZN		 3614 struct intel_ring_buffer *ring = NULL;
3615
30dbf0c0
CW		 3616 ret = i915_gem_check_is_wedged(dev);
3617 if (ret)
3618 return ret;
3619
2549d6c2
CW		 3620 ret = validate_exec_list(exec_list, args->buffer_count);
3621 if (ret)
3622 return ret;
3623
673a394b
EA		 3624#if WATCH_EXEC
3625 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3626 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3627#endif
d1b851fc
ZN		 3628 if (args->flags & I915_EXEC_BSD) {
3629 if (!HAS_BSD(dev)) {
3630 DRM_ERROR("execbuf with wrong flag\n");
3631 return -EINVAL;
3632 }
3633 ring = &dev_priv->bsd_ring;
3634 } else {
3635 ring = &dev_priv->render_ring;
3636 }
3637
4f481ed2
EA		 3638 if (args->buffer_count < 1) {
3639 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3640 return -EINVAL;
3641 }
c8e0f93a 3642 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
76446cac
JB		 3643 if (object_list == NULL) {
3644 DRM_ERROR("Failed to allocate object list for %d buffers\n",
673a394b
EA		 3645 args->buffer_count);
3646 ret = -ENOMEM;
3647 goto pre_mutex_err;
3648 }
673a394b 3649
201361a5 3650 if (args->num_cliprects != 0) {
9a298b2a
EA		 3651 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3652 GFP_KERNEL);
a40e8d31
OA		 3653 if (cliprects == NULL) {
3654 ret = -ENOMEM;
201361a5 3655 goto pre_mutex_err;
a40e8d31 3656 }
201361a5
EA		 3657
3658 ret = copy_from_user(cliprects,
3659 (struct drm_clip_rect __user *)
3660 (uintptr_t) args->cliprects_ptr,
3661 sizeof(*cliprects) * args->num_cliprects);
3662 if (ret != 0) {
3663 DRM_ERROR("copy %d cliprects failed: %d\n",
3664 args->num_cliprects, ret);
c877cdce 3665 ret = -EFAULT;
201361a5
EA		 3666 goto pre_mutex_err;
3667 }
3668 }
3669
8dc5d147
CW		 3670 request = kzalloc(sizeof(*request), GFP_KERNEL);
3671 if (request == NULL) {
3672 ret = -ENOMEM;
3673 goto pre_mutex_err;
3674 }
3675
76c1dec1
CW		 3676 ret = i915_mutex_lock_interruptible(dev);
3677 if (ret)
3678 goto pre_mutex_err;
673a394b 3679
673a394b 3680 if (dev_priv->mm.suspended) {
673a394b 3681 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3682 ret = -EBUSY;
3683 goto pre_mutex_err;
673a394b
EA		 3684 }
3685
ac94a962 3686 /* Look up object handles */
673a394b
EA		 3687 for (i = 0; i < args->buffer_count; i++) {
3688 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3689 exec_list[i].handle);
3690 if (object_list[i] == NULL) {
3691 DRM_ERROR("Invalid object handle %d at index %d\n",
3692 exec_list[i].handle, i);
0ce907f8
CW		 3693 /* prevent error path from reading uninitialized data */
3694 args->buffer_count = i + 1;
bf79cb91 3695 ret = -ENOENT;
673a394b
EA		 3696 goto err;
3697 }
b70d11da 3698
23010e43 3699 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3700 if (obj_priv->in_execbuffer) {
3701 DRM_ERROR("Object %p appears more than once in object list\n",
3702 object_list[i]);
0ce907f8
CW		 3703 /* prevent error path from reading uninitialized data */
3704 args->buffer_count = i + 1;
bf79cb91 3705 ret = -EINVAL;
b70d11da
KH		 3706 goto err;
3707 }
3708 obj_priv->in_execbuffer = true;
ac94a962 3709 }
673a394b 3710
ac94a962
KP		 3711 /* Pin and relocate */
3712 for (pin_tries = 0; ; pin_tries++) {
3713 ret = 0;
40a5f0de 3714
ac94a962
KP		 3715 for (i = 0; i < args->buffer_count; i++) {
3716 object_list[i]->pending_read_domains = 0;
3717 object_list[i]->pending_write_domain = 0;
3718 ret = i915_gem_object_pin_and_relocate(object_list[i],
3719 file_priv,
2549d6c2 3720 &exec_list[i]);
ac94a962
KP		 3721 if (ret)
3722 break;
3723 pinned = i + 1;
3724 }
3725 /* success */
3726 if (ret == 0)
3727 break;
3728
3729 /* error other than GTT full, or we've already tried again */
2939e1f5 3730 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3731 if (ret != -ERESTARTSYS) {
3732 unsigned long long total_size = 0;
3d1cc470
CW		 3733 int num_fences = 0;
3734 for (i = 0; i < args->buffer_count; i++) {
43b27f40 3735 obj_priv = to_intel_bo(object_list[i]);
3d1cc470 3736
07f73f69 3737 total_size += object_list[i]->size;
3d1cc470
CW		 3738 num_fences +=
3739 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3740 obj_priv->tiling_mode != I915_TILING_NONE;
3741 }
3742 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
07f73f69 3743 pinned+1, args->buffer_count,
3d1cc470
CW		 3744 total_size, num_fences,
3745 ret);
73aa808f
CW		 3746 DRM_ERROR("%u objects [%u pinned, %u GTT], "
3747 "%zu object bytes [%zu pinned], "
3748 "%zu /%zu gtt bytes\n",
3749 dev_priv->mm.object_count,
3750 dev_priv->mm.pin_count,
3751 dev_priv->mm.gtt_count,
3752 dev_priv->mm.object_memory,
3753 dev_priv->mm.pin_memory,
3754 dev_priv->mm.gtt_memory,
3755 dev_priv->mm.gtt_total);
07f73f69 3756 }
673a394b
EA		 3757 goto err;
3758 }
ac94a962
KP		 3759
3760 /* unpin all of our buffers */
3761 for (i = 0; i < pinned; i++)
3762 i915_gem_object_unpin(object_list[i]);
b1177636 3763 pinned = 0;
ac94a962
KP		 3764
3765 /* evict everyone we can from the aperture */
3766 ret = i915_gem_evict_everything(dev);
07f73f69 3767 if (ret && ret != -ENOSPC)
ac94a962 3768 goto err;
673a394b
EA		 3769 }
3770
3771 /* Set the pending read domains for the batch buffer to COMMAND */
3772 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3773 if (batch_obj->pending_write_domain) {
3774 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3775 ret = -EINVAL;
3776 goto err;
3777 }
3778 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3779
83d60795
CW		 3780 /* Sanity check the batch buffer, prior to moving objects */
3781 exec_offset = exec_list[args->buffer_count - 1].offset;
3782 ret = i915_gem_check_execbuffer (args, exec_offset);
3783 if (ret != 0) {
3784 DRM_ERROR("execbuf with invalid offset/length\n");
3785 goto err;
3786 }
3787
646f0f6e
KP		 3788 /* Zero the global flush/invalidate flags. These
3789 * will be modified as new domains are computed
3790 * for each object
3791 */
3792 dev->invalidate_domains = 0;
3793 dev->flush_domains = 0;
9220434a 3794 dev_priv->mm.flush_rings = 0;
646f0f6e 3795
673a394b
EA		 3796 for (i = 0; i < args->buffer_count; i++) {
3797 struct drm_gem_object *obj = object_list[i];
673a394b 3798
646f0f6e 3799 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3800 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3801 }
3802
646f0f6e
KP		 3803 if (dev->invalidate_domains | dev->flush_domains) {
3804#if WATCH_EXEC
3805 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3806 __func__,
3807 dev->invalidate_domains,
3808 dev->flush_domains);
3809#endif
c78ec30b 3810 i915_gem_flush(dev, file_priv,
646f0f6e 3811 dev->invalidate_domains,
9220434a
CW		 3812 dev->flush_domains,
3813 dev_priv->mm.flush_rings);
a6910434
DV		 3814 }
3815
efbeed96
EA		 3816 for (i = 0; i < args->buffer_count; i++) {
3817 struct drm_gem_object *obj = object_list[i];
23010e43 3818 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
1c5d22f7 3819 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3820
3821 obj->write_domain = obj->pending_write_domain;
99fcb766
DV		 3822 if (obj->write_domain)
3823 list_move_tail(&obj_priv->gpu_write_list,
3824 &dev_priv->mm.gpu_write_list);
99fcb766 3825
1c5d22f7
CW		 3826 trace_i915_gem_object_change_domain(obj,
3827 obj->read_domains,
3828 old_write_domain);
efbeed96
EA		 3829 }
3830
673a394b
EA		 3831#if WATCH_COHERENCY
3832 for (i = 0; i < args->buffer_count; i++) {
3833 i915_gem_object_check_coherency(object_list[i],
3834 exec_list[i].handle);
3835 }
3836#endif
3837
673a394b 3838#if WATCH_EXEC
6911a9b8 3839 i915_gem_dump_object(batch_obj,
673a394b
EA		 3840 args->batch_len,
3841 __func__,
3842 ~0);
3843#endif
3844
e59f2bac
CW		 3845 /* Check for any pending flips. As we only maintain a flip queue depth
3846 * of 1, we can simply insert a WAIT for the next display flip prior
3847 * to executing the batch and avoid stalling the CPU.
3848 */
3849 flips = 0;
3850 for (i = 0; i < args->buffer_count; i++) {
3851 if (object_list[i]->write_domain)
3852 flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip);
3853 }
3854 if (flips) {
3855 int plane, flip_mask;
3856
3857 for (plane = 0; flips >> plane; plane++) {
3858 if (((flips >> plane) & 1) == 0)
3859 continue;
3860
3861 if (plane)
3862 flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
3863 else
3864 flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
3865
3866 intel_ring_begin(dev, ring, 2);
3867 intel_ring_emit(dev, ring,
3868 MI_WAIT_FOR_EVENT | flip_mask);
3869 intel_ring_emit(dev, ring, MI_NOOP);
3870 intel_ring_advance(dev, ring);
3871 }
3872 }
3873
673a394b 3874 /* Exec the batchbuffer */
852835f3 3875 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
e59f2bac 3876 cliprects, exec_offset);
673a394b
EA		 3877 if (ret) {
3878 DRM_ERROR("dispatch failed %d\n", ret);
3879 goto err;
3880 }
3881
3882 /*
3883 * Ensure that the commands in the batch buffer are
3884 * finished before the interrupt fires
3885 */
8a1a49f9 3886 i915_retire_commands(dev, ring);
673a394b 3887
617dbe27
DV		 3888 for (i = 0; i < args->buffer_count; i++) {
3889 struct drm_gem_object *obj = object_list[i];
3890 obj_priv = to_intel_bo(obj);
3891
3892 i915_gem_object_move_to_active(obj, ring);
617dbe27 3893 }
a56ba56c 3894
5c12a07e 3895 i915_add_request(dev, file_priv, request, ring);
8dc5d147 3896 request = NULL;
673a394b 3897
673a394b 3898err:
aad87dff
JL		 3899 for (i = 0; i < pinned; i++)
3900 i915_gem_object_unpin(object_list[i]);
3901
b70d11da
KH		 3902 for (i = 0; i < args->buffer_count; i++) {
3903 if (object_list[i]) {
23010e43 3904 obj_priv = to_intel_bo(object_list[i]);
b70d11da
KH		 3905 obj_priv->in_execbuffer = false;
3906 }
aad87dff 3907 drm_gem_object_unreference(object_list[i]);
b70d11da 3908 }
673a394b 3909
673a394b
EA		 3910 mutex_unlock(&dev->struct_mutex);
3911
93533c29 3912pre_mutex_err:
8e7d2b2c 3913 drm_free_large(object_list);
9a298b2a 3914 kfree(cliprects);
8dc5d147 3915 kfree(request);
673a394b
EA		 3916
3917 return ret;
3918}
3919
76446cac
JB		 3920/*
3921 * Legacy execbuffer just creates an exec2 list from the original exec object
3922 * list array and passes it to the real function.
3923 */
3924int
3925i915_gem_execbuffer(struct drm_device *dev, void *data,
3926 struct drm_file *file_priv)
3927{
3928 struct drm_i915_gem_execbuffer *args = data;
3929 struct drm_i915_gem_execbuffer2 exec2;
3930 struct drm_i915_gem_exec_object *exec_list = NULL;
3931 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3932 int ret, i;
3933
3934#if WATCH_EXEC
3935 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3936 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3937#endif
3938
3939 if (args->buffer_count < 1) {
3940 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3941 return -EINVAL;
3942 }
3943
3944 /* Copy in the exec list from userland */
3945 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3946 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3947 if (exec_list == NULL || exec2_list == NULL) {
3948 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3949 args->buffer_count);
3950 drm_free_large(exec_list);
3951 drm_free_large(exec2_list);
3952 return -ENOMEM;
3953 }
3954 ret = copy_from_user(exec_list,
3955 (struct drm_i915_relocation_entry __user *)
3956 (uintptr_t) args->buffers_ptr,
3957 sizeof(*exec_list) * args->buffer_count);
3958 if (ret != 0) {
3959 DRM_ERROR("copy %d exec entries failed %d\n",
3960 args->buffer_count, ret);
3961 drm_free_large(exec_list);
3962 drm_free_large(exec2_list);
3963 return -EFAULT;
3964 }
3965
3966 for (i = 0; i < args->buffer_count; i++) {
3967 exec2_list[i].handle = exec_list[i].handle;
3968 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3969 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3970 exec2_list[i].alignment = exec_list[i].alignment;
3971 exec2_list[i].offset = exec_list[i].offset;
a6c45cf0 3972 if (INTEL_INFO(dev)->gen < 4)
76446cac
JB		 3973 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3974 else
3975 exec2_list[i].flags = 0;
3976 }
3977
3978 exec2.buffers_ptr = args->buffers_ptr;
3979 exec2.buffer_count = args->buffer_count;
3980 exec2.batch_start_offset = args->batch_start_offset;
3981 exec2.batch_len = args->batch_len;
3982 exec2.DR1 = args->DR1;
3983 exec2.DR4 = args->DR4;
3984 exec2.num_cliprects = args->num_cliprects;
3985 exec2.cliprects_ptr = args->cliprects_ptr;
852835f3 3986 exec2.flags = I915_EXEC_RENDER;
76446cac
JB		 3987
3988 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3989 if (!ret) {
3990 /* Copy the new buffer offsets back to the user's exec list. */
3991 for (i = 0; i < args->buffer_count; i++)
3992 exec_list[i].offset = exec2_list[i].offset;
3993 /* ... and back out to userspace */
3994 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3995 (uintptr_t) args->buffers_ptr,
3996 exec_list,
3997 sizeof(*exec_list) * args->buffer_count);
3998 if (ret) {
3999 ret = -EFAULT;
4000 DRM_ERROR("failed to copy %d exec entries "
4001 "back to user (%d)\n",
4002 args->buffer_count, ret);
4003 }
76446cac
JB		 4004 }
4005
4006 drm_free_large(exec_list);
4007 drm_free_large(exec2_list);
4008 return ret;
4009}
4010
4011int
4012i915_gem_execbuffer2(struct drm_device *dev, void *data,
4013 struct drm_file *file_priv)
4014{
4015 struct drm_i915_gem_execbuffer2 *args = data;
4016 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4017 int ret;
4018
4019#if WATCH_EXEC
4020 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4021 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4022#endif
4023
4024 if (args->buffer_count < 1) {
4025 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4026 return -EINVAL;
4027 }
4028
4029 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4030 if (exec2_list == NULL) {
4031 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4032 args->buffer_count);
4033 return -ENOMEM;
4034 }
4035 ret = copy_from_user(exec2_list,
4036 (struct drm_i915_relocation_entry __user *)
4037 (uintptr_t) args->buffers_ptr,
4038 sizeof(*exec2_list) * args->buffer_count);
4039 if (ret != 0) {
4040 DRM_ERROR("copy %d exec entries failed %d\n",
4041 args->buffer_count, ret);
4042 drm_free_large(exec2_list);
4043 return -EFAULT;
4044 }
4045
4046 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4047 if (!ret) {
4048 /* Copy the new buffer offsets back to the user's exec list. */
4049 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4050 (uintptr_t) args->buffers_ptr,
4051 exec2_list,
4052 sizeof(*exec2_list) * args->buffer_count);
4053 if (ret) {
4054 ret = -EFAULT;
4055 DRM_ERROR("failed to copy %d exec entries "
4056 "back to user (%d)\n",
4057 args->buffer_count, ret);
4058 }
4059 }
4060
4061 drm_free_large(exec2_list);
4062 return ret;
4063}
4064
673a394b
EA		 4065int
4066i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4067{
4068 struct drm_device *dev = obj->dev;
f13d3f73 4069 struct drm_i915_private *dev_priv = dev->dev_private;
23010e43 4070 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b
EA		 4071 int ret;
4072
778c3544 4073 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
23bc5982 4074 WARN_ON(i915_verify_lists(dev));
ac0c6b5a
CW		 4075
4076 if (obj_priv->gtt_space != NULL) {
4077 if (alignment == 0)
4078 alignment = i915_gem_get_gtt_alignment(obj);
4079 if (obj_priv->gtt_offset & (alignment - 1)) {
ae7d49d8
CW		 4080 WARN(obj_priv->pin_count,
4081 "bo is already pinned with incorrect alignment:"
4082 " offset=%x, req.alignment=%x\n",
4083 obj_priv->gtt_offset, alignment);
ac0c6b5a
CW		 4084 ret = i915_gem_object_unbind(obj);
4085 if (ret)
4086 return ret;
4087 }
4088 }
4089
673a394b
EA		 4090 if (obj_priv->gtt_space == NULL) {
4091 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 4092 if (ret)
673a394b 4093 return ret;
22c344e9 4094 }
76446cac 4095
673a394b
EA		 4096 obj_priv->pin_count++;
4097
4098 /* If the object is not active and not pending a flush,
4099 * remove it from the inactive list
4100 */
4101 if (obj_priv->pin_count == 1) {
73aa808f 4102 i915_gem_info_add_pin(dev_priv, obj->size);
f13d3f73
CW		 4103 if (!obj_priv->active)
4104 list_move_tail(&obj_priv->list,
4105 &dev_priv->mm.pinned_list);
673a394b 4106 }
673a394b 4107
23bc5982 4108 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4109 return 0;
4110}
4111
4112void
4113i915_gem_object_unpin(struct drm_gem_object *obj)
4114{
4115 struct drm_device *dev = obj->dev;
4116 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4117 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
673a394b 4118
23bc5982 4119 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4120 obj_priv->pin_count--;
4121 BUG_ON(obj_priv->pin_count < 0);
4122 BUG_ON(obj_priv->gtt_space == NULL);
4123
4124 /* If the object is no longer pinned, and is
4125 * neither active nor being flushed, then stick it on
4126 * the inactive list
4127 */
4128 if (obj_priv->pin_count == 0) {
f13d3f73 4129 if (!obj_priv->active)
673a394b
EA		 4130 list_move_tail(&obj_priv->list,
4131 &dev_priv->mm.inactive_list);
73aa808f 4132 i915_gem_info_remove_pin(dev_priv, obj->size);
673a394b 4133 }
23bc5982 4134 WARN_ON(i915_verify_lists(dev));
673a394b
EA		 4135}
4136
4137int
4138i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4139 struct drm_file *file_priv)
4140{
4141 struct drm_i915_gem_pin *args = data;
4142 struct drm_gem_object *obj;
4143 struct drm_i915_gem_object *obj_priv;
4144 int ret;
4145
673a394b
EA		 4146 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4147 if (obj == NULL) {
4148 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4149 args->handle);
bf79cb91 4150 return -ENOENT;
673a394b 4151 }
23010e43 4152 obj_priv = to_intel_bo(obj);
673a394b 4153
76c1dec1
CW		 4154 ret = i915_mutex_lock_interruptible(dev);
4155 if (ret) {
4156 drm_gem_object_unreference_unlocked(obj);
4157 return ret;
4158 }
4159
bb6baf76
CW		 4160 if (obj_priv->madv != I915_MADV_WILLNEED) {
4161 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4162 drm_gem_object_unreference(obj);
4163 mutex_unlock(&dev->struct_mutex);
4164 return -EINVAL;
4165 }
4166
79e53945
JB		 4167 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4168 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4169 args->handle);
96dec61d 4170 drm_gem_object_unreference(obj);
673a394b 4171 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4172 return -EINVAL;
4173 }
4174
4175 obj_priv->user_pin_count++;
4176 obj_priv->pin_filp = file_priv;
4177 if (obj_priv->user_pin_count == 1) {
4178 ret = i915_gem_object_pin(obj, args->alignment);
4179 if (ret != 0) {
4180 drm_gem_object_unreference(obj);
4181 mutex_unlock(&dev->struct_mutex);
4182 return ret;
4183 }
673a394b
EA		 4184 }
4185
4186 /* XXX - flush the CPU caches for pinned objects
4187 * as the X server doesn't manage domains yet
4188 */
e47c68e9 4189 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4190 args->offset = obj_priv->gtt_offset;
4191 drm_gem_object_unreference(obj);
4192 mutex_unlock(&dev->struct_mutex);
4193
4194 return 0;
4195}
4196
4197int
4198i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4199 struct drm_file *file_priv)
4200{
4201 struct drm_i915_gem_pin *args = data;
4202 struct drm_gem_object *obj;
79e53945 4203 struct drm_i915_gem_object *obj_priv;
76c1dec1 4204 int ret;
673a394b
EA		 4205
4206 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4207 if (obj == NULL) {
4208 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4209 args->handle);
bf79cb91 4210 return -ENOENT;
673a394b
EA		 4211 }
4212
23010e43 4213 obj_priv = to_intel_bo(obj);
76c1dec1
CW		 4214
4215 ret = i915_mutex_lock_interruptible(dev);
4216 if (ret) {
4217 drm_gem_object_unreference_unlocked(obj);
4218 return ret;
4219 }
4220
79e53945
JB		 4221 if (obj_priv->pin_filp != file_priv) {
4222 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4223 args->handle);
4224 drm_gem_object_unreference(obj);
4225 mutex_unlock(&dev->struct_mutex);
4226 return -EINVAL;
4227 }
4228 obj_priv->user_pin_count--;
4229 if (obj_priv->user_pin_count == 0) {
4230 obj_priv->pin_filp = NULL;
4231 i915_gem_object_unpin(obj);
4232 }
673a394b
EA		 4233
4234 drm_gem_object_unreference(obj);
4235 mutex_unlock(&dev->struct_mutex);
4236 return 0;
4237}
4238
4239int
4240i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4241 struct drm_file *file_priv)
4242{
4243 struct drm_i915_gem_busy *args = data;
4244 struct drm_gem_object *obj;
4245 struct drm_i915_gem_object *obj_priv;
30dbf0c0
CW		 4246 int ret;
4247
673a394b
EA		 4248 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4249 if (obj == NULL) {
4250 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4251 args->handle);
bf79cb91 4252 return -ENOENT;
673a394b
EA		 4253 }
4254
76c1dec1
CW		 4255 ret = i915_mutex_lock_interruptible(dev);
4256 if (ret) {
4257 drm_gem_object_unreference_unlocked(obj);
4258 return ret;
30dbf0c0
CW		 4259 }
4260
0be555b6
CW		 4261 /* Count all active objects as busy, even if they are currently not used
4262 * by the gpu. Users of this interface expect objects to eventually
4263 * become non-busy without any further actions, therefore emit any
4264 * necessary flushes here.
c4de0a5d 4265 */
0be555b6
CW		 4266 obj_priv = to_intel_bo(obj);
4267 args->busy = obj_priv->active;
4268 if (args->busy) {
4269 /* Unconditionally flush objects, even when the gpu still uses this
4270 * object. Userspace calling this function indicates that it wants to
4271 * use this buffer rather sooner than later, so issuing the required
4272 * flush earlier is beneficial.
4273 */
c78ec30b
CW		 4274 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4275 i915_gem_flush_ring(dev, file_priv,
9220434a
CW		 4276 obj_priv->ring,
4277 0, obj->write_domain);
0be555b6
CW		 4278
4279 /* Update the active list for the hardware's current position.
4280 * Otherwise this only updates on a delayed timer or when irqs
4281 * are actually unmasked, and our working set ends up being
4282 * larger than required.
4283 */
4284 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4285
4286 args->busy = obj_priv->active;
4287 }
673a394b
EA		 4288
4289 drm_gem_object_unreference(obj);
4290 mutex_unlock(&dev->struct_mutex);
76c1dec1 4291 return 0;
673a394b
EA		 4292}
4293
4294int
4295i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4296 struct drm_file *file_priv)
4297{
4298 return i915_gem_ring_throttle(dev, file_priv);
4299}
4300
3ef94daa
CW		 4301int
4302i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4303 struct drm_file *file_priv)
4304{
4305 struct drm_i915_gem_madvise *args = data;
4306 struct drm_gem_object *obj;
4307 struct drm_i915_gem_object *obj_priv;
76c1dec1 4308 int ret;
3ef94daa
CW		 4309
4310 switch (args->madv) {
4311 case I915_MADV_DONTNEED:
4312 case I915_MADV_WILLNEED:
4313 break;
4314 default:
4315 return -EINVAL;
4316 }
4317
4318 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4319 if (obj == NULL) {
4320 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4321 args->handle);
bf79cb91 4322 return -ENOENT;
3ef94daa 4323 }
23010e43 4324 obj_priv = to_intel_bo(obj);
3ef94daa 4325
76c1dec1
CW		 4326 ret = i915_mutex_lock_interruptible(dev);
4327 if (ret) {
4328 drm_gem_object_unreference_unlocked(obj);
4329 return ret;
4330 }
4331
3ef94daa
CW		 4332 if (obj_priv->pin_count) {
4333 drm_gem_object_unreference(obj);
4334 mutex_unlock(&dev->struct_mutex);
4335
4336 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4337 return -EINVAL;
4338 }
4339
bb6baf76
CW		 4340 if (obj_priv->madv != __I915_MADV_PURGED)
4341 obj_priv->madv = args->madv;
3ef94daa 4342
2d7ef395
CW		 4343 /* if the object is no longer bound, discard its backing storage */
4344 if (i915_gem_object_is_purgeable(obj_priv) &&
4345 obj_priv->gtt_space == NULL)
4346 i915_gem_object_truncate(obj);
4347
bb6baf76
CW		 4348 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4349
3ef94daa
CW		 4350 drm_gem_object_unreference(obj);
4351 mutex_unlock(&dev->struct_mutex);
4352
4353 return 0;
4354}
4355
ac52bc56
DV		 4356struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4357 size_t size)
4358{
73aa808f 4359 struct drm_i915_private *dev_priv = dev->dev_private;
c397b908 4360 struct drm_i915_gem_object *obj;
ac52bc56 4361
c397b908
DV		 4362 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4363 if (obj == NULL)
4364 return NULL;
673a394b 4365
c397b908
DV		 4366 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4367 kfree(obj);
4368 return NULL;
4369 }
673a394b 4370
73aa808f
CW		 4371 i915_gem_info_add_obj(dev_priv, size);
4372
c397b908
DV		 4373 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4374 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
673a394b 4375
c397b908 4376 obj->agp_type = AGP_USER_MEMORY;
62b8b215 4377 obj->base.driver_private = NULL;
c397b908
DV		 4378 obj->fence_reg = I915_FENCE_REG_NONE;
4379 INIT_LIST_HEAD(&obj->list);
4380 INIT_LIST_HEAD(&obj->gpu_write_list);
c397b908 4381 obj->madv = I915_MADV_WILLNEED;
de151cf6 4382
c397b908
DV		 4383 trace_i915_gem_object_create(&obj->base);
4384
4385 return &obj->base;
4386}
4387
4388int i915_gem_init_object(struct drm_gem_object *obj)
4389{
4390 BUG();
de151cf6 4391
673a394b
EA		 4392 return 0;
4393}
4394
be72615b 4395static void i915_gem_free_object_tail(struct drm_gem_object *obj)
673a394b 4396{
de151cf6 4397 struct drm_device *dev = obj->dev;
be72615b 4398 drm_i915_private_t *dev_priv = dev->dev_private;
23010e43 4399 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
be72615b 4400 int ret;
673a394b 4401
be72615b
CW		 4402 ret = i915_gem_object_unbind(obj);
4403 if (ret == -ERESTARTSYS) {
4404 list_move(&obj_priv->list,
4405 &dev_priv->mm.deferred_free_list);
4406 return;
4407 }
673a394b 4408
7e616158
CW		 4409 if (obj_priv->mmap_offset)
4410 i915_gem_free_mmap_offset(obj);
de151cf6 4411
c397b908 4412 drm_gem_object_release(obj);
73aa808f 4413 i915_gem_info_remove_obj(dev_priv, obj->size);
c397b908 4414
9a298b2a 4415 kfree(obj_priv->page_cpu_valid);
280b713b 4416 kfree(obj_priv->bit_17);
c397b908 4417 kfree(obj_priv);
673a394b
EA		 4418}
4419
be72615b
CW		 4420void i915_gem_free_object(struct drm_gem_object *obj)
4421{
4422 struct drm_device *dev = obj->dev;
4423 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4424
4425 trace_i915_gem_object_destroy(obj);
4426
4427 while (obj_priv->pin_count > 0)
4428 i915_gem_object_unpin(obj);
4429
4430 if (obj_priv->phys_obj)
4431 i915_gem_detach_phys_object(dev, obj);
4432
4433 i915_gem_free_object_tail(obj);
4434}
4435
29105ccc
CW		 4436int
4437i915_gem_idle(struct drm_device *dev)
4438{
4439 drm_i915_private_t *dev_priv = dev->dev_private;
4440 int ret;
28dfe52a 4441
29105ccc 4442 mutex_lock(&dev->struct_mutex);
1c5d22f7 4443
8187a2b7 4444 if (dev_priv->mm.suspended ||
d1b851fc
ZN		 4445 (dev_priv->render_ring.gem_object == NULL) ||
4446 (HAS_BSD(dev) &&
4447 dev_priv->bsd_ring.gem_object == NULL)) {
29105ccc
CW		 4448 mutex_unlock(&dev->struct_mutex);
4449 return 0;
28dfe52a
EA		 4450 }
4451
29105ccc 4452 ret = i915_gpu_idle(dev);
6dbe2772
KP		 4453 if (ret) {
4454 mutex_unlock(&dev->struct_mutex);
673a394b 4455 return ret;
6dbe2772 4456 }
673a394b 4457
29105ccc
CW		 4458 /* Under UMS, be paranoid and evict. */
4459 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
b47eb4a2 4460 ret = i915_gem_evict_inactive(dev);
29105ccc
CW		 4461 if (ret) {
4462 mutex_unlock(&dev->struct_mutex);
4463 return ret;
4464 }
4465 }
4466
4467 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4468 * We need to replace this with a semaphore, or something.
4469 * And not confound mm.suspended!
4470 */
4471 dev_priv->mm.suspended = 1;
bc0c7f14 4472 del_timer_sync(&dev_priv->hangcheck_timer);
29105ccc
CW		 4473
4474 i915_kernel_lost_context(dev);
6dbe2772 4475 i915_gem_cleanup_ringbuffer(dev);
29105ccc 4476
6dbe2772
KP		 4477 mutex_unlock(&dev->struct_mutex);
4478
29105ccc
CW		 4479 /* Cancel the retire work handler, which should be idle now. */
4480 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4481
673a394b
EA		 4482 return 0;
4483}
4484
e552eb70
JB		 4485/*
4486 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4487 * over cache flushing.
4488 */
8187a2b7 4489static int
e552eb70
JB		 4490i915_gem_init_pipe_control(struct drm_device *dev)
4491{
4492 drm_i915_private_t *dev_priv = dev->dev_private;
4493 struct drm_gem_object *obj;
4494 struct drm_i915_gem_object *obj_priv;
4495 int ret;
4496
34dc4d44 4497 obj = i915_gem_alloc_object(dev, 4096);
e552eb70
JB		 4498 if (obj == NULL) {
4499 DRM_ERROR("Failed to allocate seqno page\n");
4500 ret = -ENOMEM;
4501 goto err;
4502 }
4503 obj_priv = to_intel_bo(obj);
4504 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4505
4506 ret = i915_gem_object_pin(obj, 4096);
4507 if (ret)
4508 goto err_unref;
4509
4510 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4511 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4512 if (dev_priv->seqno_page == NULL)
4513 goto err_unpin;
4514
4515 dev_priv->seqno_obj = obj;
4516 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4517
4518 return 0;
4519
4520err_unpin:
4521 i915_gem_object_unpin(obj);
4522err_unref:
4523 drm_gem_object_unreference(obj);
4524err:
4525 return ret;
4526}
4527
8187a2b7
ZN		 4528
4529static void
e552eb70
JB		 4530i915_gem_cleanup_pipe_control(struct drm_device *dev)
4531{
4532 drm_i915_private_t *dev_priv = dev->dev_private;
4533 struct drm_gem_object *obj;
4534 struct drm_i915_gem_object *obj_priv;
4535
4536 obj = dev_priv->seqno_obj;
4537 obj_priv = to_intel_bo(obj);
4538 kunmap(obj_priv->pages[0]);
4539 i915_gem_object_unpin(obj);
4540 drm_gem_object_unreference(obj);
4541 dev_priv->seqno_obj = NULL;
4542
4543 dev_priv->seqno_page = NULL;
673a394b
EA		 4544}
4545
8187a2b7
ZN		 4546int
4547i915_gem_init_ringbuffer(struct drm_device *dev)
4548{
4549 drm_i915_private_t *dev_priv = dev->dev_private;
4550 int ret;
68f95ba9 4551
8187a2b7
ZN		 4552 if (HAS_PIPE_CONTROL(dev)) {
4553 ret = i915_gem_init_pipe_control(dev);
4554 if (ret)
4555 return ret;
4556 }
68f95ba9 4557
5c1143bb 4558 ret = intel_init_render_ring_buffer(dev);
68f95ba9
CW		 4559 if (ret)
4560 goto cleanup_pipe_control;
4561
4562 if (HAS_BSD(dev)) {
5c1143bb 4563 ret = intel_init_bsd_ring_buffer(dev);
68f95ba9
CW		 4564 if (ret)
4565 goto cleanup_render_ring;
d1b851fc 4566 }
68f95ba9 4567
6f392d54
CW		 4568 dev_priv->next_seqno = 1;
4569
68f95ba9
CW		 4570 return 0;
4571
4572cleanup_render_ring:
4573 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4574cleanup_pipe_control:
4575 if (HAS_PIPE_CONTROL(dev))
4576 i915_gem_cleanup_pipe_control(dev);
8187a2b7
ZN		 4577 return ret;
4578}
4579
4580void
4581i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4582{
4583 drm_i915_private_t *dev_priv = dev->dev_private;
4584
4585 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
d1b851fc
ZN		 4586 if (HAS_BSD(dev))
4587 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
8187a2b7
ZN		 4588 if (HAS_PIPE_CONTROL(dev))
4589 i915_gem_cleanup_pipe_control(dev);
4590}
4591
673a394b
EA		 4592int
4593i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4594 struct drm_file *file_priv)
4595{
4596 drm_i915_private_t *dev_priv = dev->dev_private;
4597 int ret;
4598
79e53945
JB		 4599 if (drm_core_check_feature(dev, DRIVER_MODESET))
4600 return 0;
4601
ba1234d1 4602 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4603 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4604 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4605 }
4606
673a394b 4607 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4608 dev_priv->mm.suspended = 0;
4609
4610 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4611 if (ret != 0) {
4612 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4613 return ret;
d816f6ac 4614 }
9bb2d6f9 4615
852835f3 4616 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
d1b851fc 4617 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
673a394b
EA		 4618 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4619 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
852835f3 4620 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
d1b851fc 4621 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
673a394b 4622 mutex_unlock(&dev->struct_mutex);
dbb19d30 4623
5f35308b
CW		 4624 ret = drm_irq_install(dev);
4625 if (ret)
4626 goto cleanup_ringbuffer;
dbb19d30 4627
673a394b 4628 return 0;
5f35308b
CW		 4629
4630cleanup_ringbuffer:
4631 mutex_lock(&dev->struct_mutex);
4632 i915_gem_cleanup_ringbuffer(dev);
4633 dev_priv->mm.suspended = 1;
4634 mutex_unlock(&dev->struct_mutex);
4635
4636 return ret;
673a394b
EA		 4637}
4638
4639int
4640i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4641 struct drm_file *file_priv)
4642{
79e53945
JB		 4643 if (drm_core_check_feature(dev, DRIVER_MODESET))
4644 return 0;
4645
dbb19d30 4646 drm_irq_uninstall(dev);
e6890f6f 4647 return i915_gem_idle(dev);
673a394b
EA		 4648}
4649
4650void
4651i915_gem_lastclose(struct drm_device *dev)
4652{
4653 int ret;
673a394b 4654
e806b495
EA		 4655 if (drm_core_check_feature(dev, DRIVER_MODESET))
4656 return;
4657
6dbe2772
KP		 4658 ret = i915_gem_idle(dev);
4659 if (ret)
4660 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4661}
4662
4663void
4664i915_gem_load(struct drm_device *dev)
4665{
b5aa8a0f 4666 int i;
673a394b
EA		 4667 drm_i915_private_t *dev_priv = dev->dev_private;
4668
673a394b 4669 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
99fcb766 4670 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
673a394b 4671 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
f13d3f73 4672 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
a09ba7fa 4673 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
be72615b 4674 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
852835f3
ZN		 4675 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4676 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
d1b851fc
ZN		 4677 if (HAS_BSD(dev)) {
4678 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4679 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4680 }
007cc8ac
DV		 4681 for (i = 0; i < 16; i++)
4682 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
673a394b
EA		 4683 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4684 i915_gem_retire_work_handler);
30dbf0c0 4685 init_completion(&dev_priv->error_completion);
31169714
CW		 4686 spin_lock(&shrink_list_lock);
4687 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4688 spin_unlock(&shrink_list_lock);
4689
94400120
DA		 4690 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4691 if (IS_GEN3(dev)) {
4692 u32 tmp = I915_READ(MI_ARB_STATE);
4693 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4694 /* arb state is a masked write, so set bit + bit in mask */
4695 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4696 I915_WRITE(MI_ARB_STATE, tmp);
4697 }
4698 }
4699
de151cf6 4700 /* Old X drivers will take 0-2 for front, back, depth buffers */
b397c836
EA		 4701 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4702 dev_priv->fence_reg_start = 3;
de151cf6 4703
a6c45cf0 4704 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4705 dev_priv->num_fence_regs = 16;
4706 else
4707 dev_priv->num_fence_regs = 8;
4708
b5aa8a0f 4709 /* Initialize fence registers to zero */
a6c45cf0
CW		 4710 switch (INTEL_INFO(dev)->gen) {
4711 case 6:
4712 for (i = 0; i < 16; i++)
4713 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4714 break;
4715 case 5:
4716 case 4:
b5aa8a0f
GH		 4717 for (i = 0; i < 16; i++)
4718 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
a6c45cf0
CW		 4719 break;
4720 case 3:
b5aa8a0f
GH		 4721 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4722 for (i = 0; i < 8; i++)
4723 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
a6c45cf0
CW		 4724 case 2:
4725 for (i = 0; i < 8; i++)
4726 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4727 break;
b5aa8a0f 4728 }
673a394b 4729 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4730 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4731}
71acb5eb
DA		 4732
4733/*
4734 * Create a physically contiguous memory object for this object
4735 * e.g. for cursor + overlay regs
4736 */
995b6762
CW		 4737static int i915_gem_init_phys_object(struct drm_device *dev,
4738 int id, int size, int align)
71acb5eb
DA		 4739{
4740 drm_i915_private_t *dev_priv = dev->dev_private;
4741 struct drm_i915_gem_phys_object *phys_obj;
4742 int ret;
4743
4744 if (dev_priv->mm.phys_objs[id - 1] || !size)
4745 return 0;
4746
9a298b2a 4747 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4748 if (!phys_obj)
4749 return -ENOMEM;
4750
4751 phys_obj->id = id;
4752
6eeefaf3 4753 phys_obj->handle = drm_pci_alloc(dev, size, align);
71acb5eb
DA		 4754 if (!phys_obj->handle) {
4755 ret = -ENOMEM;
4756 goto kfree_obj;
4757 }
4758#ifdef CONFIG_X86
4759 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4760#endif
4761
4762 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4763
4764 return 0;
4765kfree_obj:
9a298b2a 4766 kfree(phys_obj);
71acb5eb
DA		 4767 return ret;
4768}
4769
995b6762 4770static void i915_gem_free_phys_object(struct drm_device *dev, int id)
71acb5eb
DA		 4771{
4772 drm_i915_private_t *dev_priv = dev->dev_private;
4773 struct drm_i915_gem_phys_object *phys_obj;
4774
4775 if (!dev_priv->mm.phys_objs[id - 1])
4776 return;
4777
4778 phys_obj = dev_priv->mm.phys_objs[id - 1];
4779 if (phys_obj->cur_obj) {
4780 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4781 }
4782
4783#ifdef CONFIG_X86
4784 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4785#endif
4786 drm_pci_free(dev, phys_obj->handle);
4787 kfree(phys_obj);
4788 dev_priv->mm.phys_objs[id - 1] = NULL;
4789}
4790
4791void i915_gem_free_all_phys_object(struct drm_device *dev)
4792{
4793 int i;
4794
260883c8 4795 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4796 i915_gem_free_phys_object(dev, i);
4797}
4798
4799void i915_gem_detach_phys_object(struct drm_device *dev,
4800 struct drm_gem_object *obj)
4801{
4802 struct drm_i915_gem_object *obj_priv;
4803 int i;
4804 int ret;
4805 int page_count;
4806
23010e43 4807 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4808 if (!obj_priv->phys_obj)
4809 return;
4810
4bdadb97 4811 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4812 if (ret)
4813 goto out;
4814
4815 page_count = obj->size / PAGE_SIZE;
4816
4817 for (i = 0; i < page_count; i++) {
856fa198 4818 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4819 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4820
4821 memcpy(dst, src, PAGE_SIZE);
4822 kunmap_atomic(dst, KM_USER0);
4823 }
856fa198 4824 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4825 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4826
4827 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4828out:
4829 obj_priv->phys_obj->cur_obj = NULL;
4830 obj_priv->phys_obj = NULL;
4831}
4832
4833int
4834i915_gem_attach_phys_object(struct drm_device *dev,
6eeefaf3
CW		 4835 struct drm_gem_object *obj,
4836 int id,
4837 int align)
71acb5eb
DA		 4838{
4839 drm_i915_private_t *dev_priv = dev->dev_private;
4840 struct drm_i915_gem_object *obj_priv;
4841 int ret = 0;
4842 int page_count;
4843 int i;
4844
4845 if (id > I915_MAX_PHYS_OBJECT)
4846 return -EINVAL;
4847
23010e43 4848 obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4849
4850 if (obj_priv->phys_obj) {
4851 if (obj_priv->phys_obj->id == id)
4852 return 0;
4853 i915_gem_detach_phys_object(dev, obj);
4854 }
4855
71acb5eb
DA		 4856 /* create a new object */
4857 if (!dev_priv->mm.phys_objs[id - 1]) {
4858 ret = i915_gem_init_phys_object(dev, id,
6eeefaf3 4859 obj->size, align);
71acb5eb 4860 if (ret) {
aeb565df 4861 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4862 goto out;
4863 }
4864 }
4865
4866 /* bind to the object */
4867 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4868 obj_priv->phys_obj->cur_obj = obj;
4869
4bdadb97 4870 ret = i915_gem_object_get_pages(obj, 0);
71acb5eb
DA		 4871 if (ret) {
4872 DRM_ERROR("failed to get page list\n");
4873 goto out;
4874 }
4875
4876 page_count = obj->size / PAGE_SIZE;
4877
4878 for (i = 0; i < page_count; i++) {
856fa198 4879 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4880 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4881
4882 memcpy(dst, src, PAGE_SIZE);
4883 kunmap_atomic(src, KM_USER0);
4884 }
4885
d78b47b9
CW		 4886 i915_gem_object_put_pages(obj);
4887
71acb5eb
DA		 4888 return 0;
4889out:
4890 return ret;
4891}
4892
4893static int
4894i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4895 struct drm_i915_gem_pwrite *args,
4896 struct drm_file *file_priv)
4897{
23010e43 4898 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
71acb5eb
DA		 4899 void *obj_addr;
4900 int ret;
4901 char __user *user_data;
4902
4903 user_data = (char __user *) (uintptr_t) args->data_ptr;
4904 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4905
44d98a61 4906 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4907 ret = copy_from_user(obj_addr, user_data, args->size);
4908 if (ret)
4909 return -EFAULT;
4910
4911 drm_agp_chipset_flush(dev);
4912 return 0;
4913}
b962442e 4914
f787a5f5 4915void i915_gem_release(struct drm_device *dev, struct drm_file *file)
b962442e 4916{
f787a5f5 4917 struct drm_i915_file_private *file_priv = file->driver_priv;
b962442e
EA		 4918
4919 /* Clean up our request list when the client is going away, so that
4920 * later retire_requests won't dereference our soon-to-be-gone
4921 * file_priv.
4922 */
1c25595f 4923 spin_lock(&file_priv->mm.lock);
f787a5f5
CW		 4924 while (!list_empty(&file_priv->mm.request_list)) {
4925 struct drm_i915_gem_request *request;
4926
4927 request = list_first_entry(&file_priv->mm.request_list,
4928 struct drm_i915_gem_request,
4929 client_list);
4930 list_del(&request->client_list);
4931 request->file_priv = NULL;
4932 }
1c25595f 4933 spin_unlock(&file_priv->mm.lock);
b962442e 4934}
31169714 4935
1637ef41
CW		 4936static int
4937i915_gpu_is_active(struct drm_device *dev)
4938{
4939 drm_i915_private_t *dev_priv = dev->dev_private;
4940 int lists_empty;
4941
1637ef41 4942 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
852835f3 4943 list_empty(&dev_priv->render_ring.active_list);
d1b851fc
ZN		 4944 if (HAS_BSD(dev))
4945 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
1637ef41
CW		 4946
4947 return !lists_empty;
4948}
4949
31169714 4950static int
7f8275d0 4951i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
31169714
CW		 4952{
4953 drm_i915_private_t *dev_priv, *next_dev;
4954 struct drm_i915_gem_object *obj_priv, *next_obj;
4955 int cnt = 0;
4956 int would_deadlock = 1;
4957
4958 /* "fast-path" to count number of available objects */
4959 if (nr_to_scan == 0) {
4960 spin_lock(&shrink_list_lock);
4961 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4962 struct drm_device *dev = dev_priv->dev;
4963
4964 if (mutex_trylock(&dev->struct_mutex)) {
4965 list_for_each_entry(obj_priv,
4966 &dev_priv->mm.inactive_list,
4967 list)
4968 cnt++;
4969 mutex_unlock(&dev->struct_mutex);
4970 }
4971 }
4972 spin_unlock(&shrink_list_lock);
4973
4974 return (cnt / 100) * sysctl_vfs_cache_pressure;
4975 }
4976
4977 spin_lock(&shrink_list_lock);
4978
1637ef41 4979rescan:
31169714
CW		 4980 /* first scan for clean buffers */
4981 list_for_each_entry_safe(dev_priv, next_dev,
4982 &shrink_list, mm.shrink_list) {
4983 struct drm_device *dev = dev_priv->dev;
4984
4985 if (! mutex_trylock(&dev->struct_mutex))
4986 continue;
4987
4988 spin_unlock(&shrink_list_lock);
b09a1fec 4989 i915_gem_retire_requests(dev);
31169714
CW		 4990
4991 list_for_each_entry_safe(obj_priv, next_obj,
4992 &dev_priv->mm.inactive_list,
4993 list) {
4994 if (i915_gem_object_is_purgeable(obj_priv)) {
a8089e84 4995 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 4996 if (--nr_to_scan <= 0)
4997 break;
4998 }
4999 }
5000
5001 spin_lock(&shrink_list_lock);
5002 mutex_unlock(&dev->struct_mutex);
5003
963b4836
CW		 5004 would_deadlock = 0;
5005
31169714
CW		 5006 if (nr_to_scan <= 0)
5007 break;
5008 }
5009
5010 /* second pass, evict/count anything still on the inactive list */
5011 list_for_each_entry_safe(dev_priv, next_dev,
5012 &shrink_list, mm.shrink_list) {
5013 struct drm_device *dev = dev_priv->dev;
5014
5015 if (! mutex_trylock(&dev->struct_mutex))
5016 continue;
5017
5018 spin_unlock(&shrink_list_lock);
5019
5020 list_for_each_entry_safe(obj_priv, next_obj,
5021 &dev_priv->mm.inactive_list,
5022 list) {
5023 if (nr_to_scan > 0) {
a8089e84 5024 i915_gem_object_unbind(&obj_priv->base);
31169714
CW		 5025 nr_to_scan--;
5026 } else
5027 cnt++;
5028 }
5029
5030 spin_lock(&shrink_list_lock);
5031 mutex_unlock(&dev->struct_mutex);
5032
5033 would_deadlock = 0;
5034 }
5035
1637ef41
CW		 5036 if (nr_to_scan) {
5037 int active = 0;
5038
5039 /*
5040 * We are desperate for pages, so as a last resort, wait
5041 * for the GPU to finish and discard whatever we can.
5042 * This has a dramatic impact to reduce the number of
5043 * OOM-killer events whilst running the GPU aggressively.
5044 */
5045 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5046 struct drm_device *dev = dev_priv->dev;
5047
5048 if (!mutex_trylock(&dev->struct_mutex))
5049 continue;
5050
5051 spin_unlock(&shrink_list_lock);
5052
5053 if (i915_gpu_is_active(dev)) {
5054 i915_gpu_idle(dev);
5055 active++;
5056 }
5057
5058 spin_lock(&shrink_list_lock);
5059 mutex_unlock(&dev->struct_mutex);
5060 }
5061
5062 if (active)
5063 goto rescan;
5064 }
5065
31169714
CW		 5066 spin_unlock(&shrink_list_lock);
5067
5068 if (would_deadlock)
5069 return -1;
5070 else if (cnt > 0)
5071 return (cnt / 100) * sysctl_vfs_cache_pressure;
5072 else
5073 return 0;
5074}
5075
5076static struct shrinker shrinker = {
5077 .shrink = i915_gem_shrink,
5078 .seeks = DEFAULT_SEEKS,
5079};
5080
5081__init void
5082i915_gem_shrinker_init(void)
5083{
5084 register_shrinker(&shrinker);
5085}
5086
5087__exit void
5088i915_gem_shrinker_exit(void)
5089{
5090 unregister_shrinker(&shrinker);
5091}
Clone of davem's net-next-2.6 tree