]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/gpu/drm/i915/i915_gem.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: Set the error code after failing to insert new offset into mm ht.
[net-next-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
1c5d22f7 32#include "i915_trace.h"
652c393a 33#include "intel_drv.h"
673a394b 34#include <linux/swap.h>
79e53945 35#include <linux/pci.h>
673a394b 36
28dfe52a
EA		 37#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
38
e47c68e9
EA		 39static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
40static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
41static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 42static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
43 int write);
44static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
45 uint64_t offset,
46 uint64_t size);
47static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
673a394b 48static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
de151cf6
JB		 49static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
50 unsigned alignment);
de151cf6 51static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
07f73f69 52static int i915_gem_evict_something(struct drm_device *dev, int min_size);
ab5ee576 53static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
71acb5eb
DA		 54static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
55 struct drm_i915_gem_pwrite *args,
56 struct drm_file *file_priv);
673a394b 57
31169714
CW		 58static LIST_HEAD(shrink_list);
59static DEFINE_SPINLOCK(shrink_list_lock);
60
79e53945
JB		 61int i915_gem_do_init(struct drm_device *dev, unsigned long start,
62 unsigned long end)
673a394b
EA		 63{
64 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 65
79e53945
JB		 66 if (start >= end ||
67 (start & (PAGE_SIZE - 1)) != 0 ||
68 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 69 return -EINVAL;
70 }
71
79e53945
JB		 72 drm_mm_init(&dev_priv->mm.gtt_space, start,
73 end - start);
673a394b 74
79e53945
JB		 75 dev->gtt_total = (uint32_t) (end - start);
76
77 return 0;
78}
673a394b 79
79e53945
JB		 80int
81i915_gem_init_ioctl(struct drm_device *dev, void *data,
82 struct drm_file *file_priv)
83{
84 struct drm_i915_gem_init *args = data;
85 int ret;
86
87 mutex_lock(&dev->struct_mutex);
88 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 89 mutex_unlock(&dev->struct_mutex);
90
79e53945 91 return ret;
673a394b
EA		 92}
93
5a125c3c
EA		 94int
95i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
96 struct drm_file *file_priv)
97{
5a125c3c 98 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 99
100 if (!(dev->driver->driver_features & DRIVER_GEM))
101 return -ENODEV;
102
103 args->aper_size = dev->gtt_total;
2678d9d6
KP		 104 args->aper_available_size = (args->aper_size -
105 atomic_read(&dev->pin_memory));
5a125c3c
EA		 106
107 return 0;
108}
109
673a394b
EA		 110
111/**
112 * Creates a new mm object and returns a handle to it.
113 */
114int
115i915_gem_create_ioctl(struct drm_device *dev, void *data,
116 struct drm_file *file_priv)
117{
118 struct drm_i915_gem_create *args = data;
119 struct drm_gem_object *obj;
a1a2d1d3
PP		 120 int ret;
121 u32 handle;
673a394b
EA		 122
123 args->size = roundup(args->size, PAGE_SIZE);
124
125 /* Allocate the new object */
126 obj = drm_gem_object_alloc(dev, args->size);
127 if (obj == NULL)
128 return -ENOMEM;
129
130 ret = drm_gem_handle_create(file_priv, obj, &handle);
131 mutex_lock(&dev->struct_mutex);
132 drm_gem_object_handle_unreference(obj);
133 mutex_unlock(&dev->struct_mutex);
134
135 if (ret)
136 return ret;
137
138 args->handle = handle;
139
140 return 0;
141}
142
eb01459f
EA		 143static inline int
144fast_shmem_read(struct page **pages,
145 loff_t page_base, int page_offset,
146 char __user *data,
147 int length)
148{
149 char __iomem *vaddr;
2bc43b5c 150 int unwritten;
eb01459f
EA		 151
152 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
153 if (vaddr == NULL)
154 return -ENOMEM;
2bc43b5c 155 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 156 kunmap_atomic(vaddr, KM_USER0);
157
2bc43b5c
FM		 158 if (unwritten)
159 return -EFAULT;
160
161 return 0;
eb01459f
EA		 162}
163
280b713b
EA		 164static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
165{
166 drm_i915_private_t *dev_priv = obj->dev->dev_private;
167 struct drm_i915_gem_object *obj_priv = obj->driver_private;
168
169 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
170 obj_priv->tiling_mode != I915_TILING_NONE;
171}
172
40123c1f
EA		 173static inline int
174slow_shmem_copy(struct page *dst_page,
175 int dst_offset,
176 struct page *src_page,
177 int src_offset,
178 int length)
179{
180 char *dst_vaddr, *src_vaddr;
181
182 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
183 if (dst_vaddr == NULL)
184 return -ENOMEM;
185
186 src_vaddr = kmap_atomic(src_page, KM_USER1);
187 if (src_vaddr == NULL) {
188 kunmap_atomic(dst_vaddr, KM_USER0);
189 return -ENOMEM;
190 }
191
192 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
193
194 kunmap_atomic(src_vaddr, KM_USER1);
195 kunmap_atomic(dst_vaddr, KM_USER0);
196
197 return 0;
198}
199
280b713b
EA		 200static inline int
201slow_shmem_bit17_copy(struct page *gpu_page,
202 int gpu_offset,
203 struct page *cpu_page,
204 int cpu_offset,
205 int length,
206 int is_read)
207{
208 char *gpu_vaddr, *cpu_vaddr;
209
210 /* Use the unswizzled path if this page isn't affected. */
211 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
212 if (is_read)
213 return slow_shmem_copy(cpu_page, cpu_offset,
214 gpu_page, gpu_offset, length);
215 else
216 return slow_shmem_copy(gpu_page, gpu_offset,
217 cpu_page, cpu_offset, length);
218 }
219
220 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
221 if (gpu_vaddr == NULL)
222 return -ENOMEM;
223
224 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
225 if (cpu_vaddr == NULL) {
226 kunmap_atomic(gpu_vaddr, KM_USER0);
227 return -ENOMEM;
228 }
229
230 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
231 * XORing with the other bits (A9 for Y, A9 and A10 for X)
232 */
233 while (length > 0) {
234 int cacheline_end = ALIGN(gpu_offset + 1, 64);
235 int this_length = min(cacheline_end - gpu_offset, length);
236 int swizzled_gpu_offset = gpu_offset ^ 64;
237
238 if (is_read) {
239 memcpy(cpu_vaddr + cpu_offset,
240 gpu_vaddr + swizzled_gpu_offset,
241 this_length);
242 } else {
243 memcpy(gpu_vaddr + swizzled_gpu_offset,
244 cpu_vaddr + cpu_offset,
245 this_length);
246 }
247 cpu_offset += this_length;
248 gpu_offset += this_length;
249 length -= this_length;
250 }
251
252 kunmap_atomic(cpu_vaddr, KM_USER1);
253 kunmap_atomic(gpu_vaddr, KM_USER0);
254
255 return 0;
256}
257
eb01459f
EA		 258/**
259 * This is the fast shmem pread path, which attempts to copy_from_user directly
260 * from the backing pages of the object to the user's address space. On a
261 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
262 */
263static int
264i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
265 struct drm_i915_gem_pread *args,
266 struct drm_file *file_priv)
267{
268 struct drm_i915_gem_object *obj_priv = obj->driver_private;
269 ssize_t remain;
270 loff_t offset, page_base;
271 char __user *user_data;
272 int page_offset, page_length;
273 int ret;
274
275 user_data = (char __user *) (uintptr_t) args->data_ptr;
276 remain = args->size;
277
278 mutex_lock(&dev->struct_mutex);
279
280 ret = i915_gem_object_get_pages(obj);
281 if (ret != 0)
282 goto fail_unlock;
283
284 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
285 args->size);
286 if (ret != 0)
287 goto fail_put_pages;
288
289 obj_priv = obj->driver_private;
290 offset = args->offset;
291
292 while (remain > 0) {
293 /* Operation in this page
294 *
295 * page_base = page offset within aperture
296 * page_offset = offset within page
297 * page_length = bytes to copy for this page
298 */
299 page_base = (offset & ~(PAGE_SIZE-1));
300 page_offset = offset & (PAGE_SIZE-1);
301 page_length = remain;
302 if ((page_offset + remain) > PAGE_SIZE)
303 page_length = PAGE_SIZE - page_offset;
304
305 ret = fast_shmem_read(obj_priv->pages,
306 page_base, page_offset,
307 user_data, page_length);
308 if (ret)
309 goto fail_put_pages;
310
311 remain -= page_length;
312 user_data += page_length;
313 offset += page_length;
314 }
315
316fail_put_pages:
317 i915_gem_object_put_pages(obj);
318fail_unlock:
319 mutex_unlock(&dev->struct_mutex);
320
321 return ret;
322}
323
07f73f69
CW		 324static inline gfp_t
325i915_gem_object_get_page_gfp_mask (struct drm_gem_object *obj)
326{
327 return mapping_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping);
328}
329
330static inline void
331i915_gem_object_set_page_gfp_mask (struct drm_gem_object *obj, gfp_t gfp)
332{
333 mapping_set_gfp_mask(obj->filp->f_path.dentry->d_inode->i_mapping, gfp);
334}
335
336static int
337i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
338{
339 int ret;
340
341 ret = i915_gem_object_get_pages(obj);
342
343 /* If we've insufficient memory to map in the pages, attempt
344 * to make some space by throwing out some old buffers.
345 */
346 if (ret == -ENOMEM) {
347 struct drm_device *dev = obj->dev;
348 gfp_t gfp;
349
350 ret = i915_gem_evict_something(dev, obj->size);
351 if (ret)
352 return ret;
353
354 gfp = i915_gem_object_get_page_gfp_mask(obj);
355 i915_gem_object_set_page_gfp_mask(obj, gfp & ~__GFP_NORETRY);
356 ret = i915_gem_object_get_pages(obj);
357 i915_gem_object_set_page_gfp_mask (obj, gfp);
358 }
359
360 return ret;
361}
362
eb01459f
EA		 363/**
364 * This is the fallback shmem pread path, which allocates temporary storage
365 * in kernel space to copy_to_user into outside of the struct_mutex, so we
366 * can copy out of the object's backing pages while holding the struct mutex
367 * and not take page faults.
368 */
369static int
370i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
371 struct drm_i915_gem_pread *args,
372 struct drm_file *file_priv)
373{
374 struct drm_i915_gem_object *obj_priv = obj->driver_private;
375 struct mm_struct *mm = current->mm;
376 struct page **user_pages;
377 ssize_t remain;
378 loff_t offset, pinned_pages, i;
379 loff_t first_data_page, last_data_page, num_pages;
380 int shmem_page_index, shmem_page_offset;
381 int data_page_index, data_page_offset;
382 int page_length;
383 int ret;
384 uint64_t data_ptr = args->data_ptr;
280b713b 385 int do_bit17_swizzling;
eb01459f
EA		 386
387 remain = args->size;
388
389 /* Pin the user pages containing the data. We can't fault while
390 * holding the struct mutex, yet we want to hold it while
391 * dereferencing the user data.
392 */
393 first_data_page = data_ptr / PAGE_SIZE;
394 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
395 num_pages = last_data_page - first_data_page + 1;
396
8e7d2b2c 397 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 398 if (user_pages == NULL)
399 return -ENOMEM;
400
401 down_read(&mm->mmap_sem);
402 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 403 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 404 up_read(&mm->mmap_sem);
405 if (pinned_pages < num_pages) {
406 ret = -EFAULT;
407 goto fail_put_user_pages;
408 }
409
280b713b
EA		 410 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
411
eb01459f
EA		 412 mutex_lock(&dev->struct_mutex);
413
07f73f69
CW		 414 ret = i915_gem_object_get_pages_or_evict(obj);
415 if (ret)
eb01459f
EA		 416 goto fail_unlock;
417
418 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
419 args->size);
420 if (ret != 0)
421 goto fail_put_pages;
422
423 obj_priv = obj->driver_private;
424 offset = args->offset;
425
426 while (remain > 0) {
427 /* Operation in this page
428 *
429 * shmem_page_index = page number within shmem file
430 * shmem_page_offset = offset within page in shmem file
431 * data_page_index = page number in get_user_pages return
432 * data_page_offset = offset with data_page_index page.
433 * page_length = bytes to copy for this page
434 */
435 shmem_page_index = offset / PAGE_SIZE;
436 shmem_page_offset = offset & ~PAGE_MASK;
437 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
438 data_page_offset = data_ptr & ~PAGE_MASK;
439
440 page_length = remain;
441 if ((shmem_page_offset + page_length) > PAGE_SIZE)
442 page_length = PAGE_SIZE - shmem_page_offset;
443 if ((data_page_offset + page_length) > PAGE_SIZE)
444 page_length = PAGE_SIZE - data_page_offset;
445
280b713b
EA		 446 if (do_bit17_swizzling) {
447 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
448 shmem_page_offset,
449 user_pages[data_page_index],
450 data_page_offset,
451 page_length,
452 1);
453 } else {
454 ret = slow_shmem_copy(user_pages[data_page_index],
455 data_page_offset,
456 obj_priv->pages[shmem_page_index],
457 shmem_page_offset,
458 page_length);
459 }
eb01459f
EA		 460 if (ret)
461 goto fail_put_pages;
462
463 remain -= page_length;
464 data_ptr += page_length;
465 offset += page_length;
466 }
467
468fail_put_pages:
469 i915_gem_object_put_pages(obj);
470fail_unlock:
471 mutex_unlock(&dev->struct_mutex);
472fail_put_user_pages:
473 for (i = 0; i < pinned_pages; i++) {
474 SetPageDirty(user_pages[i]);
475 page_cache_release(user_pages[i]);
476 }
8e7d2b2c 477 drm_free_large(user_pages);
eb01459f
EA		 478
479 return ret;
480}
481
673a394b
EA		 482/**
483 * Reads data from the object referenced by handle.
484 *
485 * On error, the contents of *data are undefined.
486 */
487int
488i915_gem_pread_ioctl(struct drm_device *dev, void *data,
489 struct drm_file *file_priv)
490{
491 struct drm_i915_gem_pread *args = data;
492 struct drm_gem_object *obj;
493 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 494 int ret;
495
496 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
497 if (obj == NULL)
498 return -EBADF;
499 obj_priv = obj->driver_private;
500
501 /* Bounds check source.
502 *
503 * XXX: This could use review for overflow issues...
504 */
505 if (args->offset > obj->size || args->size > obj->size ||
506 args->offset + args->size > obj->size) {
507 drm_gem_object_unreference(obj);
508 return -EINVAL;
509 }
510
280b713b 511 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 512 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 513 } else {
514 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
515 if (ret != 0)
516 ret = i915_gem_shmem_pread_slow(dev, obj, args,
517 file_priv);
518 }
673a394b
EA		 519
520 drm_gem_object_unreference(obj);
673a394b 521
eb01459f 522 return ret;
673a394b
EA		 523}
524
0839ccb8
KP		 525/* This is the fast write path which cannot handle
526 * page faults in the source data
9b7530cc 527 */
0839ccb8
KP		 528
529static inline int
530fast_user_write(struct io_mapping *mapping,
531 loff_t page_base, int page_offset,
532 char __user *user_data,
533 int length)
9b7530cc 534{
9b7530cc 535 char *vaddr_atomic;
0839ccb8 536 unsigned long unwritten;
9b7530cc 537
0839ccb8
KP		 538 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
539 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
540 user_data, length);
541 io_mapping_unmap_atomic(vaddr_atomic);
542 if (unwritten)
543 return -EFAULT;
544 return 0;
545}
546
547/* Here's the write path which can sleep for
548 * page faults
549 */
550
551static inline int
3de09aa3
EA		 552slow_kernel_write(struct io_mapping *mapping,
553 loff_t gtt_base, int gtt_offset,
554 struct page *user_page, int user_offset,
555 int length)
0839ccb8 556{
3de09aa3 557 char *src_vaddr, *dst_vaddr;
0839ccb8
KP		 558 unsigned long unwritten;
559
3de09aa3
EA		 560 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
561 src_vaddr = kmap_atomic(user_page, KM_USER1);
562 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
563 src_vaddr + user_offset,
564 length);
565 kunmap_atomic(src_vaddr, KM_USER1);
566 io_mapping_unmap_atomic(dst_vaddr);
0839ccb8
KP		 567 if (unwritten)
568 return -EFAULT;
9b7530cc 569 return 0;
9b7530cc
LT		 570}
571
40123c1f
EA		 572static inline int
573fast_shmem_write(struct page **pages,
574 loff_t page_base, int page_offset,
575 char __user *data,
576 int length)
577{
578 char __iomem *vaddr;
d0088775 579 unsigned long unwritten;
40123c1f
EA		 580
581 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
582 if (vaddr == NULL)
583 return -ENOMEM;
d0088775 584 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 585 kunmap_atomic(vaddr, KM_USER0);
586
d0088775
DA		 587 if (unwritten)
588 return -EFAULT;
40123c1f
EA		 589 return 0;
590}
591
3de09aa3
EA		 592/**
593 * This is the fast pwrite path, where we copy the data directly from the
594 * user into the GTT, uncached.
595 */
673a394b 596static int
3de09aa3
EA		 597i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
598 struct drm_i915_gem_pwrite *args,
599 struct drm_file *file_priv)
673a394b
EA		 600{
601 struct drm_i915_gem_object *obj_priv = obj->driver_private;
0839ccb8 602 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 603 ssize_t remain;
0839ccb8 604 loff_t offset, page_base;
673a394b 605 char __user *user_data;
0839ccb8
KP		 606 int page_offset, page_length;
607 int ret;
673a394b
EA		 608
609 user_data = (char __user *) (uintptr_t) args->data_ptr;
610 remain = args->size;
611 if (!access_ok(VERIFY_READ, user_data, remain))
612 return -EFAULT;
613
614
615 mutex_lock(&dev->struct_mutex);
616 ret = i915_gem_object_pin(obj, 0);
617 if (ret) {
618 mutex_unlock(&dev->struct_mutex);
619 return ret;
620 }
2ef7eeaa 621 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 622 if (ret)
623 goto fail;
624
625 obj_priv = obj->driver_private;
626 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 627
628 while (remain > 0) {
629 /* Operation in this page
630 *
0839ccb8
KP		 631 * page_base = page offset within aperture
632 * page_offset = offset within page
633 * page_length = bytes to copy for this page
673a394b 634 */
0839ccb8
KP		 635 page_base = (offset & ~(PAGE_SIZE-1));
636 page_offset = offset & (PAGE_SIZE-1);
637 page_length = remain;
638 if ((page_offset + remain) > PAGE_SIZE)
639 page_length = PAGE_SIZE - page_offset;
640
641 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
642 page_offset, user_data, page_length);
643
644 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 645 * source page isn't available. Return the error and we'll
646 * retry in the slow path.
0839ccb8 647 */
3de09aa3
EA		 648 if (ret)
649 goto fail;
673a394b 650
0839ccb8
KP		 651 remain -= page_length;
652 user_data += page_length;
653 offset += page_length;
673a394b 654 }
673a394b
EA		 655
656fail:
657 i915_gem_object_unpin(obj);
658 mutex_unlock(&dev->struct_mutex);
659
660 return ret;
661}
662
3de09aa3
EA		 663/**
664 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
665 * the memory and maps it using kmap_atomic for copying.
666 *
667 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
668 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
669 */
3043c60c 670static int
3de09aa3
EA		 671i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
672 struct drm_i915_gem_pwrite *args,
673 struct drm_file *file_priv)
673a394b 674{
3de09aa3
EA		 675 struct drm_i915_gem_object *obj_priv = obj->driver_private;
676 drm_i915_private_t *dev_priv = dev->dev_private;
677 ssize_t remain;
678 loff_t gtt_page_base, offset;
679 loff_t first_data_page, last_data_page, num_pages;
680 loff_t pinned_pages, i;
681 struct page **user_pages;
682 struct mm_struct *mm = current->mm;
683 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 684 int ret;
3de09aa3
EA		 685 uint64_t data_ptr = args->data_ptr;
686
687 remain = args->size;
688
689 /* Pin the user pages containing the data. We can't fault while
690 * holding the struct mutex, and all of the pwrite implementations
691 * want to hold it while dereferencing the user data.
692 */
693 first_data_page = data_ptr / PAGE_SIZE;
694 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
695 num_pages = last_data_page - first_data_page + 1;
696
8e7d2b2c 697 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 698 if (user_pages == NULL)
699 return -ENOMEM;
700
701 down_read(&mm->mmap_sem);
702 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
703 num_pages, 0, 0, user_pages, NULL);
704 up_read(&mm->mmap_sem);
705 if (pinned_pages < num_pages) {
706 ret = -EFAULT;
707 goto out_unpin_pages;
708 }
673a394b
EA		 709
710 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 711 ret = i915_gem_object_pin(obj, 0);
712 if (ret)
713 goto out_unlock;
714
715 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
716 if (ret)
717 goto out_unpin_object;
718
719 obj_priv = obj->driver_private;
720 offset = obj_priv->gtt_offset + args->offset;
721
722 while (remain > 0) {
723 /* Operation in this page
724 *
725 * gtt_page_base = page offset within aperture
726 * gtt_page_offset = offset within page in aperture
727 * data_page_index = page number in get_user_pages return
728 * data_page_offset = offset with data_page_index page.
729 * page_length = bytes to copy for this page
730 */
731 gtt_page_base = offset & PAGE_MASK;
732 gtt_page_offset = offset & ~PAGE_MASK;
733 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
734 data_page_offset = data_ptr & ~PAGE_MASK;
735
736 page_length = remain;
737 if ((gtt_page_offset + page_length) > PAGE_SIZE)
738 page_length = PAGE_SIZE - gtt_page_offset;
739 if ((data_page_offset + page_length) > PAGE_SIZE)
740 page_length = PAGE_SIZE - data_page_offset;
741
742 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
743 gtt_page_base, gtt_page_offset,
744 user_pages[data_page_index],
745 data_page_offset,
746 page_length);
747
748 /* If we get a fault while copying data, then (presumably) our
749 * source page isn't available. Return the error and we'll
750 * retry in the slow path.
751 */
752 if (ret)
753 goto out_unpin_object;
754
755 remain -= page_length;
756 offset += page_length;
757 data_ptr += page_length;
758 }
759
760out_unpin_object:
761 i915_gem_object_unpin(obj);
762out_unlock:
763 mutex_unlock(&dev->struct_mutex);
764out_unpin_pages:
765 for (i = 0; i < pinned_pages; i++)
766 page_cache_release(user_pages[i]);
8e7d2b2c 767 drm_free_large(user_pages);
3de09aa3
EA		 768
769 return ret;
770}
771
40123c1f
EA		 772/**
773 * This is the fast shmem pwrite path, which attempts to directly
774 * copy_from_user into the kmapped pages backing the object.
775 */
3043c60c 776static int
40123c1f
EA		 777i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
778 struct drm_i915_gem_pwrite *args,
779 struct drm_file *file_priv)
673a394b 780{
40123c1f
EA		 781 struct drm_i915_gem_object *obj_priv = obj->driver_private;
782 ssize_t remain;
783 loff_t offset, page_base;
784 char __user *user_data;
785 int page_offset, page_length;
673a394b 786 int ret;
40123c1f
EA		 787
788 user_data = (char __user *) (uintptr_t) args->data_ptr;
789 remain = args->size;
673a394b
EA		 790
791 mutex_lock(&dev->struct_mutex);
792
40123c1f
EA		 793 ret = i915_gem_object_get_pages(obj);
794 if (ret != 0)
795 goto fail_unlock;
673a394b 796
e47c68e9 797 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 798 if (ret != 0)
799 goto fail_put_pages;
800
801 obj_priv = obj->driver_private;
802 offset = args->offset;
803 obj_priv->dirty = 1;
804
805 while (remain > 0) {
806 /* Operation in this page
807 *
808 * page_base = page offset within aperture
809 * page_offset = offset within page
810 * page_length = bytes to copy for this page
811 */
812 page_base = (offset & ~(PAGE_SIZE-1));
813 page_offset = offset & (PAGE_SIZE-1);
814 page_length = remain;
815 if ((page_offset + remain) > PAGE_SIZE)
816 page_length = PAGE_SIZE - page_offset;
817
818 ret = fast_shmem_write(obj_priv->pages,
819 page_base, page_offset,
820 user_data, page_length);
821 if (ret)
822 goto fail_put_pages;
823
824 remain -= page_length;
825 user_data += page_length;
826 offset += page_length;
827 }
828
829fail_put_pages:
830 i915_gem_object_put_pages(obj);
831fail_unlock:
832 mutex_unlock(&dev->struct_mutex);
833
834 return ret;
835}
836
837/**
838 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
839 * the memory and maps it using kmap_atomic for copying.
840 *
841 * This avoids taking mmap_sem for faulting on the user's address while the
842 * struct_mutex is held.
843 */
844static int
845i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
846 struct drm_i915_gem_pwrite *args,
847 struct drm_file *file_priv)
848{
849 struct drm_i915_gem_object *obj_priv = obj->driver_private;
850 struct mm_struct *mm = current->mm;
851 struct page **user_pages;
852 ssize_t remain;
853 loff_t offset, pinned_pages, i;
854 loff_t first_data_page, last_data_page, num_pages;
855 int shmem_page_index, shmem_page_offset;
856 int data_page_index, data_page_offset;
857 int page_length;
858 int ret;
859 uint64_t data_ptr = args->data_ptr;
280b713b 860 int do_bit17_swizzling;
40123c1f
EA		 861
862 remain = args->size;
863
864 /* Pin the user pages containing the data. We can't fault while
865 * holding the struct mutex, and all of the pwrite implementations
866 * want to hold it while dereferencing the user data.
867 */
868 first_data_page = data_ptr / PAGE_SIZE;
869 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
870 num_pages = last_data_page - first_data_page + 1;
871
8e7d2b2c 872 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 873 if (user_pages == NULL)
874 return -ENOMEM;
875
876 down_read(&mm->mmap_sem);
877 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
878 num_pages, 0, 0, user_pages, NULL);
879 up_read(&mm->mmap_sem);
880 if (pinned_pages < num_pages) {
881 ret = -EFAULT;
882 goto fail_put_user_pages;
673a394b
EA		 883 }
884
280b713b
EA		 885 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
886
40123c1f
EA		 887 mutex_lock(&dev->struct_mutex);
888
07f73f69
CW		 889 ret = i915_gem_object_get_pages_or_evict(obj);
890 if (ret)
40123c1f
EA		 891 goto fail_unlock;
892
893 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
894 if (ret != 0)
895 goto fail_put_pages;
896
897 obj_priv = obj->driver_private;
673a394b 898 offset = args->offset;
40123c1f 899 obj_priv->dirty = 1;
673a394b 900
40123c1f
EA		 901 while (remain > 0) {
902 /* Operation in this page
903 *
904 * shmem_page_index = page number within shmem file
905 * shmem_page_offset = offset within page in shmem file
906 * data_page_index = page number in get_user_pages return
907 * data_page_offset = offset with data_page_index page.
908 * page_length = bytes to copy for this page
909 */
910 shmem_page_index = offset / PAGE_SIZE;
911 shmem_page_offset = offset & ~PAGE_MASK;
912 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
913 data_page_offset = data_ptr & ~PAGE_MASK;
914
915 page_length = remain;
916 if ((shmem_page_offset + page_length) > PAGE_SIZE)
917 page_length = PAGE_SIZE - shmem_page_offset;
918 if ((data_page_offset + page_length) > PAGE_SIZE)
919 page_length = PAGE_SIZE - data_page_offset;
920
280b713b
EA		 921 if (do_bit17_swizzling) {
922 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
923 shmem_page_offset,
924 user_pages[data_page_index],
925 data_page_offset,
926 page_length,
927 0);
928 } else {
929 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
930 shmem_page_offset,
931 user_pages[data_page_index],
932 data_page_offset,
933 page_length);
934 }
40123c1f
EA		 935 if (ret)
936 goto fail_put_pages;
937
938 remain -= page_length;
939 data_ptr += page_length;
940 offset += page_length;
673a394b
EA		 941 }
942
40123c1f
EA		 943fail_put_pages:
944 i915_gem_object_put_pages(obj);
945fail_unlock:
673a394b 946 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 947fail_put_user_pages:
948 for (i = 0; i < pinned_pages; i++)
949 page_cache_release(user_pages[i]);
8e7d2b2c 950 drm_free_large(user_pages);
673a394b 951
40123c1f 952 return ret;
673a394b
EA		 953}
954
955/**
956 * Writes data to the object referenced by handle.
957 *
958 * On error, the contents of the buffer that were to be modified are undefined.
959 */
960int
961i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
962 struct drm_file *file_priv)
963{
964 struct drm_i915_gem_pwrite *args = data;
965 struct drm_gem_object *obj;
966 struct drm_i915_gem_object *obj_priv;
967 int ret = 0;
968
969 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
970 if (obj == NULL)
971 return -EBADF;
972 obj_priv = obj->driver_private;
973
974 /* Bounds check destination.
975 *
976 * XXX: This could use review for overflow issues...
977 */
978 if (args->offset > obj->size || args->size > obj->size ||
979 args->offset + args->size > obj->size) {
980 drm_gem_object_unreference(obj);
981 return -EINVAL;
982 }
983
984 /* We can only do the GTT pwrite on untiled buffers, as otherwise
985 * it would end up going through the fenced access, and we'll get
986 * different detiling behavior between reading and writing.
987 * pread/pwrite currently are reading and writing from the CPU
988 * perspective, requiring manual detiling by the client.
989 */
71acb5eb
DA		 990 if (obj_priv->phys_obj)
991 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
992 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
3de09aa3
EA		 993 dev->gtt_total != 0) {
994 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
995 if (ret == -EFAULT) {
996 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
997 file_priv);
998 }
280b713b
EA		 999 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
1000 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 1001 } else {
1002 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
1003 if (ret == -EFAULT) {
1004 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
1005 file_priv);
1006 }
1007 }
673a394b
EA		 1008
1009#if WATCH_PWRITE
1010 if (ret)
1011 DRM_INFO("pwrite failed %d\n", ret);
1012#endif
1013
1014 drm_gem_object_unreference(obj);
1015
1016 return ret;
1017}
1018
1019/**
2ef7eeaa
EA		 1020 * Called when user space prepares to use an object with the CPU, either
1021 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 1022 */
1023int
1024i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1025 struct drm_file *file_priv)
1026{
a09ba7fa 1027 struct drm_i915_private *dev_priv = dev->dev_private;
673a394b
EA		 1028 struct drm_i915_gem_set_domain *args = data;
1029 struct drm_gem_object *obj;
652c393a 1030 struct drm_i915_gem_object *obj_priv;
2ef7eeaa
EA		 1031 uint32_t read_domains = args->read_domains;
1032 uint32_t write_domain = args->write_domain;
673a394b
EA		 1033 int ret;
1034
1035 if (!(dev->driver->driver_features & DRIVER_GEM))
1036 return -ENODEV;
1037
2ef7eeaa 1038 /* Only handle setting domains to types used by the CPU. */
21d509e3 1039 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1040 return -EINVAL;
1041
21d509e3 1042 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 1043 return -EINVAL;
1044
1045 /* Having something in the write domain implies it's in the read
1046 * domain, and only that read domain. Enforce that in the request.
1047 */
1048 if (write_domain != 0 && read_domains != write_domain)
1049 return -EINVAL;
1050
673a394b
EA		 1051 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1052 if (obj == NULL)
1053 return -EBADF;
652c393a 1054 obj_priv = obj->driver_private;
673a394b
EA		 1055
1056 mutex_lock(&dev->struct_mutex);
652c393a
JB		 1057
1058 intel_mark_busy(dev, obj);
1059
673a394b 1060#if WATCH_BUF
cfd43c02 1061 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
2ef7eeaa 1062 obj, obj->size, read_domains, write_domain);
673a394b 1063#endif
2ef7eeaa
EA		 1064 if (read_domains & I915_GEM_DOMAIN_GTT) {
1065 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392 1066
a09ba7fa
EA		 1067 /* Update the LRU on the fence for the CPU access that's
1068 * about to occur.
1069 */
1070 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1071 list_move_tail(&obj_priv->fence_list,
1072 &dev_priv->mm.fence_list);
1073 }
1074
02354392
EA		 1075 /* Silently promote "you're not bound, there was nothing to do"
1076 * to success, since the client was just asking us to
1077 * make sure everything was done.
1078 */
1079 if (ret == -EINVAL)
1080 ret = 0;
2ef7eeaa 1081 } else {
e47c68e9 1082 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1083 }
1084
673a394b
EA		 1085 drm_gem_object_unreference(obj);
1086 mutex_unlock(&dev->struct_mutex);
1087 return ret;
1088}
1089
1090/**
1091 * Called when user space has done writes to this buffer
1092 */
1093int
1094i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1095 struct drm_file *file_priv)
1096{
1097 struct drm_i915_gem_sw_finish *args = data;
1098 struct drm_gem_object *obj;
1099 struct drm_i915_gem_object *obj_priv;
1100 int ret = 0;
1101
1102 if (!(dev->driver->driver_features & DRIVER_GEM))
1103 return -ENODEV;
1104
1105 mutex_lock(&dev->struct_mutex);
1106 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1107 if (obj == NULL) {
1108 mutex_unlock(&dev->struct_mutex);
1109 return -EBADF;
1110 }
1111
1112#if WATCH_BUF
cfd43c02 1113 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
673a394b
EA		 1114 __func__, args->handle, obj, obj->size);
1115#endif
1116 obj_priv = obj->driver_private;
1117
1118 /* Pinned buffers may be scanout, so flush the cache */
e47c68e9
EA		 1119 if (obj_priv->pin_count)
1120 i915_gem_object_flush_cpu_write_domain(obj);
1121
673a394b
EA		 1122 drm_gem_object_unreference(obj);
1123 mutex_unlock(&dev->struct_mutex);
1124 return ret;
1125}
1126
1127/**
1128 * Maps the contents of an object, returning the address it is mapped
1129 * into.
1130 *
1131 * While the mapping holds a reference on the contents of the object, it doesn't
1132 * imply a ref on the object itself.
1133 */
1134int
1135i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1136 struct drm_file *file_priv)
1137{
1138 struct drm_i915_gem_mmap *args = data;
1139 struct drm_gem_object *obj;
1140 loff_t offset;
1141 unsigned long addr;
1142
1143 if (!(dev->driver->driver_features & DRIVER_GEM))
1144 return -ENODEV;
1145
1146 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1147 if (obj == NULL)
1148 return -EBADF;
1149
1150 offset = args->offset;
1151
1152 down_write(&current->mm->mmap_sem);
1153 addr = do_mmap(obj->filp, 0, args->size,
1154 PROT_READ | PROT_WRITE, MAP_SHARED,
1155 args->offset);
1156 up_write(&current->mm->mmap_sem);
1157 mutex_lock(&dev->struct_mutex);
1158 drm_gem_object_unreference(obj);
1159 mutex_unlock(&dev->struct_mutex);
1160 if (IS_ERR((void *)addr))
1161 return addr;
1162
1163 args->addr_ptr = (uint64_t) addr;
1164
1165 return 0;
1166}
1167
de151cf6
JB		 1168/**
1169 * i915_gem_fault - fault a page into the GTT
1170 * vma: VMA in question
1171 * vmf: fault info
1172 *
1173 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1174 * from userspace. The fault handler takes care of binding the object to
1175 * the GTT (if needed), allocating and programming a fence register (again,
1176 * only if needed based on whether the old reg is still valid or the object
1177 * is tiled) and inserting a new PTE into the faulting process.
1178 *
1179 * Note that the faulting process may involve evicting existing objects
1180 * from the GTT and/or fence registers to make room. So performance may
1181 * suffer if the GTT working set is large or there are few fence registers
1182 * left.
1183 */
1184int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1185{
1186 struct drm_gem_object *obj = vma->vm_private_data;
1187 struct drm_device *dev = obj->dev;
1188 struct drm_i915_private *dev_priv = dev->dev_private;
1189 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1190 pgoff_t page_offset;
1191 unsigned long pfn;
1192 int ret = 0;
0f973f27 1193 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1194
1195 /* We don't use vmf->pgoff since that has the fake offset */
1196 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1197 PAGE_SHIFT;
1198
1199 /* Now bind it into the GTT if needed */
1200 mutex_lock(&dev->struct_mutex);
1201 if (!obj_priv->gtt_space) {
e67b8ce1 1202 ret = i915_gem_object_bind_to_gtt(obj, 0);
c715089f
CW		 1203 if (ret)
1204 goto unlock;
07f4f3e8 1205
14b60391 1206 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
07f4f3e8
KH		 1207
1208 ret = i915_gem_object_set_to_gtt_domain(obj, write);
c715089f
CW		 1209 if (ret)
1210 goto unlock;
de151cf6
JB		 1211 }
1212
1213 /* Need a new fence register? */
a09ba7fa 1214 if (obj_priv->tiling_mode != I915_TILING_NONE) {
8c4b8c3f 1215 ret = i915_gem_object_get_fence_reg(obj);
c715089f
CW		 1216 if (ret)
1217 goto unlock;
d9ddcb96 1218 }
de151cf6
JB		 1219
1220 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1221 page_offset;
1222
1223 /* Finally, remap it using the new GTT offset */
1224 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
c715089f 1225unlock:
de151cf6
JB		 1226 mutex_unlock(&dev->struct_mutex);
1227
1228 switch (ret) {
c715089f
CW		 1229 case 0:
1230 case -ERESTARTSYS:
1231 return VM_FAULT_NOPAGE;
de151cf6
JB		 1232 case -ENOMEM:
1233 case -EAGAIN:
1234 return VM_FAULT_OOM;
de151cf6 1235 default:
c715089f 1236 return VM_FAULT_SIGBUS;
de151cf6
JB		 1237 }
1238}
1239
1240/**
1241 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1242 * @obj: obj in question
1243 *
1244 * GEM memory mapping works by handing back to userspace a fake mmap offset
1245 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1246 * up the object based on the offset and sets up the various memory mapping
1247 * structures.
1248 *
1249 * This routine allocates and attaches a fake offset for @obj.
1250 */
1251static int
1252i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1253{
1254 struct drm_device *dev = obj->dev;
1255 struct drm_gem_mm *mm = dev->mm_private;
1256 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1257 struct drm_map_list *list;
f77d390c 1258 struct drm_local_map *map;
de151cf6
JB		 1259 int ret = 0;
1260
1261 /* Set the object up for mmap'ing */
1262 list = &obj->map_list;
9a298b2a 1263 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
de151cf6
JB		 1264 if (!list->map)
1265 return -ENOMEM;
1266
1267 map = list->map;
1268 map->type = _DRM_GEM;
1269 map->size = obj->size;
1270 map->handle = obj;
1271
1272 /* Get a DRM GEM mmap offset allocated... */
1273 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1274 obj->size / PAGE_SIZE, 0, 0);
1275 if (!list->file_offset_node) {
1276 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1277 ret = -ENOMEM;
1278 goto out_free_list;
1279 }
1280
1281 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1282 obj->size / PAGE_SIZE, 0);
1283 if (!list->file_offset_node) {
1284 ret = -ENOMEM;
1285 goto out_free_list;
1286 }
1287
1288 list->hash.key = list->file_offset_node->start;
1289 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1290 DRM_ERROR("failed to add to map hash\n");
5618ca6a 1291 ret = -ENOMEM;
de151cf6
JB		 1292 goto out_free_mm;
1293 }
1294
1295 /* By now we should be all set, any drm_mmap request on the offset
1296 * below will get to our mmap & fault handler */
1297 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1298
1299 return 0;
1300
1301out_free_mm:
1302 drm_mm_put_block(list->file_offset_node);
1303out_free_list:
9a298b2a 1304 kfree(list->map);
de151cf6
JB		 1305
1306 return ret;
1307}
1308
901782b2
CW		 1309/**
1310 * i915_gem_release_mmap - remove physical page mappings
1311 * @obj: obj in question
1312 *
1313 * Preserve the reservation of the mmaping with the DRM core code, but
1314 * relinquish ownership of the pages back to the system.
1315 *
1316 * It is vital that we remove the page mapping if we have mapped a tiled
1317 * object through the GTT and then lose the fence register due to
1318 * resource pressure. Similarly if the object has been moved out of the
1319 * aperture, than pages mapped into userspace must be revoked. Removing the
1320 * mapping will then trigger a page fault on the next user access, allowing
1321 * fixup by i915_gem_fault().
1322 */
d05ca301 1323void
901782b2
CW		 1324i915_gem_release_mmap(struct drm_gem_object *obj)
1325{
1326 struct drm_device *dev = obj->dev;
1327 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1328
1329 if (dev->dev_mapping)
1330 unmap_mapping_range(dev->dev_mapping,
1331 obj_priv->mmap_offset, obj->size, 1);
1332}
1333
ab00b3e5
JB		 1334static void
1335i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1336{
1337 struct drm_device *dev = obj->dev;
1338 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1339 struct drm_gem_mm *mm = dev->mm_private;
1340 struct drm_map_list *list;
1341
1342 list = &obj->map_list;
1343 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1344
1345 if (list->file_offset_node) {
1346 drm_mm_put_block(list->file_offset_node);
1347 list->file_offset_node = NULL;
1348 }
1349
1350 if (list->map) {
9a298b2a 1351 kfree(list->map);
ab00b3e5
JB		 1352 list->map = NULL;
1353 }
1354
1355 obj_priv->mmap_offset = 0;
1356}
1357
de151cf6
JB		 1358/**
1359 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1360 * @obj: object to check
1361 *
1362 * Return the required GTT alignment for an object, taking into account
1363 * potential fence register mapping if needed.
1364 */
1365static uint32_t
1366i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1367{
1368 struct drm_device *dev = obj->dev;
1369 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1370 int start, i;
1371
1372 /*
1373 * Minimum alignment is 4k (GTT page size), but might be greater
1374 * if a fence register is needed for the object.
1375 */
1376 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1377 return 4096;
1378
1379 /*
1380 * Previous chips need to be aligned to the size of the smallest
1381 * fence register that can contain the object.
1382 */
1383 if (IS_I9XX(dev))
1384 start = 1024*1024;
1385 else
1386 start = 512*1024;
1387
1388 for (i = start; i < obj->size; i <<= 1)
1389 ;
1390
1391 return i;
1392}
1393
1394/**
1395 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1396 * @dev: DRM device
1397 * @data: GTT mapping ioctl data
1398 * @file_priv: GEM object info
1399 *
1400 * Simply returns the fake offset to userspace so it can mmap it.
1401 * The mmap call will end up in drm_gem_mmap(), which will set things
1402 * up so we can get faults in the handler above.
1403 *
1404 * The fault handler will take care of binding the object into the GTT
1405 * (since it may have been evicted to make room for something), allocating
1406 * a fence register, and mapping the appropriate aperture address into
1407 * userspace.
1408 */
1409int
1410i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1411 struct drm_file *file_priv)
1412{
1413 struct drm_i915_gem_mmap_gtt *args = data;
1414 struct drm_i915_private *dev_priv = dev->dev_private;
1415 struct drm_gem_object *obj;
1416 struct drm_i915_gem_object *obj_priv;
1417 int ret;
1418
1419 if (!(dev->driver->driver_features & DRIVER_GEM))
1420 return -ENODEV;
1421
1422 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1423 if (obj == NULL)
1424 return -EBADF;
1425
1426 mutex_lock(&dev->struct_mutex);
1427
1428 obj_priv = obj->driver_private;
1429
ab18282d
CW		 1430 if (obj_priv->madv != I915_MADV_WILLNEED) {
1431 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1432 drm_gem_object_unreference(obj);
1433 mutex_unlock(&dev->struct_mutex);
1434 return -EINVAL;
1435 }
1436
1437
de151cf6
JB		 1438 if (!obj_priv->mmap_offset) {
1439 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1440 if (ret) {
1441 drm_gem_object_unreference(obj);
1442 mutex_unlock(&dev->struct_mutex);
de151cf6 1443 return ret;
13af1062 1444 }
de151cf6
JB		 1445 }
1446
1447 args->offset = obj_priv->mmap_offset;
1448
de151cf6
JB		 1449 /*
1450 * Pull it into the GTT so that we have a page list (makes the
1451 * initial fault faster and any subsequent flushing possible).
1452 */
1453 if (!obj_priv->agp_mem) {
e67b8ce1 1454 ret = i915_gem_object_bind_to_gtt(obj, 0);
de151cf6
JB		 1455 if (ret) {
1456 drm_gem_object_unreference(obj);
1457 mutex_unlock(&dev->struct_mutex);
1458 return ret;
1459 }
14b60391 1460 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
de151cf6
JB		 1461 }
1462
1463 drm_gem_object_unreference(obj);
1464 mutex_unlock(&dev->struct_mutex);
1465
1466 return 0;
1467}
1468
6911a9b8 1469void
856fa198 1470i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b
EA		 1471{
1472 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1473 int page_count = obj->size / PAGE_SIZE;
1474 int i;
1475
856fa198 1476 BUG_ON(obj_priv->pages_refcount == 0);
bb6baf76 1477 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
673a394b 1478
856fa198
EA		 1479 if (--obj_priv->pages_refcount != 0)
1480 return;
673a394b 1481
280b713b
EA		 1482 if (obj_priv->tiling_mode != I915_TILING_NONE)
1483 i915_gem_object_save_bit_17_swizzle(obj);
1484
3ef94daa 1485 if (obj_priv->madv == I915_MADV_DONTNEED)
13a05fd9 1486 obj_priv->dirty = 0;
3ef94daa
CW		 1487
1488 for (i = 0; i < page_count; i++) {
1489 if (obj_priv->pages[i] == NULL)
1490 break;
1491
1492 if (obj_priv->dirty)
1493 set_page_dirty(obj_priv->pages[i]);
1494
1495 if (obj_priv->madv == I915_MADV_WILLNEED)
856fa198 1496 mark_page_accessed(obj_priv->pages[i]);
3ef94daa
CW		 1497
1498 page_cache_release(obj_priv->pages[i]);
1499 }
673a394b
EA		 1500 obj_priv->dirty = 0;
1501
8e7d2b2c 1502 drm_free_large(obj_priv->pages);
856fa198 1503 obj_priv->pages = NULL;
673a394b
EA		 1504}
1505
1506static void
ce44b0ea 1507i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
673a394b
EA		 1508{
1509 struct drm_device *dev = obj->dev;
1510 drm_i915_private_t *dev_priv = dev->dev_private;
1511 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1512
1513 /* Add a reference if we're newly entering the active list. */
1514 if (!obj_priv->active) {
1515 drm_gem_object_reference(obj);
1516 obj_priv->active = 1;
1517 }
1518 /* Move from whatever list we were on to the tail of execution. */
5e118f41 1519 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1520 list_move_tail(&obj_priv->list,
1521 &dev_priv->mm.active_list);
5e118f41 1522 spin_unlock(&dev_priv->mm.active_list_lock);
ce44b0ea 1523 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1524}
1525
ce44b0ea
EA		 1526static void
1527i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1528{
1529 struct drm_device *dev = obj->dev;
1530 drm_i915_private_t *dev_priv = dev->dev_private;
1531 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1532
1533 BUG_ON(!obj_priv->active);
1534 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1535 obj_priv->last_rendering_seqno = 0;
1536}
673a394b 1537
963b4836
CW		 1538/* Immediately discard the backing storage */
1539static void
1540i915_gem_object_truncate(struct drm_gem_object *obj)
1541{
bb6baf76
CW		 1542 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1543 struct inode *inode;
963b4836 1544
bb6baf76
CW		 1545 inode = obj->filp->f_path.dentry->d_inode;
1546 if (inode->i_op->truncate)
1547 inode->i_op->truncate (inode);
1548
1549 obj_priv->madv = __I915_MADV_PURGED;
963b4836
CW		 1550}
1551
1552static inline int
1553i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1554{
1555 return obj_priv->madv == I915_MADV_DONTNEED;
1556}
1557
673a394b
EA		 1558static void
1559i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1560{
1561 struct drm_device *dev = obj->dev;
1562 drm_i915_private_t *dev_priv = dev->dev_private;
1563 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1564
1565 i915_verify_inactive(dev, __FILE__, __LINE__);
1566 if (obj_priv->pin_count != 0)
1567 list_del_init(&obj_priv->list);
1568 else
1569 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1570
ce44b0ea 1571 obj_priv->last_rendering_seqno = 0;
673a394b
EA		 1572 if (obj_priv->active) {
1573 obj_priv->active = 0;
1574 drm_gem_object_unreference(obj);
1575 }
1576 i915_verify_inactive(dev, __FILE__, __LINE__);
1577}
1578
1579/**
1580 * Creates a new sequence number, emitting a write of it to the status page
1581 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1582 *
1583 * Must be called with struct_lock held.
1584 *
1585 * Returned sequence numbers are nonzero on success.
1586 */
5a5a0c64 1587uint32_t
b962442e
EA		 1588i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1589 uint32_t flush_domains)
673a394b
EA		 1590{
1591 drm_i915_private_t *dev_priv = dev->dev_private;
b962442e 1592 struct drm_i915_file_private *i915_file_priv = NULL;
673a394b
EA		 1593 struct drm_i915_gem_request *request;
1594 uint32_t seqno;
1595 int was_empty;
1596 RING_LOCALS;
1597
b962442e
EA		 1598 if (file_priv != NULL)
1599 i915_file_priv = file_priv->driver_priv;
1600
9a298b2a 1601 request = kzalloc(sizeof(*request), GFP_KERNEL);
673a394b
EA		 1602 if (request == NULL)
1603 return 0;
1604
1605 /* Grab the seqno we're going to make this request be, and bump the
1606 * next (skipping 0 so it can be the reserved no-seqno value).
1607 */
1608 seqno = dev_priv->mm.next_gem_seqno;
1609 dev_priv->mm.next_gem_seqno++;
1610 if (dev_priv->mm.next_gem_seqno == 0)
1611 dev_priv->mm.next_gem_seqno++;
1612
1613 BEGIN_LP_RING(4);
1614 OUT_RING(MI_STORE_DWORD_INDEX);
1615 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1616 OUT_RING(seqno);
1617
1618 OUT_RING(MI_USER_INTERRUPT);
1619 ADVANCE_LP_RING();
1620
44d98a61 1621 DRM_DEBUG_DRIVER("%d\n", seqno);
673a394b
EA		 1622
1623 request->seqno = seqno;
1624 request->emitted_jiffies = jiffies;
673a394b
EA		 1625 was_empty = list_empty(&dev_priv->mm.request_list);
1626 list_add_tail(&request->list, &dev_priv->mm.request_list);
b962442e
EA		 1627 if (i915_file_priv) {
1628 list_add_tail(&request->client_list,
1629 &i915_file_priv->mm.request_list);
1630 } else {
1631 INIT_LIST_HEAD(&request->client_list);
1632 }
673a394b 1633
ce44b0ea
EA		 1634 /* Associate any objects on the flushing list matching the write
1635 * domain we're flushing with our flush.
1636 */
1637 if (flush_domains != 0) {
1638 struct drm_i915_gem_object *obj_priv, *next;
1639
1640 list_for_each_entry_safe(obj_priv, next,
1641 &dev_priv->mm.flushing_list, list) {
1642 struct drm_gem_object *obj = obj_priv->obj;
1643
1644 if ((obj->write_domain & flush_domains) ==
1645 obj->write_domain) {
1c5d22f7
CW		 1646 uint32_t old_write_domain = obj->write_domain;
1647
ce44b0ea
EA		 1648 obj->write_domain = 0;
1649 i915_gem_object_move_to_active(obj, seqno);
1c5d22f7
CW		 1650
1651 trace_i915_gem_object_change_domain(obj,
1652 obj->read_domains,
1653 old_write_domain);
ce44b0ea
EA		 1654 }
1655 }
1656
1657 }
1658
f65d9421
BG		 1659 if (!dev_priv->mm.suspended) {
1660 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1661 if (was_empty)
1662 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1663 }
673a394b
EA		 1664 return seqno;
1665}
1666
1667/**
1668 * Command execution barrier
1669 *
1670 * Ensures that all commands in the ring are finished
1671 * before signalling the CPU
1672 */
3043c60c 1673static uint32_t
673a394b
EA		 1674i915_retire_commands(struct drm_device *dev)
1675{
1676 drm_i915_private_t *dev_priv = dev->dev_private;
1677 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1678 uint32_t flush_domains = 0;
1679 RING_LOCALS;
1680
1681 /* The sampler always gets flushed on i965 (sigh) */
1682 if (IS_I965G(dev))
1683 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1684 BEGIN_LP_RING(2);
1685 OUT_RING(cmd);
1686 OUT_RING(0); /* noop */
1687 ADVANCE_LP_RING();
1688 return flush_domains;
1689}
1690
1691/**
1692 * Moves buffers associated only with the given active seqno from the active
1693 * to inactive list, potentially freeing them.
1694 */
1695static void
1696i915_gem_retire_request(struct drm_device *dev,
1697 struct drm_i915_gem_request *request)
1698{
1699 drm_i915_private_t *dev_priv = dev->dev_private;
1700
1c5d22f7
CW		 1701 trace_i915_gem_request_retire(dev, request->seqno);
1702
673a394b
EA		 1703 /* Move any buffers on the active list that are no longer referenced
1704 * by the ringbuffer to the flushing/inactive lists as appropriate.
1705 */
5e118f41 1706 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1707 while (!list_empty(&dev_priv->mm.active_list)) {
1708 struct drm_gem_object *obj;
1709 struct drm_i915_gem_object *obj_priv;
1710
1711 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1712 struct drm_i915_gem_object,
1713 list);
1714 obj = obj_priv->obj;
1715
1716 /* If the seqno being retired doesn't match the oldest in the
1717 * list, then the oldest in the list must still be newer than
1718 * this seqno.
1719 */
1720 if (obj_priv->last_rendering_seqno != request->seqno)
5e118f41 1721 goto out;
de151cf6 1722
673a394b
EA		 1723#if WATCH_LRU
1724 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1725 __func__, request->seqno, obj);
1726#endif
1727
ce44b0ea
EA		 1728 if (obj->write_domain != 0)
1729 i915_gem_object_move_to_flushing(obj);
68c84342
SL		 1730 else {
1731 /* Take a reference on the object so it won't be
1732 * freed while the spinlock is held. The list
1733 * protection for this spinlock is safe when breaking
1734 * the lock like this since the next thing we do
1735 * is just get the head of the list again.
1736 */
1737 drm_gem_object_reference(obj);
673a394b 1738 i915_gem_object_move_to_inactive(obj);
68c84342
SL		 1739 spin_unlock(&dev_priv->mm.active_list_lock);
1740 drm_gem_object_unreference(obj);
1741 spin_lock(&dev_priv->mm.active_list_lock);
1742 }
673a394b 1743 }
5e118f41
CW		 1744out:
1745 spin_unlock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1746}
1747
1748/**
1749 * Returns true if seq1 is later than seq2.
1750 */
22be1724 1751bool
673a394b
EA		 1752i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1753{
1754 return (int32_t)(seq1 - seq2) >= 0;
1755}
1756
1757uint32_t
1758i915_get_gem_seqno(struct drm_device *dev)
1759{
1760 drm_i915_private_t *dev_priv = dev->dev_private;
1761
1762 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1763}
1764
1765/**
1766 * This function clears the request list as sequence numbers are passed.
1767 */
1768void
1769i915_gem_retire_requests(struct drm_device *dev)
1770{
1771 drm_i915_private_t *dev_priv = dev->dev_private;
1772 uint32_t seqno;
1773
9d34e5db 1774 if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
6c0594a3
KW		 1775 return;
1776
673a394b
EA		 1777 seqno = i915_get_gem_seqno(dev);
1778
1779 while (!list_empty(&dev_priv->mm.request_list)) {
1780 struct drm_i915_gem_request *request;
1781 uint32_t retiring_seqno;
1782
1783 request = list_first_entry(&dev_priv->mm.request_list,
1784 struct drm_i915_gem_request,
1785 list);
1786 retiring_seqno = request->seqno;
1787
1788 if (i915_seqno_passed(seqno, retiring_seqno) ||
ba1234d1 1789 atomic_read(&dev_priv->mm.wedged)) {
673a394b
EA		 1790 i915_gem_retire_request(dev, request);
1791
1792 list_del(&request->list);
b962442e 1793 list_del(&request->client_list);
9a298b2a 1794 kfree(request);
673a394b
EA		 1795 } else
1796 break;
1797 }
9d34e5db
CW		 1798
1799 if (unlikely (dev_priv->trace_irq_seqno &&
1800 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1801 i915_user_irq_put(dev);
1802 dev_priv->trace_irq_seqno = 0;
1803 }
673a394b
EA		 1804}
1805
1806void
1807i915_gem_retire_work_handler(struct work_struct *work)
1808{
1809 drm_i915_private_t *dev_priv;
1810 struct drm_device *dev;
1811
1812 dev_priv = container_of(work, drm_i915_private_t,
1813 mm.retire_work.work);
1814 dev = dev_priv->dev;
1815
1816 mutex_lock(&dev->struct_mutex);
1817 i915_gem_retire_requests(dev);
6dbe2772
KP		 1818 if (!dev_priv->mm.suspended &&
1819 !list_empty(&dev_priv->mm.request_list))
9c9fe1f8 1820 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
673a394b
EA		 1821 mutex_unlock(&dev->struct_mutex);
1822}
1823
5a5a0c64 1824int
48764bf4 1825i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
673a394b
EA		 1826{
1827 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1828 u32 ier;
673a394b
EA		 1829 int ret = 0;
1830
1831 BUG_ON(seqno == 0);
1832
ba1234d1 1833 if (atomic_read(&dev_priv->mm.wedged))
ffed1d09
BG		 1834 return -EIO;
1835
673a394b 1836 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
f2b115e6 1837 if (IS_IRONLAKE(dev))
036a4a7d
ZW		 1838 ier = I915_READ(DEIER) | I915_READ(GTIER);
1839 else
1840 ier = I915_READ(IER);
802c7eb6
JB		 1841 if (!ier) {
1842 DRM_ERROR("something (likely vbetool) disabled "
1843 "interrupts, re-enabling\n");
1844 i915_driver_irq_preinstall(dev);
1845 i915_driver_irq_postinstall(dev);
1846 }
1847
1c5d22f7
CW		 1848 trace_i915_gem_request_wait_begin(dev, seqno);
1849
673a394b
EA		 1850 dev_priv->mm.waiting_gem_seqno = seqno;
1851 i915_user_irq_get(dev);
48764bf4
DV		 1852 if (interruptible)
1853 ret = wait_event_interruptible(dev_priv->irq_queue,
1854 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1855 atomic_read(&dev_priv->mm.wedged));
1856 else
1857 wait_event(dev_priv->irq_queue,
1858 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1859 atomic_read(&dev_priv->mm.wedged));
1860
673a394b
EA		 1861 i915_user_irq_put(dev);
1862 dev_priv->mm.waiting_gem_seqno = 0;
1c5d22f7
CW		 1863
1864 trace_i915_gem_request_wait_end(dev, seqno);
673a394b 1865 }
ba1234d1 1866 if (atomic_read(&dev_priv->mm.wedged))
673a394b
EA		 1867 ret = -EIO;
1868
1869 if (ret && ret != -ERESTARTSYS)
1870 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1871 __func__, ret, seqno, i915_get_gem_seqno(dev));
1872
1873 /* Directly dispatch request retiring. While we have the work queue
1874 * to handle this, the waiter on a request often wants an associated
1875 * buffer to have made it to the inactive list, and we would need
1876 * a separate wait queue to handle that.
1877 */
1878 if (ret == 0)
1879 i915_gem_retire_requests(dev);
1880
1881 return ret;
1882}
1883
48764bf4
DV		 1884/**
1885 * Waits for a sequence number to be signaled, and cleans up the
1886 * request and object lists appropriately for that event.
1887 */
1888static int
1889i915_wait_request(struct drm_device *dev, uint32_t seqno)
1890{
1891 return i915_do_wait_request(dev, seqno, 1);
1892}
1893
673a394b
EA		 1894static void
1895i915_gem_flush(struct drm_device *dev,
1896 uint32_t invalidate_domains,
1897 uint32_t flush_domains)
1898{
1899 drm_i915_private_t *dev_priv = dev->dev_private;
1900 uint32_t cmd;
1901 RING_LOCALS;
1902
1903#if WATCH_EXEC
1904 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1905 invalidate_domains, flush_domains);
1906#endif
1c5d22f7
CW		 1907 trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1908 invalidate_domains, flush_domains);
673a394b
EA		 1909
1910 if (flush_domains & I915_GEM_DOMAIN_CPU)
1911 drm_agp_chipset_flush(dev);
1912
21d509e3 1913 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
673a394b
EA		 1914 /*
1915 * read/write caches:
1916 *
1917 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1918 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1919 * also flushed at 2d versus 3d pipeline switches.
1920 *
1921 * read-only caches:
1922 *
1923 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1924 * MI_READ_FLUSH is set, and is always flushed on 965.
1925 *
1926 * I915_GEM_DOMAIN_COMMAND may not exist?
1927 *
1928 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1929 * invalidated when MI_EXE_FLUSH is set.
1930 *
1931 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1932 * invalidated with every MI_FLUSH.
1933 *
1934 * TLBs:
1935 *
1936 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1937 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1938 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1939 * are flushed at any MI_FLUSH.
1940 */
1941
1942 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1943 if ((invalidate_domains|flush_domains) &
1944 I915_GEM_DOMAIN_RENDER)
1945 cmd &= ~MI_NO_WRITE_FLUSH;
1946 if (!IS_I965G(dev)) {
1947 /*
1948 * On the 965, the sampler cache always gets flushed
1949 * and this bit is reserved.
1950 */
1951 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1952 cmd |= MI_READ_FLUSH;
1953 }
1954 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1955 cmd |= MI_EXE_FLUSH;
1956
1957#if WATCH_EXEC
1958 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1959#endif
1960 BEGIN_LP_RING(2);
1961 OUT_RING(cmd);
48764bf4 1962 OUT_RING(MI_NOOP);
673a394b
EA		 1963 ADVANCE_LP_RING();
1964 }
1965}
1966
1967/**
1968 * Ensures that all rendering to the object has completed and the object is
1969 * safe to unbind from the GTT or access from the CPU.
1970 */
1971static int
1972i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1973{
1974 struct drm_device *dev = obj->dev;
1975 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1976 int ret;
1977
e47c68e9
EA		 1978 /* This function only exists to support waiting for existing rendering,
1979 * not for emitting required flushes.
673a394b 1980 */
e47c68e9 1981 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 1982
1983 /* If there is rendering queued on the buffer being evicted, wait for
1984 * it.
1985 */
1986 if (obj_priv->active) {
1987#if WATCH_BUF
1988 DRM_INFO("%s: object %p wait for seqno %08x\n",
1989 __func__, obj, obj_priv->last_rendering_seqno);
1990#endif
1991 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1992 if (ret != 0)
1993 return ret;
1994 }
1995
1996 return 0;
1997}
1998
1999/**
2000 * Unbinds an object from the GTT aperture.
2001 */
0f973f27 2002int
673a394b
EA		 2003i915_gem_object_unbind(struct drm_gem_object *obj)
2004{
2005 struct drm_device *dev = obj->dev;
2006 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2007 int ret = 0;
2008
2009#if WATCH_BUF
2010 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2011 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2012#endif
2013 if (obj_priv->gtt_space == NULL)
2014 return 0;
2015
2016 if (obj_priv->pin_count != 0) {
2017 DRM_ERROR("Attempting to unbind pinned buffer\n");
2018 return -EINVAL;
2019 }
2020
5323fd04
EA		 2021 /* blow away mappings if mapped through GTT */
2022 i915_gem_release_mmap(obj);
2023
2024 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2025 i915_gem_clear_fence_reg(obj);
2026
673a394b
EA		 2027 /* Move the object to the CPU domain to ensure that
2028 * any possible CPU writes while it's not in the GTT
2029 * are flushed when we go to remap it. This will
2030 * also ensure that all pending GPU writes are finished
2031 * before we unbind.
2032 */
e47c68e9 2033 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
673a394b 2034 if (ret) {
e47c68e9
EA		 2035 if (ret != -ERESTARTSYS)
2036 DRM_ERROR("set_domain failed: %d\n", ret);
673a394b
EA		 2037 return ret;
2038 }
2039
5323fd04
EA		 2040 BUG_ON(obj_priv->active);
2041
673a394b
EA		 2042 if (obj_priv->agp_mem != NULL) {
2043 drm_unbind_agp(obj_priv->agp_mem);
2044 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2045 obj_priv->agp_mem = NULL;
2046 }
2047
856fa198 2048 i915_gem_object_put_pages(obj);
a32808c0 2049 BUG_ON(obj_priv->pages_refcount);
673a394b
EA		 2050
2051 if (obj_priv->gtt_space) {
2052 atomic_dec(&dev->gtt_count);
2053 atomic_sub(obj->size, &dev->gtt_memory);
2054
2055 drm_mm_put_block(obj_priv->gtt_space);
2056 obj_priv->gtt_space = NULL;
2057 }
2058
2059 /* Remove ourselves from the LRU list if present. */
2060 if (!list_empty(&obj_priv->list))
2061 list_del_init(&obj_priv->list);
2062
963b4836
CW		 2063 if (i915_gem_object_is_purgeable(obj_priv))
2064 i915_gem_object_truncate(obj);
2065
1c5d22f7
CW		 2066 trace_i915_gem_object_unbind(obj);
2067
673a394b
EA		 2068 return 0;
2069}
2070
07f73f69
CW		 2071static struct drm_gem_object *
2072i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2073{
2074 drm_i915_private_t *dev_priv = dev->dev_private;
2075 struct drm_i915_gem_object *obj_priv;
2076 struct drm_gem_object *best = NULL;
2077 struct drm_gem_object *first = NULL;
2078
2079 /* Try to find the smallest clean object */
2080 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2081 struct drm_gem_object *obj = obj_priv->obj;
2082 if (obj->size >= min_size) {
963b4836
CW		 2083 if ((!obj_priv->dirty ||
2084 i915_gem_object_is_purgeable(obj_priv)) &&
07f73f69
CW		 2085 (!best || obj->size < best->size)) {
2086 best = obj;
2087 if (best->size == min_size)
2088 return best;
2089 }
2090 if (!first)
2091 first = obj;
2092 }
2093 }
2094
2095 return best ? best : first;
2096}
2097
673a394b 2098static int
07f73f69
CW		 2099i915_gem_evict_everything(struct drm_device *dev)
2100{
2101 drm_i915_private_t *dev_priv = dev->dev_private;
2102 uint32_t seqno;
2103 int ret;
2104 bool lists_empty;
2105
07f73f69
CW		 2106 spin_lock(&dev_priv->mm.active_list_lock);
2107 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2108 list_empty(&dev_priv->mm.flushing_list) &&
2109 list_empty(&dev_priv->mm.active_list));
2110 spin_unlock(&dev_priv->mm.active_list_lock);
2111
9731129c 2112 if (lists_empty)
07f73f69 2113 return -ENOSPC;
07f73f69
CW		 2114
2115 /* Flush everything (on to the inactive lists) and evict */
2116 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2117 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2118 if (seqno == 0)
2119 return -ENOMEM;
2120
2121 ret = i915_wait_request(dev, seqno);
2122 if (ret)
2123 return ret;
2124
ab5ee576 2125 ret = i915_gem_evict_from_inactive_list(dev);
07f73f69
CW		 2126 if (ret)
2127 return ret;
2128
2129 spin_lock(&dev_priv->mm.active_list_lock);
2130 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2131 list_empty(&dev_priv->mm.flushing_list) &&
2132 list_empty(&dev_priv->mm.active_list));
2133 spin_unlock(&dev_priv->mm.active_list_lock);
2134 BUG_ON(!lists_empty);
2135
2136 return 0;
2137}
2138
673a394b 2139static int
07f73f69 2140i915_gem_evict_something(struct drm_device *dev, int min_size)
673a394b
EA		 2141{
2142 drm_i915_private_t *dev_priv = dev->dev_private;
2143 struct drm_gem_object *obj;
07f73f69 2144 int ret;
673a394b
EA		 2145
2146 for (;;) {
07f73f69
CW		 2147 i915_gem_retire_requests(dev);
2148
673a394b
EA		 2149 /* If there's an inactive buffer available now, grab it
2150 * and be done.
2151 */
07f73f69
CW		 2152 obj = i915_gem_find_inactive_object(dev, min_size);
2153 if (obj) {
2154 struct drm_i915_gem_object *obj_priv;
2155
673a394b
EA		 2156#if WATCH_LRU
2157 DRM_INFO("%s: evicting %p\n", __func__, obj);
2158#endif
07f73f69
CW		 2159 obj_priv = obj->driver_private;
2160 BUG_ON(obj_priv->pin_count != 0);
673a394b
EA		 2161 BUG_ON(obj_priv->active);
2162
2163 /* Wait on the rendering and unbind the buffer. */
07f73f69 2164 return i915_gem_object_unbind(obj);
673a394b
EA		 2165 }
2166
2167 /* If we didn't get anything, but the ring is still processing
07f73f69
CW		 2168 * things, wait for the next to finish and hopefully leave us
2169 * a buffer to evict.
673a394b
EA		 2170 */
2171 if (!list_empty(&dev_priv->mm.request_list)) {
2172 struct drm_i915_gem_request *request;
2173
2174 request = list_first_entry(&dev_priv->mm.request_list,
2175 struct drm_i915_gem_request,
2176 list);
2177
2178 ret = i915_wait_request(dev, request->seqno);
2179 if (ret)
07f73f69 2180 return ret;
673a394b 2181
07f73f69 2182 continue;
673a394b
EA		 2183 }
2184
2185 /* If we didn't have anything on the request list but there
2186 * are buffers awaiting a flush, emit one and try again.
2187 * When we wait on it, those buffers waiting for that flush
2188 * will get moved to inactive.
2189 */
2190 if (!list_empty(&dev_priv->mm.flushing_list)) {
07f73f69 2191 struct drm_i915_gem_object *obj_priv;
673a394b 2192
9a1e2582
CW		 2193 /* Find an object that we can immediately reuse */
2194 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2195 obj = obj_priv->obj;
2196 if (obj->size >= min_size)
2197 break;
673a394b 2198
9a1e2582
CW		 2199 obj = NULL;
2200 }
673a394b 2201
9a1e2582
CW		 2202 if (obj != NULL) {
2203 uint32_t seqno;
673a394b 2204
9a1e2582
CW		 2205 i915_gem_flush(dev,
2206 obj->write_domain,
2207 obj->write_domain);
2208 seqno = i915_add_request(dev, NULL, obj->write_domain);
2209 if (seqno == 0)
2210 return -ENOMEM;
ac94a962 2211
9a1e2582
CW		 2212 ret = i915_wait_request(dev, seqno);
2213 if (ret)
2214 return ret;
2215
2216 continue;
2217 }
673a394b
EA		 2218 }
2219
07f73f69
CW		 2220 /* If we didn't do any of the above, there's no single buffer
2221 * large enough to swap out for the new one, so just evict
2222 * everything and start again. (This should be rare.)
673a394b 2223 */
9731129c 2224 if (!list_empty (&dev_priv->mm.inactive_list))
ab5ee576 2225 return i915_gem_evict_from_inactive_list(dev);
9731129c 2226 else
07f73f69 2227 return i915_gem_evict_everything(dev);
ac94a962 2228 }
ac94a962
KP		 2229}
2230
6911a9b8 2231int
856fa198 2232i915_gem_object_get_pages(struct drm_gem_object *obj)
673a394b
EA		 2233{
2234 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2235 int page_count, i;
2236 struct address_space *mapping;
2237 struct inode *inode;
2238 struct page *page;
2239 int ret;
2240
856fa198 2241 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2242 return 0;
2243
2244 /* Get the list of pages out of our struct file. They'll be pinned
2245 * at this point until we release them.
2246 */
2247 page_count = obj->size / PAGE_SIZE;
856fa198 2248 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2249 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2250 if (obj_priv->pages == NULL) {
856fa198 2251 obj_priv->pages_refcount--;
673a394b
EA		 2252 return -ENOMEM;
2253 }
2254
2255 inode = obj->filp->f_path.dentry->d_inode;
2256 mapping = inode->i_mapping;
2257 for (i = 0; i < page_count; i++) {
2258 page = read_mapping_page(mapping, i, NULL);
2259 if (IS_ERR(page)) {
2260 ret = PTR_ERR(page);
856fa198 2261 i915_gem_object_put_pages(obj);
673a394b
EA		 2262 return ret;
2263 }
856fa198 2264 obj_priv->pages[i] = page;
673a394b 2265 }
280b713b
EA		 2266
2267 if (obj_priv->tiling_mode != I915_TILING_NONE)
2268 i915_gem_object_do_bit_17_swizzle(obj);
2269
673a394b
EA		 2270 return 0;
2271}
2272
de151cf6
JB		 2273static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2274{
2275 struct drm_gem_object *obj = reg->obj;
2276 struct drm_device *dev = obj->dev;
2277 drm_i915_private_t *dev_priv = dev->dev_private;
2278 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2279 int regnum = obj_priv->fence_reg;
2280 uint64_t val;
2281
2282 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2283 0xfffff000) << 32;
2284 val |= obj_priv->gtt_offset & 0xfffff000;
2285 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2286 if (obj_priv->tiling_mode == I915_TILING_Y)
2287 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2288 val |= I965_FENCE_REG_VALID;
2289
2290 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2291}
2292
2293static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2294{
2295 struct drm_gem_object *obj = reg->obj;
2296 struct drm_device *dev = obj->dev;
2297 drm_i915_private_t *dev_priv = dev->dev_private;
2298 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2299 int regnum = obj_priv->fence_reg;
0f973f27 2300 int tile_width;
dc529a4f 2301 uint32_t fence_reg, val;
de151cf6
JB		 2302 uint32_t pitch_val;
2303
2304 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2305 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2306 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2307 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2308 return;
2309 }
2310
0f973f27
JB		 2311 if (obj_priv->tiling_mode == I915_TILING_Y &&
2312 HAS_128_BYTE_Y_TILING(dev))
2313 tile_width = 128;
de151cf6 2314 else
0f973f27
JB		 2315 tile_width = 512;
2316
2317 /* Note: pitch better be a power of two tile widths */
2318 pitch_val = obj_priv->stride / tile_width;
2319 pitch_val = ffs(pitch_val) - 1;
de151cf6
JB		 2320
2321 val = obj_priv->gtt_offset;
2322 if (obj_priv->tiling_mode == I915_TILING_Y)
2323 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2324 val |= I915_FENCE_SIZE_BITS(obj->size);
2325 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2326 val |= I830_FENCE_REG_VALID;
2327
dc529a4f
EA		 2328 if (regnum < 8)
2329 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2330 else
2331 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2332 I915_WRITE(fence_reg, val);
de151cf6
JB		 2333}
2334
2335static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2336{
2337 struct drm_gem_object *obj = reg->obj;
2338 struct drm_device *dev = obj->dev;
2339 drm_i915_private_t *dev_priv = dev->dev_private;
2340 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2341 int regnum = obj_priv->fence_reg;
2342 uint32_t val;
2343 uint32_t pitch_val;
8d7773a3 2344 uint32_t fence_size_bits;
de151cf6 2345
8d7773a3 2346 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2347 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2348 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2349 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2350 return;
2351 }
2352
e76a16de
EA		 2353 pitch_val = obj_priv->stride / 128;
2354 pitch_val = ffs(pitch_val) - 1;
2355 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2356
de151cf6
JB		 2357 val = obj_priv->gtt_offset;
2358 if (obj_priv->tiling_mode == I915_TILING_Y)
2359 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2360 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2361 WARN_ON(fence_size_bits & ~0x00000f00);
2362 val |= fence_size_bits;
de151cf6
JB		 2363 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2364 val |= I830_FENCE_REG_VALID;
2365
2366 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
de151cf6
JB		 2367}
2368
2369/**
2370 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2371 * @obj: object to map through a fence reg
2372 *
2373 * When mapping objects through the GTT, userspace wants to be able to write
2374 * to them without having to worry about swizzling if the object is tiled.
2375 *
2376 * This function walks the fence regs looking for a free one for @obj,
2377 * stealing one if it can't find any.
2378 *
2379 * It then sets up the reg based on the object's properties: address, pitch
2380 * and tiling format.
2381 */
8c4b8c3f
CW		 2382int
2383i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
de151cf6
JB		 2384{
2385 struct drm_device *dev = obj->dev;
79e53945 2386 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6
JB		 2387 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2388 struct drm_i915_fence_reg *reg = NULL;
fc7170ba
CW		 2389 struct drm_i915_gem_object *old_obj_priv = NULL;
2390 int i, ret, avail;
de151cf6 2391
a09ba7fa
EA		 2392 /* Just update our place in the LRU if our fence is getting used. */
2393 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2394 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2395 return 0;
2396 }
2397
de151cf6
JB		 2398 switch (obj_priv->tiling_mode) {
2399 case I915_TILING_NONE:
2400 WARN(1, "allocating a fence for non-tiled object?\n");
2401 break;
2402 case I915_TILING_X:
0f973f27
JB		 2403 if (!obj_priv->stride)
2404 return -EINVAL;
2405 WARN((obj_priv->stride & (512 - 1)),
2406 "object 0x%08x is X tiled but has non-512B pitch\n",
2407 obj_priv->gtt_offset);
de151cf6
JB		 2408 break;
2409 case I915_TILING_Y:
0f973f27
JB		 2410 if (!obj_priv->stride)
2411 return -EINVAL;
2412 WARN((obj_priv->stride & (128 - 1)),
2413 "object 0x%08x is Y tiled but has non-128B pitch\n",
2414 obj_priv->gtt_offset);
de151cf6
JB		 2415 break;
2416 }
2417
2418 /* First try to find a free reg */
fc7170ba 2419 avail = 0;
de151cf6
JB		 2420 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2421 reg = &dev_priv->fence_regs[i];
2422 if (!reg->obj)
2423 break;
fc7170ba
CW		 2424
2425 old_obj_priv = reg->obj->driver_private;
2426 if (!old_obj_priv->pin_count)
2427 avail++;
de151cf6
JB		 2428 }
2429
2430 /* None available, try to steal one or wait for a user to finish */
2431 if (i == dev_priv->num_fence_regs) {
a09ba7fa 2432 struct drm_gem_object *old_obj = NULL;
de151cf6 2433
fc7170ba 2434 if (avail == 0)
2939e1f5 2435 return -ENOSPC;
fc7170ba 2436
a09ba7fa
EA		 2437 list_for_each_entry(old_obj_priv, &dev_priv->mm.fence_list,
2438 fence_list) {
2439 old_obj = old_obj_priv->obj;
d7619c4b 2440
d7619c4b
CW		 2441 if (old_obj_priv->pin_count)
2442 continue;
2443
a09ba7fa
EA		 2444 /* Take a reference, as otherwise the wait_rendering
2445 * below may cause the object to get freed out from
2446 * under us.
2447 */
2448 drm_gem_object_reference(old_obj);
2449
d7619c4b
CW		 2450 /* i915 uses fences for GPU access to tiled buffers */
2451 if (IS_I965G(dev) || !old_obj_priv->active)
de151cf6 2452 break;
d7619c4b 2453
a09ba7fa
EA		 2454 /* This brings the object to the head of the LRU if it
2455 * had been written to. The only way this should
2456 * result in us waiting longer than the expected
2457 * optimal amount of time is if there was a
2458 * fence-using buffer later that was read-only.
2459 */
2460 i915_gem_object_flush_gpu_write_domain(old_obj);
2461 ret = i915_gem_object_wait_rendering(old_obj);
58c2fb64
CW		 2462 if (ret != 0) {
2463 drm_gem_object_unreference(old_obj);
d7619c4b 2464 return ret;
58c2fb64
CW		 2465 }
2466
a09ba7fa 2467 break;
de151cf6
JB		 2468 }
2469
2470 /*
2471 * Zap this virtual mapping so we can set up a fence again
2472 * for this object next time we need it.
2473 */
58c2fb64
CW		 2474 i915_gem_release_mmap(old_obj);
2475
a09ba7fa 2476 i = old_obj_priv->fence_reg;
58c2fb64
CW		 2477 reg = &dev_priv->fence_regs[i];
2478
de151cf6 2479 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
a09ba7fa 2480 list_del_init(&old_obj_priv->fence_list);
58c2fb64 2481
a09ba7fa 2482 drm_gem_object_unreference(old_obj);
de151cf6
JB		 2483 }
2484
2485 obj_priv->fence_reg = i;
a09ba7fa
EA		 2486 list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2487
de151cf6
JB		 2488 reg->obj = obj;
2489
2490 if (IS_I965G(dev))
2491 i965_write_fence_reg(reg);
2492 else if (IS_I9XX(dev))
2493 i915_write_fence_reg(reg);
2494 else
2495 i830_write_fence_reg(reg);
d9ddcb96 2496
1c5d22f7
CW		 2497 trace_i915_gem_object_get_fence(obj, i, obj_priv->tiling_mode);
2498
d9ddcb96 2499 return 0;
de151cf6
JB		 2500}
2501
2502/**
2503 * i915_gem_clear_fence_reg - clear out fence register info
2504 * @obj: object to clear
2505 *
2506 * Zeroes out the fence register itself and clears out the associated
2507 * data structures in dev_priv and obj_priv.
2508 */
2509static void
2510i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2511{
2512 struct drm_device *dev = obj->dev;
79e53945 2513 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 2514 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2515
2516 if (IS_I965G(dev))
2517 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
dc529a4f
EA		 2518 else {
2519 uint32_t fence_reg;
2520
2521 if (obj_priv->fence_reg < 8)
2522 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2523 else
2524 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2525 8) * 4;
2526
2527 I915_WRITE(fence_reg, 0);
2528 }
de151cf6
JB		 2529
2530 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2531 obj_priv->fence_reg = I915_FENCE_REG_NONE;
a09ba7fa 2532 list_del_init(&obj_priv->fence_list);
de151cf6
JB		 2533}
2534
52dc7d32
CW		 2535/**
2536 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2537 * to the buffer to finish, and then resets the fence register.
2538 * @obj: tiled object holding a fence register.
2539 *
2540 * Zeroes out the fence register itself and clears out the associated
2541 * data structures in dev_priv and obj_priv.
2542 */
2543int
2544i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2545{
2546 struct drm_device *dev = obj->dev;
2547 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2548
2549 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2550 return 0;
2551
2552 /* On the i915, GPU access to tiled buffers is via a fence,
2553 * therefore we must wait for any outstanding access to complete
2554 * before clearing the fence.
2555 */
2556 if (!IS_I965G(dev)) {
2557 int ret;
2558
2559 i915_gem_object_flush_gpu_write_domain(obj);
2560 i915_gem_object_flush_gtt_write_domain(obj);
2561 ret = i915_gem_object_wait_rendering(obj);
2562 if (ret != 0)
2563 return ret;
2564 }
2565
2566 i915_gem_clear_fence_reg (obj);
2567
2568 return 0;
2569}
2570
673a394b
EA		 2571/**
2572 * Finds free space in the GTT aperture and binds the object there.
2573 */
2574static int
2575i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2576{
2577 struct drm_device *dev = obj->dev;
2578 drm_i915_private_t *dev_priv = dev->dev_private;
2579 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2580 struct drm_mm_node *free_space;
07f73f69
CW		 2581 bool retry_alloc = false;
2582 int ret;
673a394b 2583
9bb2d6f9
EA		 2584 if (dev_priv->mm.suspended)
2585 return -EBUSY;
3ef94daa 2586
bb6baf76 2587 if (obj_priv->madv != I915_MADV_WILLNEED) {
3ef94daa
CW		 2588 DRM_ERROR("Attempting to bind a purgeable object\n");
2589 return -EINVAL;
2590 }
2591
673a394b 2592 if (alignment == 0)
0f973f27 2593 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2594 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2595 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2596 return -EINVAL;
2597 }
2598
2599 search_free:
2600 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2601 obj->size, alignment, 0);
2602 if (free_space != NULL) {
2603 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2604 alignment);
2605 if (obj_priv->gtt_space != NULL) {
2606 obj_priv->gtt_space->private = obj;
2607 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2608 }
2609 }
2610 if (obj_priv->gtt_space == NULL) {
2611 /* If the gtt is empty and we're still having trouble
2612 * fitting our object in, we're out of memory.
2613 */
2614#if WATCH_LRU
2615 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2616#endif
07f73f69 2617 ret = i915_gem_evict_something(dev, obj->size);
9731129c 2618 if (ret)
673a394b 2619 return ret;
9731129c 2620
673a394b
EA		 2621 goto search_free;
2622 }
2623
2624#if WATCH_BUF
cfd43c02 2625 DRM_INFO("Binding object of size %zd at 0x%08x\n",
673a394b
EA		 2626 obj->size, obj_priv->gtt_offset);
2627#endif
07f73f69
CW		 2628 if (retry_alloc) {
2629 i915_gem_object_set_page_gfp_mask (obj,
2630 i915_gem_object_get_page_gfp_mask (obj) & ~__GFP_NORETRY);
2631 }
856fa198 2632 ret = i915_gem_object_get_pages(obj);
07f73f69
CW		 2633 if (retry_alloc) {
2634 i915_gem_object_set_page_gfp_mask (obj,
2635 i915_gem_object_get_page_gfp_mask (obj) | __GFP_NORETRY);
2636 }
673a394b
EA		 2637 if (ret) {
2638 drm_mm_put_block(obj_priv->gtt_space);
2639 obj_priv->gtt_space = NULL;
07f73f69
CW		 2640
2641 if (ret == -ENOMEM) {
2642 /* first try to clear up some space from the GTT */
2643 ret = i915_gem_evict_something(dev, obj->size);
2644 if (ret) {
07f73f69
CW		 2645 /* now try to shrink everyone else */
2646 if (! retry_alloc) {
2647 retry_alloc = true;
2648 goto search_free;
2649 }
2650
2651 return ret;
2652 }
2653
2654 goto search_free;
2655 }
2656
673a394b
EA		 2657 return ret;
2658 }
2659
673a394b
EA		 2660 /* Create an AGP memory structure pointing at our pages, and bind it
2661 * into the GTT.
2662 */
2663 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2664 obj_priv->pages,
07f73f69 2665 obj->size >> PAGE_SHIFT,
ba1eb1d8
KP		 2666 obj_priv->gtt_offset,
2667 obj_priv->agp_type);
673a394b 2668 if (obj_priv->agp_mem == NULL) {
856fa198 2669 i915_gem_object_put_pages(obj);
673a394b
EA		 2670 drm_mm_put_block(obj_priv->gtt_space);
2671 obj_priv->gtt_space = NULL;
07f73f69
CW		 2672
2673 ret = i915_gem_evict_something(dev, obj->size);
9731129c 2674 if (ret)
07f73f69 2675 return ret;
07f73f69
CW		 2676
2677 goto search_free;
673a394b
EA		 2678 }
2679 atomic_inc(&dev->gtt_count);
2680 atomic_add(obj->size, &dev->gtt_memory);
2681
2682 /* Assert that the object is not currently in any GPU domain. As it
2683 * wasn't in the GTT, there shouldn't be any way it could have been in
2684 * a GPU cache
2685 */
21d509e3
CW		 2686 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2687 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b 2688
1c5d22f7
CW		 2689 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2690
673a394b
EA		 2691 return 0;
2692}
2693
2694void
2695i915_gem_clflush_object(struct drm_gem_object *obj)
2696{
2697 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2698
2699 /* If we don't have a page list set up, then we're not pinned
2700 * to GPU, and we can ignore the cache flush because it'll happen
2701 * again at bind time.
2702 */
856fa198 2703 if (obj_priv->pages == NULL)
673a394b
EA		 2704 return;
2705
1c5d22f7 2706 trace_i915_gem_object_clflush(obj);
cfa16a0d 2707
856fa198 2708 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2709}
2710
e47c68e9
EA		 2711/** Flushes any GPU write domain for the object if it's dirty. */
2712static void
2713i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2714{
2715 struct drm_device *dev = obj->dev;
2716 uint32_t seqno;
1c5d22f7 2717 uint32_t old_write_domain;
e47c68e9
EA		 2718
2719 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2720 return;
2721
2722 /* Queue the GPU write cache flushing we need. */
1c5d22f7 2723 old_write_domain = obj->write_domain;
e47c68e9 2724 i915_gem_flush(dev, 0, obj->write_domain);
b962442e 2725 seqno = i915_add_request(dev, NULL, obj->write_domain);
e47c68e9
EA		 2726 obj->write_domain = 0;
2727 i915_gem_object_move_to_active(obj, seqno);
1c5d22f7
CW		 2728
2729 trace_i915_gem_object_change_domain(obj,
2730 obj->read_domains,
2731 old_write_domain);
e47c68e9
EA		 2732}
2733
2734/** Flushes the GTT write domain for the object if it's dirty. */
2735static void
2736i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2737{
1c5d22f7
CW		 2738 uint32_t old_write_domain;
2739
e47c68e9
EA		 2740 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2741 return;
2742
2743 /* No actual flushing is required for the GTT write domain. Writes
2744 * to it immediately go to main memory as far as we know, so there's
2745 * no chipset flush. It also doesn't land in render cache.
2746 */
1c5d22f7 2747 old_write_domain = obj->write_domain;
e47c68e9 2748 obj->write_domain = 0;
1c5d22f7
CW		 2749
2750 trace_i915_gem_object_change_domain(obj,
2751 obj->read_domains,
2752 old_write_domain);
e47c68e9
EA		 2753}
2754
2755/** Flushes the CPU write domain for the object if it's dirty. */
2756static void
2757i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2758{
2759 struct drm_device *dev = obj->dev;
1c5d22f7 2760 uint32_t old_write_domain;
e47c68e9
EA		 2761
2762 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2763 return;
2764
2765 i915_gem_clflush_object(obj);
2766 drm_agp_chipset_flush(dev);
1c5d22f7 2767 old_write_domain = obj->write_domain;
e47c68e9 2768 obj->write_domain = 0;
1c5d22f7
CW		 2769
2770 trace_i915_gem_object_change_domain(obj,
2771 obj->read_domains,
2772 old_write_domain);
e47c68e9
EA		 2773}
2774
6b95a207
KH		 2775void
2776i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2777{
2778 switch (obj->write_domain) {
2779 case I915_GEM_DOMAIN_GTT:
2780 i915_gem_object_flush_gtt_write_domain(obj);
2781 break;
2782 case I915_GEM_DOMAIN_CPU:
2783 i915_gem_object_flush_cpu_write_domain(obj);
2784 break;
2785 default:
2786 i915_gem_object_flush_gpu_write_domain(obj);
2787 break;
2788 }
2789}
2790
2ef7eeaa
EA		 2791/**
2792 * Moves a single object to the GTT read, and possibly write domain.
2793 *
2794 * This function returns when the move is complete, including waiting on
2795 * flushes to occur.
2796 */
79e53945 2797int
2ef7eeaa
EA		 2798i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2799{
2ef7eeaa 2800 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1c5d22f7 2801 uint32_t old_write_domain, old_read_domains;
e47c68e9 2802 int ret;
2ef7eeaa 2803
02354392
EA		 2804 /* Not valid to be called on unbound objects. */
2805 if (obj_priv->gtt_space == NULL)
2806 return -EINVAL;
2807
e47c68e9
EA		 2808 i915_gem_object_flush_gpu_write_domain(obj);
2809 /* Wait on any GPU rendering and flushing to occur. */
2810 ret = i915_gem_object_wait_rendering(obj);
2811 if (ret != 0)
2812 return ret;
2813
1c5d22f7
CW		 2814 old_write_domain = obj->write_domain;
2815 old_read_domains = obj->read_domains;
2816
e47c68e9
EA		 2817 /* If we're writing through the GTT domain, then CPU and GPU caches
2818 * will need to be invalidated at next use.
2ef7eeaa 2819 */
e47c68e9
EA		 2820 if (write)
2821 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2ef7eeaa 2822
e47c68e9 2823 i915_gem_object_flush_cpu_write_domain(obj);
2ef7eeaa 2824
e47c68e9
EA		 2825 /* It should now be out of any other write domains, and we can update
2826 * the domain values for our changes.
2827 */
2828 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2829 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2830 if (write) {
2831 obj->write_domain = I915_GEM_DOMAIN_GTT;
2832 obj_priv->dirty = 1;
2ef7eeaa
EA		 2833 }
2834
1c5d22f7
CW		 2835 trace_i915_gem_object_change_domain(obj,
2836 old_read_domains,
2837 old_write_domain);
2838
e47c68e9
EA		 2839 return 0;
2840}
2841
2842/**
2843 * Moves a single object to the CPU read, and possibly write domain.
2844 *
2845 * This function returns when the move is complete, including waiting on
2846 * flushes to occur.
2847 */
2848static int
2849i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2850{
1c5d22f7 2851 uint32_t old_write_domain, old_read_domains;
e47c68e9
EA		 2852 int ret;
2853
2854 i915_gem_object_flush_gpu_write_domain(obj);
2ef7eeaa 2855 /* Wait on any GPU rendering and flushing to occur. */
e47c68e9
EA		 2856 ret = i915_gem_object_wait_rendering(obj);
2857 if (ret != 0)
2858 return ret;
2ef7eeaa 2859
e47c68e9 2860 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2861
e47c68e9
EA		 2862 /* If we have a partially-valid cache of the object in the CPU,
2863 * finish invalidating it and free the per-page flags.
2ef7eeaa 2864 */
e47c68e9 2865 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2866
1c5d22f7
CW		 2867 old_write_domain = obj->write_domain;
2868 old_read_domains = obj->read_domains;
2869
e47c68e9
EA		 2870 /* Flush the CPU cache if it's still invalid. */
2871 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2872 i915_gem_clflush_object(obj);
2ef7eeaa 2873
e47c68e9 2874 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2875 }
2876
2877 /* It should now be out of any other write domains, and we can update
2878 * the domain values for our changes.
2879 */
e47c68e9
EA		 2880 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2881
2882 /* If we're writing through the CPU, then the GPU read domains will
2883 * need to be invalidated at next use.
2884 */
2885 if (write) {
2886 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2887 obj->write_domain = I915_GEM_DOMAIN_CPU;
2888 }
2ef7eeaa 2889
1c5d22f7
CW		 2890 trace_i915_gem_object_change_domain(obj,
2891 old_read_domains,
2892 old_write_domain);
2893
2ef7eeaa
EA		 2894 return 0;
2895}
2896
673a394b
EA		 2897/*
2898 * Set the next domain for the specified object. This
2899 * may not actually perform the necessary flushing/invaliding though,
2900 * as that may want to be batched with other set_domain operations
2901 *
2902 * This is (we hope) the only really tricky part of gem. The goal
2903 * is fairly simple -- track which caches hold bits of the object
2904 * and make sure they remain coherent. A few concrete examples may
2905 * help to explain how it works. For shorthand, we use the notation
2906 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2907 * a pair of read and write domain masks.
2908 *
2909 * Case 1: the batch buffer
2910 *
2911 * 1. Allocated
2912 * 2. Written by CPU
2913 * 3. Mapped to GTT
2914 * 4. Read by GPU
2915 * 5. Unmapped from GTT
2916 * 6. Freed
2917 *
2918 * Let's take these a step at a time
2919 *
2920 * 1. Allocated
2921 * Pages allocated from the kernel may still have
2922 * cache contents, so we set them to (CPU, CPU) always.
2923 * 2. Written by CPU (using pwrite)
2924 * The pwrite function calls set_domain (CPU, CPU) and
2925 * this function does nothing (as nothing changes)
2926 * 3. Mapped by GTT
2927 * This function asserts that the object is not
2928 * currently in any GPU-based read or write domains
2929 * 4. Read by GPU
2930 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2931 * As write_domain is zero, this function adds in the
2932 * current read domains (CPU+COMMAND, 0).
2933 * flush_domains is set to CPU.
2934 * invalidate_domains is set to COMMAND
2935 * clflush is run to get data out of the CPU caches
2936 * then i915_dev_set_domain calls i915_gem_flush to
2937 * emit an MI_FLUSH and drm_agp_chipset_flush
2938 * 5. Unmapped from GTT
2939 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2940 * flush_domains and invalidate_domains end up both zero
2941 * so no flushing/invalidating happens
2942 * 6. Freed
2943 * yay, done
2944 *
2945 * Case 2: The shared render buffer
2946 *
2947 * 1. Allocated
2948 * 2. Mapped to GTT
2949 * 3. Read/written by GPU
2950 * 4. set_domain to (CPU,CPU)
2951 * 5. Read/written by CPU
2952 * 6. Read/written by GPU
2953 *
2954 * 1. Allocated
2955 * Same as last example, (CPU, CPU)
2956 * 2. Mapped to GTT
2957 * Nothing changes (assertions find that it is not in the GPU)
2958 * 3. Read/written by GPU
2959 * execbuffer calls set_domain (RENDER, RENDER)
2960 * flush_domains gets CPU
2961 * invalidate_domains gets GPU
2962 * clflush (obj)
2963 * MI_FLUSH and drm_agp_chipset_flush
2964 * 4. set_domain (CPU, CPU)
2965 * flush_domains gets GPU
2966 * invalidate_domains gets CPU
2967 * wait_rendering (obj) to make sure all drawing is complete.
2968 * This will include an MI_FLUSH to get the data from GPU
2969 * to memory
2970 * clflush (obj) to invalidate the CPU cache
2971 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2972 * 5. Read/written by CPU
2973 * cache lines are loaded and dirtied
2974 * 6. Read written by GPU
2975 * Same as last GPU access
2976 *
2977 * Case 3: The constant buffer
2978 *
2979 * 1. Allocated
2980 * 2. Written by CPU
2981 * 3. Read by GPU
2982 * 4. Updated (written) by CPU again
2983 * 5. Read by GPU
2984 *
2985 * 1. Allocated
2986 * (CPU, CPU)
2987 * 2. Written by CPU
2988 * (CPU, CPU)
2989 * 3. Read by GPU
2990 * (CPU+RENDER, 0)
2991 * flush_domains = CPU
2992 * invalidate_domains = RENDER
2993 * clflush (obj)
2994 * MI_FLUSH
2995 * drm_agp_chipset_flush
2996 * 4. Updated (written) by CPU again
2997 * (CPU, CPU)
2998 * flush_domains = 0 (no previous write domain)
2999 * invalidate_domains = 0 (no new read domains)
3000 * 5. Read by GPU
3001 * (CPU+RENDER, 0)
3002 * flush_domains = CPU
3003 * invalidate_domains = RENDER
3004 * clflush (obj)
3005 * MI_FLUSH
3006 * drm_agp_chipset_flush
3007 */
c0d90829 3008static void
8b0e378a 3009i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 3010{
3011 struct drm_device *dev = obj->dev;
3012 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3013 uint32_t invalidate_domains = 0;
3014 uint32_t flush_domains = 0;
1c5d22f7 3015 uint32_t old_read_domains;
e47c68e9 3016
8b0e378a
EA		 3017 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3018 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b 3019
652c393a
JB		 3020 intel_mark_busy(dev, obj);
3021
673a394b
EA		 3022#if WATCH_BUF
3023 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3024 __func__, obj,
8b0e378a
EA		 3025 obj->read_domains, obj->pending_read_domains,
3026 obj->write_domain, obj->pending_write_domain);
673a394b
EA		 3027#endif
3028 /*
3029 * If the object isn't moving to a new write domain,
3030 * let the object stay in multiple read domains
3031 */
8b0e378a
EA		 3032 if (obj->pending_write_domain == 0)
3033 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 3034 else
3035 obj_priv->dirty = 1;
3036
3037 /*
3038 * Flush the current write domain if
3039 * the new read domains don't match. Invalidate
3040 * any read domains which differ from the old
3041 * write domain
3042 */
8b0e378a
EA		 3043 if (obj->write_domain &&
3044 obj->write_domain != obj->pending_read_domains) {
673a394b 3045 flush_domains |= obj->write_domain;
8b0e378a
EA		 3046 invalidate_domains |=
3047 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 3048 }
3049 /*
3050 * Invalidate any read caches which may have
3051 * stale data. That is, any new read domains.
3052 */
8b0e378a 3053 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
673a394b
EA		 3054 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3055#if WATCH_BUF
3056 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3057 __func__, flush_domains, invalidate_domains);
3058#endif
673a394b
EA		 3059 i915_gem_clflush_object(obj);
3060 }
3061
1c5d22f7
CW		 3062 old_read_domains = obj->read_domains;
3063
efbeed96
EA		 3064 /* The actual obj->write_domain will be updated with
3065 * pending_write_domain after we emit the accumulated flush for all
3066 * of our domain changes in execbuffers (which clears objects'
3067 * write_domains). So if we have a current write domain that we
3068 * aren't changing, set pending_write_domain to that.
3069 */
3070 if (flush_domains == 0 && obj->pending_write_domain == 0)
3071 obj->pending_write_domain = obj->write_domain;
8b0e378a 3072 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 3073
3074 dev->invalidate_domains |= invalidate_domains;
3075 dev->flush_domains |= flush_domains;
3076#if WATCH_BUF
3077 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3078 __func__,
3079 obj->read_domains, obj->write_domain,
3080 dev->invalidate_domains, dev->flush_domains);
3081#endif
1c5d22f7
CW		 3082
3083 trace_i915_gem_object_change_domain(obj,
3084 old_read_domains,
3085 obj->write_domain);
673a394b
EA		 3086}
3087
3088/**
e47c68e9 3089 * Moves the object from a partially CPU read to a full one.
673a394b 3090 *
e47c68e9
EA		 3091 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3092 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 3093 */
e47c68e9
EA		 3094static void
3095i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b
EA		 3096{
3097 struct drm_i915_gem_object *obj_priv = obj->driver_private;
673a394b 3098
e47c68e9
EA		 3099 if (!obj_priv->page_cpu_valid)
3100 return;
3101
3102 /* If we're partially in the CPU read domain, finish moving it in.
3103 */
3104 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3105 int i;
3106
3107 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3108 if (obj_priv->page_cpu_valid[i])
3109 continue;
856fa198 3110 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 3111 }
e47c68e9
EA		 3112 }
3113
3114 /* Free the page_cpu_valid mappings which are now stale, whether
3115 * or not we've got I915_GEM_DOMAIN_CPU.
3116 */
9a298b2a 3117 kfree(obj_priv->page_cpu_valid);
e47c68e9
EA		 3118 obj_priv->page_cpu_valid = NULL;
3119}
3120
3121/**
3122 * Set the CPU read domain on a range of the object.
3123 *
3124 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3125 * not entirely valid. The page_cpu_valid member of the object flags which
3126 * pages have been flushed, and will be respected by
3127 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3128 * of the whole object.
3129 *
3130 * This function returns when the move is complete, including waiting on
3131 * flushes to occur.
3132 */
3133static int
3134i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3135 uint64_t offset, uint64_t size)
3136{
3137 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1c5d22f7 3138 uint32_t old_read_domains;
e47c68e9 3139 int i, ret;
673a394b 3140
e47c68e9
EA		 3141 if (offset == 0 && size == obj->size)
3142 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 3143
e47c68e9
EA		 3144 i915_gem_object_flush_gpu_write_domain(obj);
3145 /* Wait on any GPU rendering and flushing to occur. */
6a47baa6 3146 ret = i915_gem_object_wait_rendering(obj);
e47c68e9 3147 if (ret != 0)
6a47baa6 3148 return ret;
e47c68e9
EA		 3149 i915_gem_object_flush_gtt_write_domain(obj);
3150
3151 /* If we're already fully in the CPU read domain, we're done. */
3152 if (obj_priv->page_cpu_valid == NULL &&
3153 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
3154 return 0;
673a394b 3155
e47c68e9
EA		 3156 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3157 * newly adding I915_GEM_DOMAIN_CPU
3158 */
673a394b 3159 if (obj_priv->page_cpu_valid == NULL) {
9a298b2a
EA		 3160 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3161 GFP_KERNEL);
e47c68e9
EA		 3162 if (obj_priv->page_cpu_valid == NULL)
3163 return -ENOMEM;
3164 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3165 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 3166
3167 /* Flush the cache on any pages that are still invalid from the CPU's
3168 * perspective.
3169 */
e47c68e9
EA		 3170 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3171 i++) {
673a394b
EA		 3172 if (obj_priv->page_cpu_valid[i])
3173 continue;
3174
856fa198 3175 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 3176
3177 obj_priv->page_cpu_valid[i] = 1;
3178 }
3179
e47c68e9
EA		 3180 /* It should now be out of any other write domains, and we can update
3181 * the domain values for our changes.
3182 */
3183 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3184
1c5d22f7 3185 old_read_domains = obj->read_domains;
e47c68e9
EA		 3186 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3187
1c5d22f7
CW		 3188 trace_i915_gem_object_change_domain(obj,
3189 old_read_domains,
3190 obj->write_domain);
3191
673a394b
EA		 3192 return 0;
3193}
3194
673a394b
EA		 3195/**
3196 * Pin an object to the GTT and evaluate the relocations landing in it.
3197 */
3198static int
3199i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3200 struct drm_file *file_priv,
40a5f0de
EA		 3201 struct drm_i915_gem_exec_object *entry,
3202 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 3203{
3204 struct drm_device *dev = obj->dev;
0839ccb8 3205 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3206 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3207 int i, ret;
0839ccb8 3208 void __iomem *reloc_page;
673a394b
EA		 3209
3210 /* Choose the GTT offset for our buffer and put it there. */
3211 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3212 if (ret)
3213 return ret;
3214
3215 entry->offset = obj_priv->gtt_offset;
3216
673a394b
EA		 3217 /* Apply the relocations, using the GTT aperture to avoid cache
3218 * flushing requirements.
3219 */
3220 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 3221 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 3222 struct drm_gem_object *target_obj;
3223 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 3224 uint32_t reloc_val, reloc_offset;
3225 uint32_t __iomem *reloc_entry;
673a394b 3226
673a394b 3227 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 3228 reloc->target_handle);
673a394b
EA		 3229 if (target_obj == NULL) {
3230 i915_gem_object_unpin(obj);
3231 return -EBADF;
3232 }
3233 target_obj_priv = target_obj->driver_private;
3234
8542a0bb
CW		 3235#if WATCH_RELOC
3236 DRM_INFO("%s: obj %p offset %08x target %d "
3237 "read %08x write %08x gtt %08x "
3238 "presumed %08x delta %08x\n",
3239 __func__,
3240 obj,
3241 (int) reloc->offset,
3242 (int) reloc->target_handle,
3243 (int) reloc->read_domains,
3244 (int) reloc->write_domain,
3245 (int) target_obj_priv->gtt_offset,
3246 (int) reloc->presumed_offset,
3247 reloc->delta);
3248#endif
3249
673a394b
EA		 3250 /* The target buffer should have appeared before us in the
3251 * exec_object list, so it should have a GTT space bound by now.
3252 */
3253 if (target_obj_priv->gtt_space == NULL) {
3254 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 3255 reloc->target_handle);
673a394b
EA		 3256 drm_gem_object_unreference(target_obj);
3257 i915_gem_object_unpin(obj);
3258 return -EINVAL;
3259 }
3260
8542a0bb 3261 /* Validate that the target is in a valid r/w GPU domain */
40a5f0de
EA		 3262 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3263 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 3264 DRM_ERROR("reloc with read/write CPU domains: "
3265 "obj %p target %d offset %d "
3266 "read %08x write %08x",
40a5f0de
EA		 3267 obj, reloc->target_handle,
3268 (int) reloc->offset,
3269 reloc->read_domains,
3270 reloc->write_domain);
491152b8
CW		 3271 drm_gem_object_unreference(target_obj);
3272 i915_gem_object_unpin(obj);
e47c68e9
EA		 3273 return -EINVAL;
3274 }
40a5f0de
EA		 3275 if (reloc->write_domain && target_obj->pending_write_domain &&
3276 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 3277 DRM_ERROR("Write domain conflict: "
3278 "obj %p target %d offset %d "
3279 "new %08x old %08x\n",
40a5f0de
EA		 3280 obj, reloc->target_handle,
3281 (int) reloc->offset,
3282 reloc->write_domain,
673a394b
EA		 3283 target_obj->pending_write_domain);
3284 drm_gem_object_unreference(target_obj);
3285 i915_gem_object_unpin(obj);
3286 return -EINVAL;
3287 }
3288
40a5f0de
EA		 3289 target_obj->pending_read_domains |= reloc->read_domains;
3290 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 3291
3292 /* If the relocation already has the right value in it, no
3293 * more work needs to be done.
3294 */
40a5f0de 3295 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 3296 drm_gem_object_unreference(target_obj);
3297 continue;
3298 }
3299
8542a0bb
CW		 3300 /* Check that the relocation address is valid... */
3301 if (reloc->offset > obj->size - 4) {
3302 DRM_ERROR("Relocation beyond object bounds: "
3303 "obj %p target %d offset %d size %d.\n",
3304 obj, reloc->target_handle,
3305 (int) reloc->offset, (int) obj->size);
3306 drm_gem_object_unreference(target_obj);
3307 i915_gem_object_unpin(obj);
3308 return -EINVAL;
3309 }
3310 if (reloc->offset & 3) {
3311 DRM_ERROR("Relocation not 4-byte aligned: "
3312 "obj %p target %d offset %d.\n",
3313 obj, reloc->target_handle,
3314 (int) reloc->offset);
3315 drm_gem_object_unreference(target_obj);
3316 i915_gem_object_unpin(obj);
3317 return -EINVAL;
3318 }
3319
3320 /* and points to somewhere within the target object. */
3321 if (reloc->delta >= target_obj->size) {
3322 DRM_ERROR("Relocation beyond target object bounds: "
3323 "obj %p target %d delta %d size %d.\n",
3324 obj, reloc->target_handle,
3325 (int) reloc->delta, (int) target_obj->size);
3326 drm_gem_object_unreference(target_obj);
3327 i915_gem_object_unpin(obj);
3328 return -EINVAL;
3329 }
3330
2ef7eeaa
EA		 3331 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3332 if (ret != 0) {
3333 drm_gem_object_unreference(target_obj);
3334 i915_gem_object_unpin(obj);
3335 return -EINVAL;
673a394b
EA		 3336 }
3337
3338 /* Map the page containing the relocation we're going to
3339 * perform.
3340 */
40a5f0de 3341 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 3342 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3343 (reloc_offset &
3344 ~(PAGE_SIZE - 1)));
3043c60c 3345 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 3346 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 3347 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b
EA		 3348
3349#if WATCH_BUF
3350 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
40a5f0de 3351 obj, (unsigned int) reloc->offset,
673a394b
EA		 3352 readl(reloc_entry), reloc_val);
3353#endif
3354 writel(reloc_val, reloc_entry);
0839ccb8 3355 io_mapping_unmap_atomic(reloc_page);
673a394b 3356
40a5f0de
EA		 3357 /* The updated presumed offset for this entry will be
3358 * copied back out to the user.
673a394b 3359 */
40a5f0de 3360 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 3361
3362 drm_gem_object_unreference(target_obj);
3363 }
3364
673a394b
EA		 3365#if WATCH_BUF
3366 if (0)
3367 i915_gem_dump_object(obj, 128, __func__, ~0);
3368#endif
3369 return 0;
3370}
3371
3372/** Dispatch a batchbuffer to the ring
3373 */
3374static int
3375i915_dispatch_gem_execbuffer(struct drm_device *dev,
3376 struct drm_i915_gem_execbuffer *exec,
201361a5 3377 struct drm_clip_rect *cliprects,
673a394b
EA		 3378 uint64_t exec_offset)
3379{
3380 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3381 int nbox = exec->num_cliprects;
3382 int i = 0, count;
83d60795 3383 uint32_t exec_start, exec_len;
673a394b
EA		 3384 RING_LOCALS;
3385
3386 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3387 exec_len = (uint32_t) exec->batch_len;
3388
8f0dc5bf 3389 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
1c5d22f7 3390
673a394b
EA		 3391 count = nbox ? nbox : 1;
3392
3393 for (i = 0; i < count; i++) {
3394 if (i < nbox) {
201361a5 3395 int ret = i915_emit_box(dev, cliprects, i,
673a394b
EA		 3396 exec->DR1, exec->DR4);
3397 if (ret)
3398 return ret;
3399 }
3400
3401 if (IS_I830(dev) || IS_845G(dev)) {
3402 BEGIN_LP_RING(4);
3403 OUT_RING(MI_BATCH_BUFFER);
3404 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3405 OUT_RING(exec_start + exec_len - 4);
3406 OUT_RING(0);
3407 ADVANCE_LP_RING();
3408 } else {
3409 BEGIN_LP_RING(2);
3410 if (IS_I965G(dev)) {
3411 OUT_RING(MI_BATCH_BUFFER_START |
3412 (2 << 6) |
3413 MI_BATCH_NON_SECURE_I965);
3414 OUT_RING(exec_start);
3415 } else {
3416 OUT_RING(MI_BATCH_BUFFER_START |
3417 (2 << 6));
3418 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3419 }
3420 ADVANCE_LP_RING();
3421 }
3422 }
3423
3424 /* XXX breadcrumb */
3425 return 0;
3426}
3427
3428/* Throttle our rendering by waiting until the ring has completed our requests
3429 * emitted over 20 msec ago.
3430 *
b962442e
EA		 3431 * Note that if we were to use the current jiffies each time around the loop,
3432 * we wouldn't escape the function with any frames outstanding if the time to
3433 * render a frame was over 20ms.
3434 *
673a394b
EA		 3435 * This should get us reasonable parallelism between CPU and GPU but also
3436 * relatively low latency when blocking on a particular request to finish.
3437 */
3438static int
3439i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3440{
3441 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3442 int ret = 0;
b962442e 3443 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
673a394b
EA		 3444
3445 mutex_lock(&dev->struct_mutex);
b962442e
EA		 3446 while (!list_empty(&i915_file_priv->mm.request_list)) {
3447 struct drm_i915_gem_request *request;
3448
3449 request = list_first_entry(&i915_file_priv->mm.request_list,
3450 struct drm_i915_gem_request,
3451 client_list);
3452
3453 if (time_after_eq(request->emitted_jiffies, recent_enough))
3454 break;
3455
3456 ret = i915_wait_request(dev, request->seqno);
3457 if (ret != 0)
3458 break;
3459 }
673a394b 3460 mutex_unlock(&dev->struct_mutex);
b962442e 3461
673a394b
EA		 3462 return ret;
3463}
3464
40a5f0de
EA		 3465static int
3466i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list,
3467 uint32_t buffer_count,
3468 struct drm_i915_gem_relocation_entry **relocs)
3469{
3470 uint32_t reloc_count = 0, reloc_index = 0, i;
3471 int ret;
3472
3473 *relocs = NULL;
3474 for (i = 0; i < buffer_count; i++) {
3475 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3476 return -EINVAL;
3477 reloc_count += exec_list[i].relocation_count;
3478 }
3479
8e7d2b2c 3480 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
40a5f0de
EA		 3481 if (*relocs == NULL)
3482 return -ENOMEM;
3483
3484 for (i = 0; i < buffer_count; i++) {
3485 struct drm_i915_gem_relocation_entry __user *user_relocs;
3486
3487 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3488
3489 ret = copy_from_user(&(*relocs)[reloc_index],
3490 user_relocs,
3491 exec_list[i].relocation_count *
3492 sizeof(**relocs));
3493 if (ret != 0) {
8e7d2b2c 3494 drm_free_large(*relocs);
40a5f0de 3495 *relocs = NULL;
2bc43b5c 3496 return -EFAULT;
40a5f0de
EA		 3497 }
3498
3499 reloc_index += exec_list[i].relocation_count;
3500 }
3501
2bc43b5c 3502 return 0;
40a5f0de
EA		 3503}
3504
3505static int
3506i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list,
3507 uint32_t buffer_count,
3508 struct drm_i915_gem_relocation_entry *relocs)
3509{
3510 uint32_t reloc_count = 0, i;
2bc43b5c 3511 int ret = 0;
40a5f0de
EA		 3512
3513 for (i = 0; i < buffer_count; i++) {
3514 struct drm_i915_gem_relocation_entry __user *user_relocs;
2bc43b5c 3515 int unwritten;
40a5f0de
EA		 3516
3517 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3518
2bc43b5c
FM		 3519 unwritten = copy_to_user(user_relocs,
3520 &relocs[reloc_count],
3521 exec_list[i].relocation_count *
3522 sizeof(*relocs));
3523
3524 if (unwritten) {
3525 ret = -EFAULT;
3526 goto err;
40a5f0de
EA		 3527 }
3528
3529 reloc_count += exec_list[i].relocation_count;
3530 }
3531
2bc43b5c 3532err:
8e7d2b2c 3533 drm_free_large(relocs);
40a5f0de
EA		 3534
3535 return ret;
3536}
3537
83d60795
CW		 3538static int
3539i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer *exec,
3540 uint64_t exec_offset)
3541{
3542 uint32_t exec_start, exec_len;
3543
3544 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3545 exec_len = (uint32_t) exec->batch_len;
3546
3547 if ((exec_start | exec_len) & 0x7)
3548 return -EINVAL;
3549
3550 if (!exec_start)
3551 return -EINVAL;
3552
3553 return 0;
3554}
3555
6b95a207
KH		 3556static int
3557i915_gem_wait_for_pending_flip(struct drm_device *dev,
3558 struct drm_gem_object **object_list,
3559 int count)
3560{
3561 drm_i915_private_t *dev_priv = dev->dev_private;
3562 struct drm_i915_gem_object *obj_priv;
3563 DEFINE_WAIT(wait);
3564 int i, ret = 0;
3565
3566 for (;;) {
3567 prepare_to_wait(&dev_priv->pending_flip_queue,
3568 &wait, TASK_INTERRUPTIBLE);
3569 for (i = 0; i < count; i++) {
3570 obj_priv = object_list[i]->driver_private;
3571 if (atomic_read(&obj_priv->pending_flip) > 0)
3572 break;
3573 }
3574 if (i == count)
3575 break;
3576
3577 if (!signal_pending(current)) {
3578 mutex_unlock(&dev->struct_mutex);
3579 schedule();
3580 mutex_lock(&dev->struct_mutex);
3581 continue;
3582 }
3583 ret = -ERESTARTSYS;
3584 break;
3585 }
3586 finish_wait(&dev_priv->pending_flip_queue, &wait);
3587
3588 return ret;
3589}
3590
673a394b
EA		 3591int
3592i915_gem_execbuffer(struct drm_device *dev, void *data,
3593 struct drm_file *file_priv)
3594{
3595 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3596 struct drm_i915_gem_execbuffer *args = data;
3597 struct drm_i915_gem_exec_object *exec_list = NULL;
3598 struct drm_gem_object **object_list = NULL;
3599 struct drm_gem_object *batch_obj;
b70d11da 3600 struct drm_i915_gem_object *obj_priv;
201361a5 3601 struct drm_clip_rect *cliprects = NULL;
40a5f0de
EA		 3602 struct drm_i915_gem_relocation_entry *relocs;
3603 int ret, ret2, i, pinned = 0;
673a394b 3604 uint64_t exec_offset;
40a5f0de 3605 uint32_t seqno, flush_domains, reloc_index;
6b95a207 3606 int pin_tries, flips;
673a394b
EA		 3607
3608#if WATCH_EXEC
3609 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3610 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3611#endif
3612
4f481ed2
EA		 3613 if (args->buffer_count < 1) {
3614 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3615 return -EINVAL;
3616 }
673a394b 3617 /* Copy in the exec list from userland */
c8e0f93a
EA		 3618 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3619 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
673a394b
EA		 3620 if (exec_list == NULL || object_list == NULL) {
3621 DRM_ERROR("Failed to allocate exec or object list "
3622 "for %d buffers\n",
3623 args->buffer_count);
3624 ret = -ENOMEM;
3625 goto pre_mutex_err;
3626 }
3627 ret = copy_from_user(exec_list,
3628 (struct drm_i915_relocation_entry __user *)
3629 (uintptr_t) args->buffers_ptr,
3630 sizeof(*exec_list) * args->buffer_count);
3631 if (ret != 0) {
3632 DRM_ERROR("copy %d exec entries failed %d\n",
3633 args->buffer_count, ret);
3634 goto pre_mutex_err;
3635 }
3636
201361a5 3637 if (args->num_cliprects != 0) {
9a298b2a
EA		 3638 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3639 GFP_KERNEL);
201361a5
EA		 3640 if (cliprects == NULL)
3641 goto pre_mutex_err;
3642
3643 ret = copy_from_user(cliprects,
3644 (struct drm_clip_rect __user *)
3645 (uintptr_t) args->cliprects_ptr,
3646 sizeof(*cliprects) * args->num_cliprects);
3647 if (ret != 0) {
3648 DRM_ERROR("copy %d cliprects failed: %d\n",
3649 args->num_cliprects, ret);
3650 goto pre_mutex_err;
3651 }
3652 }
3653
40a5f0de
EA		 3654 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3655 &relocs);
3656 if (ret != 0)
3657 goto pre_mutex_err;
3658
673a394b
EA		 3659 mutex_lock(&dev->struct_mutex);
3660
3661 i915_verify_inactive(dev, __FILE__, __LINE__);
3662
ba1234d1 3663 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 3664 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3665 ret = -EIO;
3666 goto pre_mutex_err;
673a394b
EA		 3667 }
3668
3669 if (dev_priv->mm.suspended) {
673a394b 3670 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3671 ret = -EBUSY;
3672 goto pre_mutex_err;
673a394b
EA		 3673 }
3674
ac94a962 3675 /* Look up object handles */
6b95a207 3676 flips = 0;
673a394b
EA		 3677 for (i = 0; i < args->buffer_count; i++) {
3678 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3679 exec_list[i].handle);
3680 if (object_list[i] == NULL) {
3681 DRM_ERROR("Invalid object handle %d at index %d\n",
3682 exec_list[i].handle, i);
3683 ret = -EBADF;
3684 goto err;
3685 }
b70d11da
KH		 3686
3687 obj_priv = object_list[i]->driver_private;
3688 if (obj_priv->in_execbuffer) {
3689 DRM_ERROR("Object %p appears more than once in object list\n",
3690 object_list[i]);
3691 ret = -EBADF;
3692 goto err;
3693 }
3694 obj_priv->in_execbuffer = true;
6b95a207
KH		 3695 flips += atomic_read(&obj_priv->pending_flip);
3696 }
3697
3698 if (flips > 0) {
3699 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3700 args->buffer_count);
3701 if (ret)
3702 goto err;
ac94a962 3703 }
673a394b 3704
ac94a962
KP		 3705 /* Pin and relocate */
3706 for (pin_tries = 0; ; pin_tries++) {
3707 ret = 0;
40a5f0de
EA		 3708 reloc_index = 0;
3709
ac94a962
KP		 3710 for (i = 0; i < args->buffer_count; i++) {
3711 object_list[i]->pending_read_domains = 0;
3712 object_list[i]->pending_write_domain = 0;
3713 ret = i915_gem_object_pin_and_relocate(object_list[i],
3714 file_priv,
40a5f0de
EA		 3715 &exec_list[i],
3716 &relocs[reloc_index]);
ac94a962
KP		 3717 if (ret)
3718 break;
3719 pinned = i + 1;
40a5f0de 3720 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3721 }
3722 /* success */
3723 if (ret == 0)
3724 break;
3725
3726 /* error other than GTT full, or we've already tried again */
2939e1f5 3727 if (ret != -ENOSPC || pin_tries >= 1) {
07f73f69
CW		 3728 if (ret != -ERESTARTSYS) {
3729 unsigned long long total_size = 0;
3730 for (i = 0; i < args->buffer_count; i++)
3731 total_size += object_list[i]->size;
3732 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3733 pinned+1, args->buffer_count,
3734 total_size, ret);
3735 DRM_ERROR("%d objects [%d pinned], "
3736 "%d object bytes [%d pinned], "
3737 "%d/%d gtt bytes\n",
3738 atomic_read(&dev->object_count),
3739 atomic_read(&dev->pin_count),
3740 atomic_read(&dev->object_memory),
3741 atomic_read(&dev->pin_memory),
3742 atomic_read(&dev->gtt_memory),
3743 dev->gtt_total);
3744 }
673a394b
EA		 3745 goto err;
3746 }
ac94a962
KP		 3747
3748 /* unpin all of our buffers */
3749 for (i = 0; i < pinned; i++)
3750 i915_gem_object_unpin(object_list[i]);
b1177636 3751 pinned = 0;
ac94a962
KP		 3752
3753 /* evict everyone we can from the aperture */
3754 ret = i915_gem_evict_everything(dev);
07f73f69 3755 if (ret && ret != -ENOSPC)
ac94a962 3756 goto err;
673a394b
EA		 3757 }
3758
3759 /* Set the pending read domains for the batch buffer to COMMAND */
3760 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3761 if (batch_obj->pending_write_domain) {
3762 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3763 ret = -EINVAL;
3764 goto err;
3765 }
3766 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3767
83d60795
CW		 3768 /* Sanity check the batch buffer, prior to moving objects */
3769 exec_offset = exec_list[args->buffer_count - 1].offset;
3770 ret = i915_gem_check_execbuffer (args, exec_offset);
3771 if (ret != 0) {
3772 DRM_ERROR("execbuf with invalid offset/length\n");
3773 goto err;
3774 }
3775
673a394b
EA		 3776 i915_verify_inactive(dev, __FILE__, __LINE__);
3777
646f0f6e
KP		 3778 /* Zero the global flush/invalidate flags. These
3779 * will be modified as new domains are computed
3780 * for each object
3781 */
3782 dev->invalidate_domains = 0;
3783 dev->flush_domains = 0;
3784
673a394b
EA		 3785 for (i = 0; i < args->buffer_count; i++) {
3786 struct drm_gem_object *obj = object_list[i];
673a394b 3787
646f0f6e 3788 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3789 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3790 }
3791
3792 i915_verify_inactive(dev, __FILE__, __LINE__);
3793
646f0f6e
KP		 3794 if (dev->invalidate_domains | dev->flush_domains) {
3795#if WATCH_EXEC
3796 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3797 __func__,
3798 dev->invalidate_domains,
3799 dev->flush_domains);
3800#endif
3801 i915_gem_flush(dev,
3802 dev->invalidate_domains,
3803 dev->flush_domains);
3804 if (dev->flush_domains)
b962442e
EA		 3805 (void)i915_add_request(dev, file_priv,
3806 dev->flush_domains);
646f0f6e 3807 }
673a394b 3808
efbeed96
EA		 3809 for (i = 0; i < args->buffer_count; i++) {
3810 struct drm_gem_object *obj = object_list[i];
1c5d22f7 3811 uint32_t old_write_domain = obj->write_domain;
efbeed96
EA		 3812
3813 obj->write_domain = obj->pending_write_domain;
1c5d22f7
CW		 3814 trace_i915_gem_object_change_domain(obj,
3815 obj->read_domains,
3816 old_write_domain);
efbeed96
EA		 3817 }
3818
673a394b
EA		 3819 i915_verify_inactive(dev, __FILE__, __LINE__);
3820
3821#if WATCH_COHERENCY
3822 for (i = 0; i < args->buffer_count; i++) {
3823 i915_gem_object_check_coherency(object_list[i],
3824 exec_list[i].handle);
3825 }
3826#endif
3827
673a394b 3828#if WATCH_EXEC
6911a9b8 3829 i915_gem_dump_object(batch_obj,
673a394b
EA		 3830 args->batch_len,
3831 __func__,
3832 ~0);
3833#endif
3834
673a394b 3835 /* Exec the batchbuffer */
201361a5 3836 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
673a394b
EA		 3837 if (ret) {
3838 DRM_ERROR("dispatch failed %d\n", ret);
3839 goto err;
3840 }
3841
3842 /*
3843 * Ensure that the commands in the batch buffer are
3844 * finished before the interrupt fires
3845 */
3846 flush_domains = i915_retire_commands(dev);
3847
3848 i915_verify_inactive(dev, __FILE__, __LINE__);
3849
3850 /*
3851 * Get a seqno representing the execution of the current buffer,
3852 * which we can wait on. We would like to mitigate these interrupts,
3853 * likely by only creating seqnos occasionally (so that we have
3854 * *some* interrupts representing completion of buffers that we can
3855 * wait on when trying to clear up gtt space).
3856 */
b962442e 3857 seqno = i915_add_request(dev, file_priv, flush_domains);
673a394b 3858 BUG_ON(seqno == 0);
673a394b
EA		 3859 for (i = 0; i < args->buffer_count; i++) {
3860 struct drm_gem_object *obj = object_list[i];
673a394b 3861
ce44b0ea 3862 i915_gem_object_move_to_active(obj, seqno);
673a394b
EA		 3863#if WATCH_LRU
3864 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3865#endif
3866 }
3867#if WATCH_LRU
3868 i915_dump_lru(dev, __func__);
3869#endif
3870
3871 i915_verify_inactive(dev, __FILE__, __LINE__);
3872
673a394b 3873err:
aad87dff
JL		 3874 for (i = 0; i < pinned; i++)
3875 i915_gem_object_unpin(object_list[i]);
3876
b70d11da
KH		 3877 for (i = 0; i < args->buffer_count; i++) {
3878 if (object_list[i]) {
3879 obj_priv = object_list[i]->driver_private;
3880 obj_priv->in_execbuffer = false;
3881 }
aad87dff 3882 drm_gem_object_unreference(object_list[i]);
b70d11da 3883 }
673a394b 3884
673a394b
EA		 3885 mutex_unlock(&dev->struct_mutex);
3886
a35f2e2b
RD		 3887 if (!ret) {
3888 /* Copy the new buffer offsets back to the user's exec list. */
3889 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3890 (uintptr_t) args->buffers_ptr,
3891 exec_list,
3892 sizeof(*exec_list) * args->buffer_count);
2bc43b5c
FM		 3893 if (ret) {
3894 ret = -EFAULT;
a35f2e2b
RD		 3895 DRM_ERROR("failed to copy %d exec entries "
3896 "back to user (%d)\n",
3897 args->buffer_count, ret);
2bc43b5c 3898 }
a35f2e2b
RD		 3899 }
3900
40a5f0de
EA		 3901 /* Copy the updated relocations out regardless of current error
3902 * state. Failure to update the relocs would mean that the next
3903 * time userland calls execbuf, it would do so with presumed offset
3904 * state that didn't match the actual object state.
3905 */
3906 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3907 relocs);
3908 if (ret2 != 0) {
3909 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3910
3911 if (ret == 0)
3912 ret = ret2;
3913 }
3914
673a394b 3915pre_mutex_err:
8e7d2b2c
JB		 3916 drm_free_large(object_list);
3917 drm_free_large(exec_list);
9a298b2a 3918 kfree(cliprects);
673a394b
EA		 3919
3920 return ret;
3921}
3922
3923int
3924i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
3925{
3926 struct drm_device *dev = obj->dev;
3927 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3928 int ret;
3929
3930 i915_verify_inactive(dev, __FILE__, __LINE__);
3931 if (obj_priv->gtt_space == NULL) {
3932 ret = i915_gem_object_bind_to_gtt(obj, alignment);
9731129c 3933 if (ret)
673a394b 3934 return ret;
22c344e9
CW		 3935 }
3936 /*
3937 * Pre-965 chips need a fence register set up in order to
3938 * properly handle tiled surfaces.
3939 */
a09ba7fa 3940 if (!IS_I965G(dev) && obj_priv->tiling_mode != I915_TILING_NONE) {
8c4b8c3f 3941 ret = i915_gem_object_get_fence_reg(obj);
22c344e9
CW		 3942 if (ret != 0) {
3943 if (ret != -EBUSY && ret != -ERESTARTSYS)
3944 DRM_ERROR("Failure to install fence: %d\n",
3945 ret);
3946 return ret;
3947 }
673a394b
EA		 3948 }
3949 obj_priv->pin_count++;
3950
3951 /* If the object is not active and not pending a flush,
3952 * remove it from the inactive list
3953 */
3954 if (obj_priv->pin_count == 1) {
3955 atomic_inc(&dev->pin_count);
3956 atomic_add(obj->size, &dev->pin_memory);
3957 if (!obj_priv->active &&
21d509e3 3958 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
673a394b
EA		 3959 !list_empty(&obj_priv->list))
3960 list_del_init(&obj_priv->list);
3961 }
3962 i915_verify_inactive(dev, __FILE__, __LINE__);
3963
3964 return 0;
3965}
3966
3967void
3968i915_gem_object_unpin(struct drm_gem_object *obj)
3969{
3970 struct drm_device *dev = obj->dev;
3971 drm_i915_private_t *dev_priv = dev->dev_private;
3972 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3973
3974 i915_verify_inactive(dev, __FILE__, __LINE__);
3975 obj_priv->pin_count--;
3976 BUG_ON(obj_priv->pin_count < 0);
3977 BUG_ON(obj_priv->gtt_space == NULL);
3978
3979 /* If the object is no longer pinned, and is
3980 * neither active nor being flushed, then stick it on
3981 * the inactive list
3982 */
3983 if (obj_priv->pin_count == 0) {
3984 if (!obj_priv->active &&
21d509e3 3985 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
673a394b
EA		 3986 list_move_tail(&obj_priv->list,
3987 &dev_priv->mm.inactive_list);
3988 atomic_dec(&dev->pin_count);
3989 atomic_sub(obj->size, &dev->pin_memory);
3990 }
3991 i915_verify_inactive(dev, __FILE__, __LINE__);
3992}
3993
3994int
3995i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3996 struct drm_file *file_priv)
3997{
3998 struct drm_i915_gem_pin *args = data;
3999 struct drm_gem_object *obj;
4000 struct drm_i915_gem_object *obj_priv;
4001 int ret;
4002
4003 mutex_lock(&dev->struct_mutex);
4004
4005 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4006 if (obj == NULL) {
4007 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4008 args->handle);
4009 mutex_unlock(&dev->struct_mutex);
4010 return -EBADF;
4011 }
4012 obj_priv = obj->driver_private;
4013
bb6baf76
CW		 4014 if (obj_priv->madv != I915_MADV_WILLNEED) {
4015 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3ef94daa
CW		 4016 drm_gem_object_unreference(obj);
4017 mutex_unlock(&dev->struct_mutex);
4018 return -EINVAL;
4019 }
4020
79e53945
JB		 4021 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4022 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4023 args->handle);
96dec61d 4024 drm_gem_object_unreference(obj);
673a394b 4025 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 4026 return -EINVAL;
4027 }
4028
4029 obj_priv->user_pin_count++;
4030 obj_priv->pin_filp = file_priv;
4031 if (obj_priv->user_pin_count == 1) {
4032 ret = i915_gem_object_pin(obj, args->alignment);
4033 if (ret != 0) {
4034 drm_gem_object_unreference(obj);
4035 mutex_unlock(&dev->struct_mutex);
4036 return ret;
4037 }
673a394b
EA		 4038 }
4039
4040 /* XXX - flush the CPU caches for pinned objects
4041 * as the X server doesn't manage domains yet
4042 */
e47c68e9 4043 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 4044 args->offset = obj_priv->gtt_offset;
4045 drm_gem_object_unreference(obj);
4046 mutex_unlock(&dev->struct_mutex);
4047
4048 return 0;
4049}
4050
4051int
4052i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4053 struct drm_file *file_priv)
4054{
4055 struct drm_i915_gem_pin *args = data;
4056 struct drm_gem_object *obj;
79e53945 4057 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 4058
4059 mutex_lock(&dev->struct_mutex);
4060
4061 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4062 if (obj == NULL) {
4063 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4064 args->handle);
4065 mutex_unlock(&dev->struct_mutex);
4066 return -EBADF;
4067 }
4068
79e53945
JB		 4069 obj_priv = obj->driver_private;
4070 if (obj_priv->pin_filp != file_priv) {
4071 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4072 args->handle);
4073 drm_gem_object_unreference(obj);
4074 mutex_unlock(&dev->struct_mutex);
4075 return -EINVAL;
4076 }
4077 obj_priv->user_pin_count--;
4078 if (obj_priv->user_pin_count == 0) {
4079 obj_priv->pin_filp = NULL;
4080 i915_gem_object_unpin(obj);
4081 }
673a394b
EA		 4082
4083 drm_gem_object_unreference(obj);
4084 mutex_unlock(&dev->struct_mutex);
4085 return 0;
4086}
4087
4088int
4089i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4090 struct drm_file *file_priv)
4091{
4092 struct drm_i915_gem_busy *args = data;
4093 struct drm_gem_object *obj;
4094 struct drm_i915_gem_object *obj_priv;
4095
673a394b
EA		 4096 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4097 if (obj == NULL) {
4098 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4099 args->handle);
673a394b
EA		 4100 return -EBADF;
4101 }
4102
b1ce786c 4103 mutex_lock(&dev->struct_mutex);
f21289b3
EA		 4104 /* Update the active list for the hardware's current position.
4105 * Otherwise this only updates on a delayed timer or when irqs are
4106 * actually unmasked, and our working set ends up being larger than
4107 * required.
4108 */
4109 i915_gem_retire_requests(dev);
4110
673a394b 4111 obj_priv = obj->driver_private;
c4de0a5d
EA		 4112 /* Don't count being on the flushing list against the object being
4113 * done. Otherwise, a buffer left on the flushing list but not getting
4114 * flushed (because nobody's flushing that domain) won't ever return
4115 * unbusy and get reused by libdrm's bo cache. The other expected
4116 * consumer of this interface, OpenGL's occlusion queries, also specs
4117 * that the objects get unbusy "eventually" without any interference.
4118 */
4119 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
673a394b
EA		 4120
4121 drm_gem_object_unreference(obj);
4122 mutex_unlock(&dev->struct_mutex);
4123 return 0;
4124}
4125
4126int
4127i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4128 struct drm_file *file_priv)
4129{
4130 return i915_gem_ring_throttle(dev, file_priv);
4131}
4132
3ef94daa
CW		 4133int
4134i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4135 struct drm_file *file_priv)
4136{
4137 struct drm_i915_gem_madvise *args = data;
4138 struct drm_gem_object *obj;
4139 struct drm_i915_gem_object *obj_priv;
4140
4141 switch (args->madv) {
4142 case I915_MADV_DONTNEED:
4143 case I915_MADV_WILLNEED:
4144 break;
4145 default:
4146 return -EINVAL;
4147 }
4148
4149 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4150 if (obj == NULL) {
4151 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4152 args->handle);
4153 return -EBADF;
4154 }
4155
4156 mutex_lock(&dev->struct_mutex);
4157 obj_priv = obj->driver_private;
4158
4159 if (obj_priv->pin_count) {
4160 drm_gem_object_unreference(obj);
4161 mutex_unlock(&dev->struct_mutex);
4162
4163 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4164 return -EINVAL;
4165 }
4166
bb6baf76
CW		 4167 if (obj_priv->madv != __I915_MADV_PURGED)
4168 obj_priv->madv = args->madv;
3ef94daa 4169
2d7ef395
CW		 4170 /* if the object is no longer bound, discard its backing storage */
4171 if (i915_gem_object_is_purgeable(obj_priv) &&
4172 obj_priv->gtt_space == NULL)
4173 i915_gem_object_truncate(obj);
4174
bb6baf76
CW		 4175 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4176
3ef94daa
CW		 4177 drm_gem_object_unreference(obj);
4178 mutex_unlock(&dev->struct_mutex);
4179
4180 return 0;
4181}
4182
673a394b
EA		 4183int i915_gem_init_object(struct drm_gem_object *obj)
4184{
4185 struct drm_i915_gem_object *obj_priv;
4186
9a298b2a 4187 obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
673a394b
EA		 4188 if (obj_priv == NULL)
4189 return -ENOMEM;
4190
4191 /*
4192 * We've just allocated pages from the kernel,
4193 * so they've just been written by the CPU with
4194 * zeros. They'll need to be clflushed before we
4195 * use them with the GPU.
4196 */
4197 obj->write_domain = I915_GEM_DOMAIN_CPU;
4198 obj->read_domains = I915_GEM_DOMAIN_CPU;
4199
ba1eb1d8
KP		 4200 obj_priv->agp_type = AGP_USER_MEMORY;
4201
673a394b
EA		 4202 obj->driver_private = obj_priv;
4203 obj_priv->obj = obj;
de151cf6 4204 obj_priv->fence_reg = I915_FENCE_REG_NONE;
673a394b 4205 INIT_LIST_HEAD(&obj_priv->list);
a09ba7fa 4206 INIT_LIST_HEAD(&obj_priv->fence_list);
3ef94daa 4207 obj_priv->madv = I915_MADV_WILLNEED;
de151cf6 4208
1c5d22f7 4209 trace_i915_gem_object_create(obj);
de151cf6 4210
673a394b
EA		 4211 return 0;
4212}
4213
4214void i915_gem_free_object(struct drm_gem_object *obj)
4215{
de151cf6 4216 struct drm_device *dev = obj->dev;
673a394b
EA		 4217 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4218
1c5d22f7
CW		 4219 trace_i915_gem_object_destroy(obj);
4220
673a394b
EA		 4221 while (obj_priv->pin_count > 0)
4222 i915_gem_object_unpin(obj);
4223
71acb5eb
DA		 4224 if (obj_priv->phys_obj)
4225 i915_gem_detach_phys_object(dev, obj);
4226
673a394b
EA		 4227 i915_gem_object_unbind(obj);
4228
7e616158
CW		 4229 if (obj_priv->mmap_offset)
4230 i915_gem_free_mmap_offset(obj);
de151cf6 4231
9a298b2a 4232 kfree(obj_priv->page_cpu_valid);
280b713b 4233 kfree(obj_priv->bit_17);
9a298b2a 4234 kfree(obj->driver_private);
673a394b
EA		 4235}
4236
ab5ee576 4237/** Unbinds all inactive objects. */
673a394b 4238static int
ab5ee576 4239i915_gem_evict_from_inactive_list(struct drm_device *dev)
673a394b 4240{
ab5ee576 4241 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 4242
ab5ee576
CW		 4243 while (!list_empty(&dev_priv->mm.inactive_list)) {
4244 struct drm_gem_object *obj;
4245 int ret;
673a394b 4246
ab5ee576
CW		 4247 obj = list_first_entry(&dev_priv->mm.inactive_list,
4248 struct drm_i915_gem_object,
4249 list)->obj;
673a394b
EA		 4250
4251 ret = i915_gem_object_unbind(obj);
4252 if (ret != 0) {
ab5ee576 4253 DRM_ERROR("Error unbinding object: %d\n", ret);
673a394b
EA		 4254 return ret;
4255 }
4256 }
4257
673a394b
EA		 4258 return 0;
4259}
4260
5669fcac 4261int
673a394b
EA		 4262i915_gem_idle(struct drm_device *dev)
4263{
4264 drm_i915_private_t *dev_priv = dev->dev_private;
4265 uint32_t seqno, cur_seqno, last_seqno;
4266 int stuck, ret;
4267
6dbe2772
KP		 4268 mutex_lock(&dev->struct_mutex);
4269
4270 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4271 mutex_unlock(&dev->struct_mutex);
673a394b 4272 return 0;
6dbe2772 4273 }
673a394b
EA		 4274
4275 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4276 * We need to replace this with a semaphore, or something.
4277 */
4278 dev_priv->mm.suspended = 1;
f65d9421 4279 del_timer(&dev_priv->hangcheck_timer);
673a394b 4280
6dbe2772
KP		 4281 /* Cancel the retire work handler, wait for it to finish if running
4282 */
4283 mutex_unlock(&dev->struct_mutex);
4284 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4285 mutex_lock(&dev->struct_mutex);
4286
673a394b
EA		 4287 i915_kernel_lost_context(dev);
4288
4289 /* Flush the GPU along with all non-CPU write domains
4290 */
21d509e3
CW		 4291 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
4292 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
673a394b
EA		 4293
4294 if (seqno == 0) {
4295 mutex_unlock(&dev->struct_mutex);
4296 return -ENOMEM;
4297 }
4298
4299 dev_priv->mm.waiting_gem_seqno = seqno;
4300 last_seqno = 0;
4301 stuck = 0;
4302 for (;;) {
4303 cur_seqno = i915_get_gem_seqno(dev);
4304 if (i915_seqno_passed(cur_seqno, seqno))
4305 break;
4306 if (last_seqno == cur_seqno) {
4307 if (stuck++ > 100) {
4308 DRM_ERROR("hardware wedged\n");
ba1234d1 4309 atomic_set(&dev_priv->mm.wedged, 1);
673a394b
EA		 4310 DRM_WAKEUP(&dev_priv->irq_queue);
4311 break;
4312 }
4313 }
4314 msleep(10);
4315 last_seqno = cur_seqno;
4316 }
4317 dev_priv->mm.waiting_gem_seqno = 0;
4318
4319 i915_gem_retire_requests(dev);
4320
5e118f41 4321 spin_lock(&dev_priv->mm.active_list_lock);
ba1234d1 4322 if (!atomic_read(&dev_priv->mm.wedged)) {
28dfe52a
EA		 4323 /* Active and flushing should now be empty as we've
4324 * waited for a sequence higher than any pending execbuffer
4325 */
4326 WARN_ON(!list_empty(&dev_priv->mm.active_list));
4327 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
4328 /* Request should now be empty as we've also waited
4329 * for the last request in the list
4330 */
4331 WARN_ON(!list_empty(&dev_priv->mm.request_list));
4332 }
673a394b 4333
28dfe52a
EA		 4334 /* Empty the active and flushing lists to inactive. If there's
4335 * anything left at this point, it means that we're wedged and
4336 * nothing good's going to happen by leaving them there. So strip
4337 * the GPU domains and just stuff them onto inactive.
673a394b 4338 */
28dfe52a 4339 while (!list_empty(&dev_priv->mm.active_list)) {
1c5d22f7
CW		 4340 struct drm_gem_object *obj;
4341 uint32_t old_write_domain;
673a394b 4342
1c5d22f7
CW		 4343 obj = list_first_entry(&dev_priv->mm.active_list,
4344 struct drm_i915_gem_object,
4345 list)->obj;
4346 old_write_domain = obj->write_domain;
4347 obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
4348 i915_gem_object_move_to_inactive(obj);
4349
4350 trace_i915_gem_object_change_domain(obj,
4351 obj->read_domains,
4352 old_write_domain);
28dfe52a 4353 }
5e118f41 4354 spin_unlock(&dev_priv->mm.active_list_lock);
28dfe52a
EA		 4355
4356 while (!list_empty(&dev_priv->mm.flushing_list)) {
1c5d22f7
CW		 4357 struct drm_gem_object *obj;
4358 uint32_t old_write_domain;
28dfe52a 4359
1c5d22f7
CW		 4360 obj = list_first_entry(&dev_priv->mm.flushing_list,
4361 struct drm_i915_gem_object,
4362 list)->obj;
4363 old_write_domain = obj->write_domain;
4364 obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
4365 i915_gem_object_move_to_inactive(obj);
4366
4367 trace_i915_gem_object_change_domain(obj,
4368 obj->read_domains,
4369 old_write_domain);
28dfe52a
EA		 4370 }
4371
4372
4373 /* Move all inactive buffers out of the GTT. */
ab5ee576 4374 ret = i915_gem_evict_from_inactive_list(dev);
28dfe52a 4375 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
6dbe2772
KP		 4376 if (ret) {
4377 mutex_unlock(&dev->struct_mutex);
673a394b 4378 return ret;
6dbe2772 4379 }
673a394b 4380
6dbe2772
KP		 4381 i915_gem_cleanup_ringbuffer(dev);
4382 mutex_unlock(&dev->struct_mutex);
4383
673a394b
EA		 4384 return 0;
4385}
4386
4387static int
4388i915_gem_init_hws(struct drm_device *dev)
4389{
4390 drm_i915_private_t *dev_priv = dev->dev_private;
4391 struct drm_gem_object *obj;
4392 struct drm_i915_gem_object *obj_priv;
4393 int ret;
4394
4395 /* If we need a physical address for the status page, it's already
4396 * initialized at driver load time.
4397 */
4398 if (!I915_NEED_GFX_HWS(dev))
4399 return 0;
4400
4401 obj = drm_gem_object_alloc(dev, 4096);
4402 if (obj == NULL) {
4403 DRM_ERROR("Failed to allocate status page\n");
4404 return -ENOMEM;
4405 }
4406 obj_priv = obj->driver_private;
ba1eb1d8 4407 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
673a394b
EA		 4408
4409 ret = i915_gem_object_pin(obj, 4096);
4410 if (ret != 0) {
4411 drm_gem_object_unreference(obj);
4412 return ret;
4413 }
4414
4415 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
673a394b 4416
856fa198 4417 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
ba1eb1d8 4418 if (dev_priv->hw_status_page == NULL) {
673a394b
EA		 4419 DRM_ERROR("Failed to map status page.\n");
4420 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
3eb2ee77 4421 i915_gem_object_unpin(obj);
673a394b
EA		 4422 drm_gem_object_unreference(obj);
4423 return -EINVAL;
4424 }
4425 dev_priv->hws_obj = obj;
673a394b
EA		 4426 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4427 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
ba1eb1d8 4428 I915_READ(HWS_PGA); /* posting read */
44d98a61 4429 DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
673a394b
EA		 4430
4431 return 0;
4432}
4433
85a7bb98
CW		 4434static void
4435i915_gem_cleanup_hws(struct drm_device *dev)
4436{
4437 drm_i915_private_t *dev_priv = dev->dev_private;
bab2d1f6
CW		 4438 struct drm_gem_object *obj;
4439 struct drm_i915_gem_object *obj_priv;
85a7bb98
CW		 4440
4441 if (dev_priv->hws_obj == NULL)
4442 return;
4443
bab2d1f6
CW		 4444 obj = dev_priv->hws_obj;
4445 obj_priv = obj->driver_private;
4446
856fa198 4447 kunmap(obj_priv->pages[0]);
85a7bb98
CW		 4448 i915_gem_object_unpin(obj);
4449 drm_gem_object_unreference(obj);
4450 dev_priv->hws_obj = NULL;
bab2d1f6 4451
85a7bb98
CW		 4452 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4453 dev_priv->hw_status_page = NULL;
4454
4455 /* Write high address into HWS_PGA when disabling. */
4456 I915_WRITE(HWS_PGA, 0x1ffff000);
4457}
4458
79e53945 4459int
673a394b
EA		 4460i915_gem_init_ringbuffer(struct drm_device *dev)
4461{
4462 drm_i915_private_t *dev_priv = dev->dev_private;
4463 struct drm_gem_object *obj;
4464 struct drm_i915_gem_object *obj_priv;
79e53945 4465 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
673a394b 4466 int ret;
50aa253d 4467 u32 head;
673a394b
EA		 4468
4469 ret = i915_gem_init_hws(dev);
4470 if (ret != 0)
4471 return ret;
4472
4473 obj = drm_gem_object_alloc(dev, 128 * 1024);
4474 if (obj == NULL) {
4475 DRM_ERROR("Failed to allocate ringbuffer\n");
85a7bb98 4476 i915_gem_cleanup_hws(dev);
673a394b
EA		 4477 return -ENOMEM;
4478 }
4479 obj_priv = obj->driver_private;
4480
4481 ret = i915_gem_object_pin(obj, 4096);
4482 if (ret != 0) {
4483 drm_gem_object_unreference(obj);
85a7bb98 4484 i915_gem_cleanup_hws(dev);
673a394b
EA		 4485 return ret;
4486 }
4487
4488 /* Set up the kernel mapping for the ring. */
79e53945 4489 ring->Size = obj->size;
673a394b 4490
79e53945
JB		 4491 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4492 ring->map.size = obj->size;
4493 ring->map.type = 0;
4494 ring->map.flags = 0;
4495 ring->map.mtrr = 0;
673a394b 4496
79e53945
JB		 4497 drm_core_ioremap_wc(&ring->map, dev);
4498 if (ring->map.handle == NULL) {
673a394b
EA		 4499 DRM_ERROR("Failed to map ringbuffer.\n");
4500 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
47ed185a 4501 i915_gem_object_unpin(obj);
673a394b 4502 drm_gem_object_unreference(obj);
85a7bb98 4503 i915_gem_cleanup_hws(dev);
673a394b
EA		 4504 return -EINVAL;
4505 }
79e53945
JB		 4506 ring->ring_obj = obj;
4507 ring->virtual_start = ring->map.handle;
673a394b
EA		 4508
4509 /* Stop the ring if it's running. */
4510 I915_WRITE(PRB0_CTL, 0);
673a394b 4511 I915_WRITE(PRB0_TAIL, 0);
50aa253d 4512 I915_WRITE(PRB0_HEAD, 0);
673a394b
EA		 4513
4514 /* Initialize the ring. */
4515 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
50aa253d
KP		 4516 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4517
4518 /* G45 ring initialization fails to reset head to zero */
4519 if (head != 0) {
4520 DRM_ERROR("Ring head not reset to zero "
4521 "ctl %08x head %08x tail %08x start %08x\n",
4522 I915_READ(PRB0_CTL),
4523 I915_READ(PRB0_HEAD),
4524 I915_READ(PRB0_TAIL),
4525 I915_READ(PRB0_START));
4526 I915_WRITE(PRB0_HEAD, 0);
4527
4528 DRM_ERROR("Ring head forced to zero "
4529 "ctl %08x head %08x tail %08x start %08x\n",
4530 I915_READ(PRB0_CTL),
4531 I915_READ(PRB0_HEAD),
4532 I915_READ(PRB0_TAIL),
4533 I915_READ(PRB0_START));
4534 }
4535
673a394b
EA		 4536 I915_WRITE(PRB0_CTL,
4537 ((obj->size - 4096) & RING_NR_PAGES) |
4538 RING_NO_REPORT |
4539 RING_VALID);
4540
50aa253d
KP		 4541 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4542
4543 /* If the head is still not zero, the ring is dead */
4544 if (head != 0) {
4545 DRM_ERROR("Ring initialization failed "
4546 "ctl %08x head %08x tail %08x start %08x\n",
4547 I915_READ(PRB0_CTL),
4548 I915_READ(PRB0_HEAD),
4549 I915_READ(PRB0_TAIL),
4550 I915_READ(PRB0_START));
4551 return -EIO;
4552 }
4553
673a394b 4554 /* Update our cache of the ring state */
79e53945
JB		 4555 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4556 i915_kernel_lost_context(dev);
4557 else {
4558 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4559 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4560 ring->space = ring->head - (ring->tail + 8);
4561 if (ring->space < 0)
4562 ring->space += ring->Size;
4563 }
673a394b
EA		 4564
4565 return 0;
4566}
4567
79e53945 4568void
673a394b
EA		 4569i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4570{
4571 drm_i915_private_t *dev_priv = dev->dev_private;
4572
4573 if (dev_priv->ring.ring_obj == NULL)
4574 return;
4575
4576 drm_core_ioremapfree(&dev_priv->ring.map, dev);
4577
4578 i915_gem_object_unpin(dev_priv->ring.ring_obj);
4579 drm_gem_object_unreference(dev_priv->ring.ring_obj);
4580 dev_priv->ring.ring_obj = NULL;
4581 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4582
85a7bb98 4583 i915_gem_cleanup_hws(dev);
673a394b
EA		 4584}
4585
4586int
4587i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4588 struct drm_file *file_priv)
4589{
4590 drm_i915_private_t *dev_priv = dev->dev_private;
4591 int ret;
4592
79e53945
JB		 4593 if (drm_core_check_feature(dev, DRIVER_MODESET))
4594 return 0;
4595
ba1234d1 4596 if (atomic_read(&dev_priv->mm.wedged)) {
673a394b 4597 DRM_ERROR("Reenabling wedged hardware, good luck\n");
ba1234d1 4598 atomic_set(&dev_priv->mm.wedged, 0);
673a394b
EA		 4599 }
4600
673a394b 4601 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4602 dev_priv->mm.suspended = 0;
4603
4604 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4605 if (ret != 0) {
4606 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4607 return ret;
d816f6ac 4608 }
9bb2d6f9 4609
5e118f41 4610 spin_lock(&dev_priv->mm.active_list_lock);
673a394b 4611 BUG_ON(!list_empty(&dev_priv->mm.active_list));
5e118f41
CW		 4612 spin_unlock(&dev_priv->mm.active_list_lock);
4613
673a394b
EA		 4614 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4615 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4616 BUG_ON(!list_empty(&dev_priv->mm.request_list));
673a394b 4617 mutex_unlock(&dev->struct_mutex);
dbb19d30
KH		 4618
4619 drm_irq_install(dev);
4620
673a394b
EA		 4621 return 0;
4622}
4623
4624int
4625i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4626 struct drm_file *file_priv)
4627{
79e53945
JB		 4628 if (drm_core_check_feature(dev, DRIVER_MODESET))
4629 return 0;
4630
dbb19d30 4631 drm_irq_uninstall(dev);
e6890f6f 4632 return i915_gem_idle(dev);
673a394b
EA		 4633}
4634
4635void
4636i915_gem_lastclose(struct drm_device *dev)
4637{
4638 int ret;
673a394b 4639
e806b495
EA		 4640 if (drm_core_check_feature(dev, DRIVER_MODESET))
4641 return;
4642
6dbe2772
KP		 4643 ret = i915_gem_idle(dev);
4644 if (ret)
4645 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4646}
4647
4648void
4649i915_gem_load(struct drm_device *dev)
4650{
b5aa8a0f 4651 int i;
673a394b
EA		 4652 drm_i915_private_t *dev_priv = dev->dev_private;
4653
5e118f41 4654 spin_lock_init(&dev_priv->mm.active_list_lock);
673a394b
EA		 4655 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4656 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4657 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4658 INIT_LIST_HEAD(&dev_priv->mm.request_list);
a09ba7fa 4659 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
673a394b
EA		 4660 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4661 i915_gem_retire_work_handler);
4662 dev_priv->mm.next_gem_seqno = 1;
4663
31169714
CW		 4664 spin_lock(&shrink_list_lock);
4665 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4666 spin_unlock(&shrink_list_lock);
4667
de151cf6
JB		 4668 /* Old X drivers will take 0-2 for front, back, depth buffers */
4669 dev_priv->fence_reg_start = 3;
4670
0f973f27 4671 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4672 dev_priv->num_fence_regs = 16;
4673 else
4674 dev_priv->num_fence_regs = 8;
4675
b5aa8a0f
GH		 4676 /* Initialize fence registers to zero */
4677 if (IS_I965G(dev)) {
4678 for (i = 0; i < 16; i++)
4679 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4680 } else {
4681 for (i = 0; i < 8; i++)
4682 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4683 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4684 for (i = 0; i < 8; i++)
4685 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4686 }
673a394b 4687 i915_gem_detect_bit_6_swizzle(dev);
6b95a207 4688 init_waitqueue_head(&dev_priv->pending_flip_queue);
673a394b 4689}
71acb5eb
DA		 4690
4691/*
4692 * Create a physically contiguous memory object for this object
4693 * e.g. for cursor + overlay regs
4694 */
4695int i915_gem_init_phys_object(struct drm_device *dev,
4696 int id, int size)
4697{
4698 drm_i915_private_t *dev_priv = dev->dev_private;
4699 struct drm_i915_gem_phys_object *phys_obj;
4700 int ret;
4701
4702 if (dev_priv->mm.phys_objs[id - 1] || !size)
4703 return 0;
4704
9a298b2a 4705 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
71acb5eb
DA		 4706 if (!phys_obj)
4707 return -ENOMEM;
4708
4709 phys_obj->id = id;
4710
4711 phys_obj->handle = drm_pci_alloc(dev, size, 0, 0xffffffff);
4712 if (!phys_obj->handle) {
4713 ret = -ENOMEM;
4714 goto kfree_obj;
4715 }
4716#ifdef CONFIG_X86
4717 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4718#endif
4719
4720 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4721
4722 return 0;
4723kfree_obj:
9a298b2a 4724 kfree(phys_obj);
71acb5eb
DA		 4725 return ret;
4726}
4727
4728void i915_gem_free_phys_object(struct drm_device *dev, int id)
4729{
4730 drm_i915_private_t *dev_priv = dev->dev_private;
4731 struct drm_i915_gem_phys_object *phys_obj;
4732
4733 if (!dev_priv->mm.phys_objs[id - 1])
4734 return;
4735
4736 phys_obj = dev_priv->mm.phys_objs[id - 1];
4737 if (phys_obj->cur_obj) {
4738 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4739 }
4740
4741#ifdef CONFIG_X86
4742 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4743#endif
4744 drm_pci_free(dev, phys_obj->handle);
4745 kfree(phys_obj);
4746 dev_priv->mm.phys_objs[id - 1] = NULL;
4747}
4748
4749void i915_gem_free_all_phys_object(struct drm_device *dev)
4750{
4751 int i;
4752
260883c8 4753 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4754 i915_gem_free_phys_object(dev, i);
4755}
4756
4757void i915_gem_detach_phys_object(struct drm_device *dev,
4758 struct drm_gem_object *obj)
4759{
4760 struct drm_i915_gem_object *obj_priv;
4761 int i;
4762 int ret;
4763 int page_count;
4764
4765 obj_priv = obj->driver_private;
4766 if (!obj_priv->phys_obj)
4767 return;
4768
856fa198 4769 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4770 if (ret)
4771 goto out;
4772
4773 page_count = obj->size / PAGE_SIZE;
4774
4775 for (i = 0; i < page_count; i++) {
856fa198 4776 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4777 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4778
4779 memcpy(dst, src, PAGE_SIZE);
4780 kunmap_atomic(dst, KM_USER0);
4781 }
856fa198 4782 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb 4783 drm_agp_chipset_flush(dev);
d78b47b9
CW		 4784
4785 i915_gem_object_put_pages(obj);
71acb5eb
DA		 4786out:
4787 obj_priv->phys_obj->cur_obj = NULL;
4788 obj_priv->phys_obj = NULL;
4789}
4790
4791int
4792i915_gem_attach_phys_object(struct drm_device *dev,
4793 struct drm_gem_object *obj, int id)
4794{
4795 drm_i915_private_t *dev_priv = dev->dev_private;
4796 struct drm_i915_gem_object *obj_priv;
4797 int ret = 0;
4798 int page_count;
4799 int i;
4800
4801 if (id > I915_MAX_PHYS_OBJECT)
4802 return -EINVAL;
4803
4804 obj_priv = obj->driver_private;
4805
4806 if (obj_priv->phys_obj) {
4807 if (obj_priv->phys_obj->id == id)
4808 return 0;
4809 i915_gem_detach_phys_object(dev, obj);
4810 }
4811
4812
4813 /* create a new object */
4814 if (!dev_priv->mm.phys_objs[id - 1]) {
4815 ret = i915_gem_init_phys_object(dev, id,
4816 obj->size);
4817 if (ret) {
aeb565df 4818 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4819 goto out;
4820 }
4821 }
4822
4823 /* bind to the object */
4824 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4825 obj_priv->phys_obj->cur_obj = obj;
4826
856fa198 4827 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4828 if (ret) {
4829 DRM_ERROR("failed to get page list\n");
4830 goto out;
4831 }
4832
4833 page_count = obj->size / PAGE_SIZE;
4834
4835 for (i = 0; i < page_count; i++) {
856fa198 4836 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4837 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4838
4839 memcpy(dst, src, PAGE_SIZE);
4840 kunmap_atomic(src, KM_USER0);
4841 }
4842
d78b47b9
CW		 4843 i915_gem_object_put_pages(obj);
4844
71acb5eb
DA		 4845 return 0;
4846out:
4847 return ret;
4848}
4849
4850static int
4851i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4852 struct drm_i915_gem_pwrite *args,
4853 struct drm_file *file_priv)
4854{
4855 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4856 void *obj_addr;
4857 int ret;
4858 char __user *user_data;
4859
4860 user_data = (char __user *) (uintptr_t) args->data_ptr;
4861 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4862
44d98a61 4863 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4864 ret = copy_from_user(obj_addr, user_data, args->size);
4865 if (ret)
4866 return -EFAULT;
4867
4868 drm_agp_chipset_flush(dev);
4869 return 0;
4870}
b962442e
EA		 4871
4872void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4873{
4874 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4875
4876 /* Clean up our request list when the client is going away, so that
4877 * later retire_requests won't dereference our soon-to-be-gone
4878 * file_priv.
4879 */
4880 mutex_lock(&dev->struct_mutex);
4881 while (!list_empty(&i915_file_priv->mm.request_list))
4882 list_del_init(i915_file_priv->mm.request_list.next);
4883 mutex_unlock(&dev->struct_mutex);
4884}
31169714 4885
31169714
CW		 4886static int
4887i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
4888{
4889 drm_i915_private_t *dev_priv, *next_dev;
4890 struct drm_i915_gem_object *obj_priv, *next_obj;
4891 int cnt = 0;
4892 int would_deadlock = 1;
4893
4894 /* "fast-path" to count number of available objects */
4895 if (nr_to_scan == 0) {
4896 spin_lock(&shrink_list_lock);
4897 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4898 struct drm_device *dev = dev_priv->dev;
4899
4900 if (mutex_trylock(&dev->struct_mutex)) {
4901 list_for_each_entry(obj_priv,
4902 &dev_priv->mm.inactive_list,
4903 list)
4904 cnt++;
4905 mutex_unlock(&dev->struct_mutex);
4906 }
4907 }
4908 spin_unlock(&shrink_list_lock);
4909
4910 return (cnt / 100) * sysctl_vfs_cache_pressure;
4911 }
4912
4913 spin_lock(&shrink_list_lock);
4914
4915 /* first scan for clean buffers */
4916 list_for_each_entry_safe(dev_priv, next_dev,
4917 &shrink_list, mm.shrink_list) {
4918 struct drm_device *dev = dev_priv->dev;
4919
4920 if (! mutex_trylock(&dev->struct_mutex))
4921 continue;
4922
4923 spin_unlock(&shrink_list_lock);
4924
4925 i915_gem_retire_requests(dev);
4926
4927 list_for_each_entry_safe(obj_priv, next_obj,
4928 &dev_priv->mm.inactive_list,
4929 list) {
4930 if (i915_gem_object_is_purgeable(obj_priv)) {
963b4836 4931 i915_gem_object_unbind(obj_priv->obj);
31169714
CW		 4932 if (--nr_to_scan <= 0)
4933 break;
4934 }
4935 }
4936
4937 spin_lock(&shrink_list_lock);
4938 mutex_unlock(&dev->struct_mutex);
4939
963b4836
CW		 4940 would_deadlock = 0;
4941
31169714
CW		 4942 if (nr_to_scan <= 0)
4943 break;
4944 }
4945
4946 /* second pass, evict/count anything still on the inactive list */
4947 list_for_each_entry_safe(dev_priv, next_dev,
4948 &shrink_list, mm.shrink_list) {
4949 struct drm_device *dev = dev_priv->dev;
4950
4951 if (! mutex_trylock(&dev->struct_mutex))
4952 continue;
4953
4954 spin_unlock(&shrink_list_lock);
4955
4956 list_for_each_entry_safe(obj_priv, next_obj,
4957 &dev_priv->mm.inactive_list,
4958 list) {
4959 if (nr_to_scan > 0) {
963b4836 4960 i915_gem_object_unbind(obj_priv->obj);
31169714
CW		 4961 nr_to_scan--;
4962 } else
4963 cnt++;
4964 }
4965
4966 spin_lock(&shrink_list_lock);
4967 mutex_unlock(&dev->struct_mutex);
4968
4969 would_deadlock = 0;
4970 }
4971
4972 spin_unlock(&shrink_list_lock);
4973
4974 if (would_deadlock)
4975 return -1;
4976 else if (cnt > 0)
4977 return (cnt / 100) * sysctl_vfs_cache_pressure;
4978 else
4979 return 0;
4980}
4981
4982static struct shrinker shrinker = {
4983 .shrink = i915_gem_shrink,
4984 .seeks = DEFAULT_SEEKS,
4985};
4986
4987__init void
4988i915_gem_shrinker_init(void)
4989{
4990 register_shrinker(&shrinker);
4991}
4992
4993__exit void
4994i915_gem_shrinker_exit(void)
4995{
4996 unregister_shrinker(&shrinker);
4997}
Clone of davem's net-next-2.6 tree