]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/gpu/drm/i915/i915_gem.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm/i915: use I915_GEM_GPU_DOMAINS
[net-next-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32#include <linux/swap.h>
79e53945 33#include <linux/pci.h>
673a394b 34
28dfe52a
EA		 35#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
36
e47c68e9
EA		 37static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
38static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
39static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 40static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
41 int write);
42static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
43 uint64_t offset,
44 uint64_t size);
45static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
673a394b 46static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
de151cf6
JB		 47static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
48 unsigned alignment);
0f973f27 49static int i915_gem_object_get_fence_reg(struct drm_gem_object *obj, bool write);
de151cf6
JB		 50static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
51static int i915_gem_evict_something(struct drm_device *dev);
71acb5eb
DA		 52static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
53 struct drm_i915_gem_pwrite *args,
54 struct drm_file *file_priv);
673a394b 55
79e53945
JB		 56int i915_gem_do_init(struct drm_device *dev, unsigned long start,
57 unsigned long end)
673a394b
EA		 58{
59 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 60
79e53945
JB		 61 if (start >= end ||
62 (start & (PAGE_SIZE - 1)) != 0 ||
63 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 64 return -EINVAL;
65 }
66
79e53945
JB		 67 drm_mm_init(&dev_priv->mm.gtt_space, start,
68 end - start);
673a394b 69
79e53945
JB		 70 dev->gtt_total = (uint32_t) (end - start);
71
72 return 0;
73}
673a394b 74
79e53945
JB		 75int
76i915_gem_init_ioctl(struct drm_device *dev, void *data,
77 struct drm_file *file_priv)
78{
79 struct drm_i915_gem_init *args = data;
80 int ret;
81
82 mutex_lock(&dev->struct_mutex);
83 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 84 mutex_unlock(&dev->struct_mutex);
85
79e53945 86 return ret;
673a394b
EA		 87}
88
5a125c3c
EA		 89int
90i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
91 struct drm_file *file_priv)
92{
5a125c3c 93 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 94
95 if (!(dev->driver->driver_features & DRIVER_GEM))
96 return -ENODEV;
97
98 args->aper_size = dev->gtt_total;
2678d9d6
KP		 99 args->aper_available_size = (args->aper_size -
100 atomic_read(&dev->pin_memory));
5a125c3c
EA		 101
102 return 0;
103}
104
673a394b
EA		 105
106/**
107 * Creates a new mm object and returns a handle to it.
108 */
109int
110i915_gem_create_ioctl(struct drm_device *dev, void *data,
111 struct drm_file *file_priv)
112{
113 struct drm_i915_gem_create *args = data;
114 struct drm_gem_object *obj;
115 int handle, ret;
116
117 args->size = roundup(args->size, PAGE_SIZE);
118
119 /* Allocate the new object */
120 obj = drm_gem_object_alloc(dev, args->size);
121 if (obj == NULL)
122 return -ENOMEM;
123
124 ret = drm_gem_handle_create(file_priv, obj, &handle);
125 mutex_lock(&dev->struct_mutex);
126 drm_gem_object_handle_unreference(obj);
127 mutex_unlock(&dev->struct_mutex);
128
129 if (ret)
130 return ret;
131
132 args->handle = handle;
133
134 return 0;
135}
136
eb01459f
EA		 137static inline int
138fast_shmem_read(struct page **pages,
139 loff_t page_base, int page_offset,
140 char __user *data,
141 int length)
142{
143 char __iomem *vaddr;
2bc43b5c 144 int unwritten;
eb01459f
EA		 145
146 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
147 if (vaddr == NULL)
148 return -ENOMEM;
2bc43b5c 149 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
eb01459f
EA		 150 kunmap_atomic(vaddr, KM_USER0);
151
2bc43b5c
FM		 152 if (unwritten)
153 return -EFAULT;
154
155 return 0;
eb01459f
EA		 156}
157
280b713b
EA		 158static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
159{
160 drm_i915_private_t *dev_priv = obj->dev->dev_private;
161 struct drm_i915_gem_object *obj_priv = obj->driver_private;
162
163 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
164 obj_priv->tiling_mode != I915_TILING_NONE;
165}
166
40123c1f
EA		 167static inline int
168slow_shmem_copy(struct page *dst_page,
169 int dst_offset,
170 struct page *src_page,
171 int src_offset,
172 int length)
173{
174 char *dst_vaddr, *src_vaddr;
175
176 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
177 if (dst_vaddr == NULL)
178 return -ENOMEM;
179
180 src_vaddr = kmap_atomic(src_page, KM_USER1);
181 if (src_vaddr == NULL) {
182 kunmap_atomic(dst_vaddr, KM_USER0);
183 return -ENOMEM;
184 }
185
186 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
187
188 kunmap_atomic(src_vaddr, KM_USER1);
189 kunmap_atomic(dst_vaddr, KM_USER0);
190
191 return 0;
192}
193
280b713b
EA		 194static inline int
195slow_shmem_bit17_copy(struct page *gpu_page,
196 int gpu_offset,
197 struct page *cpu_page,
198 int cpu_offset,
199 int length,
200 int is_read)
201{
202 char *gpu_vaddr, *cpu_vaddr;
203
204 /* Use the unswizzled path if this page isn't affected. */
205 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
206 if (is_read)
207 return slow_shmem_copy(cpu_page, cpu_offset,
208 gpu_page, gpu_offset, length);
209 else
210 return slow_shmem_copy(gpu_page, gpu_offset,
211 cpu_page, cpu_offset, length);
212 }
213
214 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
215 if (gpu_vaddr == NULL)
216 return -ENOMEM;
217
218 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
219 if (cpu_vaddr == NULL) {
220 kunmap_atomic(gpu_vaddr, KM_USER0);
221 return -ENOMEM;
222 }
223
224 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
225 * XORing with the other bits (A9 for Y, A9 and A10 for X)
226 */
227 while (length > 0) {
228 int cacheline_end = ALIGN(gpu_offset + 1, 64);
229 int this_length = min(cacheline_end - gpu_offset, length);
230 int swizzled_gpu_offset = gpu_offset ^ 64;
231
232 if (is_read) {
233 memcpy(cpu_vaddr + cpu_offset,
234 gpu_vaddr + swizzled_gpu_offset,
235 this_length);
236 } else {
237 memcpy(gpu_vaddr + swizzled_gpu_offset,
238 cpu_vaddr + cpu_offset,
239 this_length);
240 }
241 cpu_offset += this_length;
242 gpu_offset += this_length;
243 length -= this_length;
244 }
245
246 kunmap_atomic(cpu_vaddr, KM_USER1);
247 kunmap_atomic(gpu_vaddr, KM_USER0);
248
249 return 0;
250}
251
eb01459f
EA		 252/**
253 * This is the fast shmem pread path, which attempts to copy_from_user directly
254 * from the backing pages of the object to the user's address space. On a
255 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
256 */
257static int
258i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
259 struct drm_i915_gem_pread *args,
260 struct drm_file *file_priv)
261{
262 struct drm_i915_gem_object *obj_priv = obj->driver_private;
263 ssize_t remain;
264 loff_t offset, page_base;
265 char __user *user_data;
266 int page_offset, page_length;
267 int ret;
268
269 user_data = (char __user *) (uintptr_t) args->data_ptr;
270 remain = args->size;
271
272 mutex_lock(&dev->struct_mutex);
273
274 ret = i915_gem_object_get_pages(obj);
275 if (ret != 0)
276 goto fail_unlock;
277
278 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
279 args->size);
280 if (ret != 0)
281 goto fail_put_pages;
282
283 obj_priv = obj->driver_private;
284 offset = args->offset;
285
286 while (remain > 0) {
287 /* Operation in this page
288 *
289 * page_base = page offset within aperture
290 * page_offset = offset within page
291 * page_length = bytes to copy for this page
292 */
293 page_base = (offset & ~(PAGE_SIZE-1));
294 page_offset = offset & (PAGE_SIZE-1);
295 page_length = remain;
296 if ((page_offset + remain) > PAGE_SIZE)
297 page_length = PAGE_SIZE - page_offset;
298
299 ret = fast_shmem_read(obj_priv->pages,
300 page_base, page_offset,
301 user_data, page_length);
302 if (ret)
303 goto fail_put_pages;
304
305 remain -= page_length;
306 user_data += page_length;
307 offset += page_length;
308 }
309
310fail_put_pages:
311 i915_gem_object_put_pages(obj);
312fail_unlock:
313 mutex_unlock(&dev->struct_mutex);
314
315 return ret;
316}
317
318/**
319 * This is the fallback shmem pread path, which allocates temporary storage
320 * in kernel space to copy_to_user into outside of the struct_mutex, so we
321 * can copy out of the object's backing pages while holding the struct mutex
322 * and not take page faults.
323 */
324static int
325i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
326 struct drm_i915_gem_pread *args,
327 struct drm_file *file_priv)
328{
329 struct drm_i915_gem_object *obj_priv = obj->driver_private;
330 struct mm_struct *mm = current->mm;
331 struct page **user_pages;
332 ssize_t remain;
333 loff_t offset, pinned_pages, i;
334 loff_t first_data_page, last_data_page, num_pages;
335 int shmem_page_index, shmem_page_offset;
336 int data_page_index, data_page_offset;
337 int page_length;
338 int ret;
339 uint64_t data_ptr = args->data_ptr;
280b713b 340 int do_bit17_swizzling;
eb01459f
EA		 341
342 remain = args->size;
343
344 /* Pin the user pages containing the data. We can't fault while
345 * holding the struct mutex, yet we want to hold it while
346 * dereferencing the user data.
347 */
348 first_data_page = data_ptr / PAGE_SIZE;
349 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
350 num_pages = last_data_page - first_data_page + 1;
351
8e7d2b2c 352 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
eb01459f
EA		 353 if (user_pages == NULL)
354 return -ENOMEM;
355
356 down_read(&mm->mmap_sem);
357 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
e5e9ecde 358 num_pages, 1, 0, user_pages, NULL);
eb01459f
EA		 359 up_read(&mm->mmap_sem);
360 if (pinned_pages < num_pages) {
361 ret = -EFAULT;
362 goto fail_put_user_pages;
363 }
364
280b713b
EA		 365 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
366
eb01459f
EA		 367 mutex_lock(&dev->struct_mutex);
368
369 ret = i915_gem_object_get_pages(obj);
370 if (ret != 0)
371 goto fail_unlock;
372
373 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
374 args->size);
375 if (ret != 0)
376 goto fail_put_pages;
377
378 obj_priv = obj->driver_private;
379 offset = args->offset;
380
381 while (remain > 0) {
382 /* Operation in this page
383 *
384 * shmem_page_index = page number within shmem file
385 * shmem_page_offset = offset within page in shmem file
386 * data_page_index = page number in get_user_pages return
387 * data_page_offset = offset with data_page_index page.
388 * page_length = bytes to copy for this page
389 */
390 shmem_page_index = offset / PAGE_SIZE;
391 shmem_page_offset = offset & ~PAGE_MASK;
392 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
393 data_page_offset = data_ptr & ~PAGE_MASK;
394
395 page_length = remain;
396 if ((shmem_page_offset + page_length) > PAGE_SIZE)
397 page_length = PAGE_SIZE - shmem_page_offset;
398 if ((data_page_offset + page_length) > PAGE_SIZE)
399 page_length = PAGE_SIZE - data_page_offset;
400
280b713b
EA		 401 if (do_bit17_swizzling) {
402 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
403 shmem_page_offset,
404 user_pages[data_page_index],
405 data_page_offset,
406 page_length,
407 1);
408 } else {
409 ret = slow_shmem_copy(user_pages[data_page_index],
410 data_page_offset,
411 obj_priv->pages[shmem_page_index],
412 shmem_page_offset,
413 page_length);
414 }
eb01459f
EA		 415 if (ret)
416 goto fail_put_pages;
417
418 remain -= page_length;
419 data_ptr += page_length;
420 offset += page_length;
421 }
422
423fail_put_pages:
424 i915_gem_object_put_pages(obj);
425fail_unlock:
426 mutex_unlock(&dev->struct_mutex);
427fail_put_user_pages:
428 for (i = 0; i < pinned_pages; i++) {
429 SetPageDirty(user_pages[i]);
430 page_cache_release(user_pages[i]);
431 }
8e7d2b2c 432 drm_free_large(user_pages);
eb01459f
EA		 433
434 return ret;
435}
436
673a394b
EA		 437/**
438 * Reads data from the object referenced by handle.
439 *
440 * On error, the contents of *data are undefined.
441 */
442int
443i915_gem_pread_ioctl(struct drm_device *dev, void *data,
444 struct drm_file *file_priv)
445{
446 struct drm_i915_gem_pread *args = data;
447 struct drm_gem_object *obj;
448 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 449 int ret;
450
451 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
452 if (obj == NULL)
453 return -EBADF;
454 obj_priv = obj->driver_private;
455
456 /* Bounds check source.
457 *
458 * XXX: This could use review for overflow issues...
459 */
460 if (args->offset > obj->size || args->size > obj->size ||
461 args->offset + args->size > obj->size) {
462 drm_gem_object_unreference(obj);
463 return -EINVAL;
464 }
465
280b713b 466 if (i915_gem_object_needs_bit17_swizzle(obj)) {
eb01459f 467 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
280b713b
EA		 468 } else {
469 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
470 if (ret != 0)
471 ret = i915_gem_shmem_pread_slow(dev, obj, args,
472 file_priv);
473 }
673a394b
EA		 474
475 drm_gem_object_unreference(obj);
673a394b 476
eb01459f 477 return ret;
673a394b
EA		 478}
479
0839ccb8
KP		 480/* This is the fast write path which cannot handle
481 * page faults in the source data
9b7530cc 482 */
0839ccb8
KP		 483
484static inline int
485fast_user_write(struct io_mapping *mapping,
486 loff_t page_base, int page_offset,
487 char __user *user_data,
488 int length)
9b7530cc 489{
9b7530cc 490 char *vaddr_atomic;
0839ccb8 491 unsigned long unwritten;
9b7530cc 492
0839ccb8
KP		 493 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
494 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
495 user_data, length);
496 io_mapping_unmap_atomic(vaddr_atomic);
497 if (unwritten)
498 return -EFAULT;
499 return 0;
500}
501
502/* Here's the write path which can sleep for
503 * page faults
504 */
505
506static inline int
3de09aa3
EA		 507slow_kernel_write(struct io_mapping *mapping,
508 loff_t gtt_base, int gtt_offset,
509 struct page *user_page, int user_offset,
510 int length)
0839ccb8 511{
3de09aa3 512 char *src_vaddr, *dst_vaddr;
0839ccb8
KP		 513 unsigned long unwritten;
514
3de09aa3
EA		 515 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
516 src_vaddr = kmap_atomic(user_page, KM_USER1);
517 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
518 src_vaddr + user_offset,
519 length);
520 kunmap_atomic(src_vaddr, KM_USER1);
521 io_mapping_unmap_atomic(dst_vaddr);
0839ccb8
KP		 522 if (unwritten)
523 return -EFAULT;
9b7530cc 524 return 0;
9b7530cc
LT		 525}
526
40123c1f
EA		 527static inline int
528fast_shmem_write(struct page **pages,
529 loff_t page_base, int page_offset,
530 char __user *data,
531 int length)
532{
533 char __iomem *vaddr;
d0088775 534 unsigned long unwritten;
40123c1f
EA		 535
536 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
537 if (vaddr == NULL)
538 return -ENOMEM;
d0088775 539 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
40123c1f
EA		 540 kunmap_atomic(vaddr, KM_USER0);
541
d0088775
DA		 542 if (unwritten)
543 return -EFAULT;
40123c1f
EA		 544 return 0;
545}
546
3de09aa3
EA		 547/**
548 * This is the fast pwrite path, where we copy the data directly from the
549 * user into the GTT, uncached.
550 */
673a394b 551static int
3de09aa3
EA		 552i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
553 struct drm_i915_gem_pwrite *args,
554 struct drm_file *file_priv)
673a394b
EA		 555{
556 struct drm_i915_gem_object *obj_priv = obj->driver_private;
0839ccb8 557 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 558 ssize_t remain;
0839ccb8 559 loff_t offset, page_base;
673a394b 560 char __user *user_data;
0839ccb8
KP		 561 int page_offset, page_length;
562 int ret;
673a394b
EA		 563
564 user_data = (char __user *) (uintptr_t) args->data_ptr;
565 remain = args->size;
566 if (!access_ok(VERIFY_READ, user_data, remain))
567 return -EFAULT;
568
569
570 mutex_lock(&dev->struct_mutex);
571 ret = i915_gem_object_pin(obj, 0);
572 if (ret) {
573 mutex_unlock(&dev->struct_mutex);
574 return ret;
575 }
2ef7eeaa 576 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 577 if (ret)
578 goto fail;
579
580 obj_priv = obj->driver_private;
581 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 582
583 while (remain > 0) {
584 /* Operation in this page
585 *
0839ccb8
KP		 586 * page_base = page offset within aperture
587 * page_offset = offset within page
588 * page_length = bytes to copy for this page
673a394b 589 */
0839ccb8
KP		 590 page_base = (offset & ~(PAGE_SIZE-1));
591 page_offset = offset & (PAGE_SIZE-1);
592 page_length = remain;
593 if ((page_offset + remain) > PAGE_SIZE)
594 page_length = PAGE_SIZE - page_offset;
595
596 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
597 page_offset, user_data, page_length);
598
599 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 600 * source page isn't available. Return the error and we'll
601 * retry in the slow path.
0839ccb8 602 */
3de09aa3
EA		 603 if (ret)
604 goto fail;
673a394b 605
0839ccb8
KP		 606 remain -= page_length;
607 user_data += page_length;
608 offset += page_length;
673a394b 609 }
673a394b
EA		 610
611fail:
612 i915_gem_object_unpin(obj);
613 mutex_unlock(&dev->struct_mutex);
614
615 return ret;
616}
617
3de09aa3
EA		 618/**
619 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
620 * the memory and maps it using kmap_atomic for copying.
621 *
622 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
623 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
624 */
3043c60c 625static int
3de09aa3
EA		 626i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
627 struct drm_i915_gem_pwrite *args,
628 struct drm_file *file_priv)
673a394b 629{
3de09aa3
EA		 630 struct drm_i915_gem_object *obj_priv = obj->driver_private;
631 drm_i915_private_t *dev_priv = dev->dev_private;
632 ssize_t remain;
633 loff_t gtt_page_base, offset;
634 loff_t first_data_page, last_data_page, num_pages;
635 loff_t pinned_pages, i;
636 struct page **user_pages;
637 struct mm_struct *mm = current->mm;
638 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 639 int ret;
3de09aa3
EA		 640 uint64_t data_ptr = args->data_ptr;
641
642 remain = args->size;
643
644 /* Pin the user pages containing the data. We can't fault while
645 * holding the struct mutex, and all of the pwrite implementations
646 * want to hold it while dereferencing the user data.
647 */
648 first_data_page = data_ptr / PAGE_SIZE;
649 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
650 num_pages = last_data_page - first_data_page + 1;
651
8e7d2b2c 652 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
3de09aa3
EA		 653 if (user_pages == NULL)
654 return -ENOMEM;
655
656 down_read(&mm->mmap_sem);
657 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
658 num_pages, 0, 0, user_pages, NULL);
659 up_read(&mm->mmap_sem);
660 if (pinned_pages < num_pages) {
661 ret = -EFAULT;
662 goto out_unpin_pages;
663 }
673a394b
EA		 664
665 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 666 ret = i915_gem_object_pin(obj, 0);
667 if (ret)
668 goto out_unlock;
669
670 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
671 if (ret)
672 goto out_unpin_object;
673
674 obj_priv = obj->driver_private;
675 offset = obj_priv->gtt_offset + args->offset;
676
677 while (remain > 0) {
678 /* Operation in this page
679 *
680 * gtt_page_base = page offset within aperture
681 * gtt_page_offset = offset within page in aperture
682 * data_page_index = page number in get_user_pages return
683 * data_page_offset = offset with data_page_index page.
684 * page_length = bytes to copy for this page
685 */
686 gtt_page_base = offset & PAGE_MASK;
687 gtt_page_offset = offset & ~PAGE_MASK;
688 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
689 data_page_offset = data_ptr & ~PAGE_MASK;
690
691 page_length = remain;
692 if ((gtt_page_offset + page_length) > PAGE_SIZE)
693 page_length = PAGE_SIZE - gtt_page_offset;
694 if ((data_page_offset + page_length) > PAGE_SIZE)
695 page_length = PAGE_SIZE - data_page_offset;
696
697 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
698 gtt_page_base, gtt_page_offset,
699 user_pages[data_page_index],
700 data_page_offset,
701 page_length);
702
703 /* If we get a fault while copying data, then (presumably) our
704 * source page isn't available. Return the error and we'll
705 * retry in the slow path.
706 */
707 if (ret)
708 goto out_unpin_object;
709
710 remain -= page_length;
711 offset += page_length;
712 data_ptr += page_length;
713 }
714
715out_unpin_object:
716 i915_gem_object_unpin(obj);
717out_unlock:
718 mutex_unlock(&dev->struct_mutex);
719out_unpin_pages:
720 for (i = 0; i < pinned_pages; i++)
721 page_cache_release(user_pages[i]);
8e7d2b2c 722 drm_free_large(user_pages);
3de09aa3
EA		 723
724 return ret;
725}
726
40123c1f
EA		 727/**
728 * This is the fast shmem pwrite path, which attempts to directly
729 * copy_from_user into the kmapped pages backing the object.
730 */
3043c60c 731static int
40123c1f
EA		 732i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
733 struct drm_i915_gem_pwrite *args,
734 struct drm_file *file_priv)
673a394b 735{
40123c1f
EA		 736 struct drm_i915_gem_object *obj_priv = obj->driver_private;
737 ssize_t remain;
738 loff_t offset, page_base;
739 char __user *user_data;
740 int page_offset, page_length;
673a394b 741 int ret;
40123c1f
EA		 742
743 user_data = (char __user *) (uintptr_t) args->data_ptr;
744 remain = args->size;
673a394b
EA		 745
746 mutex_lock(&dev->struct_mutex);
747
40123c1f
EA		 748 ret = i915_gem_object_get_pages(obj);
749 if (ret != 0)
750 goto fail_unlock;
673a394b 751
e47c68e9 752 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 753 if (ret != 0)
754 goto fail_put_pages;
755
756 obj_priv = obj->driver_private;
757 offset = args->offset;
758 obj_priv->dirty = 1;
759
760 while (remain > 0) {
761 /* Operation in this page
762 *
763 * page_base = page offset within aperture
764 * page_offset = offset within page
765 * page_length = bytes to copy for this page
766 */
767 page_base = (offset & ~(PAGE_SIZE-1));
768 page_offset = offset & (PAGE_SIZE-1);
769 page_length = remain;
770 if ((page_offset + remain) > PAGE_SIZE)
771 page_length = PAGE_SIZE - page_offset;
772
773 ret = fast_shmem_write(obj_priv->pages,
774 page_base, page_offset,
775 user_data, page_length);
776 if (ret)
777 goto fail_put_pages;
778
779 remain -= page_length;
780 user_data += page_length;
781 offset += page_length;
782 }
783
784fail_put_pages:
785 i915_gem_object_put_pages(obj);
786fail_unlock:
787 mutex_unlock(&dev->struct_mutex);
788
789 return ret;
790}
791
792/**
793 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
794 * the memory and maps it using kmap_atomic for copying.
795 *
796 * This avoids taking mmap_sem for faulting on the user's address while the
797 * struct_mutex is held.
798 */
799static int
800i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
801 struct drm_i915_gem_pwrite *args,
802 struct drm_file *file_priv)
803{
804 struct drm_i915_gem_object *obj_priv = obj->driver_private;
805 struct mm_struct *mm = current->mm;
806 struct page **user_pages;
807 ssize_t remain;
808 loff_t offset, pinned_pages, i;
809 loff_t first_data_page, last_data_page, num_pages;
810 int shmem_page_index, shmem_page_offset;
811 int data_page_index, data_page_offset;
812 int page_length;
813 int ret;
814 uint64_t data_ptr = args->data_ptr;
280b713b 815 int do_bit17_swizzling;
40123c1f
EA		 816
817 remain = args->size;
818
819 /* Pin the user pages containing the data. We can't fault while
820 * holding the struct mutex, and all of the pwrite implementations
821 * want to hold it while dereferencing the user data.
822 */
823 first_data_page = data_ptr / PAGE_SIZE;
824 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
825 num_pages = last_data_page - first_data_page + 1;
826
8e7d2b2c 827 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
40123c1f
EA		 828 if (user_pages == NULL)
829 return -ENOMEM;
830
831 down_read(&mm->mmap_sem);
832 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
833 num_pages, 0, 0, user_pages, NULL);
834 up_read(&mm->mmap_sem);
835 if (pinned_pages < num_pages) {
836 ret = -EFAULT;
837 goto fail_put_user_pages;
673a394b
EA		 838 }
839
280b713b
EA		 840 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
841
40123c1f
EA		 842 mutex_lock(&dev->struct_mutex);
843
844 ret = i915_gem_object_get_pages(obj);
845 if (ret != 0)
846 goto fail_unlock;
847
848 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
849 if (ret != 0)
850 goto fail_put_pages;
851
852 obj_priv = obj->driver_private;
673a394b 853 offset = args->offset;
40123c1f 854 obj_priv->dirty = 1;
673a394b 855
40123c1f
EA		 856 while (remain > 0) {
857 /* Operation in this page
858 *
859 * shmem_page_index = page number within shmem file
860 * shmem_page_offset = offset within page in shmem file
861 * data_page_index = page number in get_user_pages return
862 * data_page_offset = offset with data_page_index page.
863 * page_length = bytes to copy for this page
864 */
865 shmem_page_index = offset / PAGE_SIZE;
866 shmem_page_offset = offset & ~PAGE_MASK;
867 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
868 data_page_offset = data_ptr & ~PAGE_MASK;
869
870 page_length = remain;
871 if ((shmem_page_offset + page_length) > PAGE_SIZE)
872 page_length = PAGE_SIZE - shmem_page_offset;
873 if ((data_page_offset + page_length) > PAGE_SIZE)
874 page_length = PAGE_SIZE - data_page_offset;
875
280b713b
EA		 876 if (do_bit17_swizzling) {
877 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
878 shmem_page_offset,
879 user_pages[data_page_index],
880 data_page_offset,
881 page_length,
882 0);
883 } else {
884 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
885 shmem_page_offset,
886 user_pages[data_page_index],
887 data_page_offset,
888 page_length);
889 }
40123c1f
EA		 890 if (ret)
891 goto fail_put_pages;
892
893 remain -= page_length;
894 data_ptr += page_length;
895 offset += page_length;
673a394b
EA		 896 }
897
40123c1f
EA		 898fail_put_pages:
899 i915_gem_object_put_pages(obj);
900fail_unlock:
673a394b 901 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 902fail_put_user_pages:
903 for (i = 0; i < pinned_pages; i++)
904 page_cache_release(user_pages[i]);
8e7d2b2c 905 drm_free_large(user_pages);
673a394b 906
40123c1f 907 return ret;
673a394b
EA		 908}
909
910/**
911 * Writes data to the object referenced by handle.
912 *
913 * On error, the contents of the buffer that were to be modified are undefined.
914 */
915int
916i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
917 struct drm_file *file_priv)
918{
919 struct drm_i915_gem_pwrite *args = data;
920 struct drm_gem_object *obj;
921 struct drm_i915_gem_object *obj_priv;
922 int ret = 0;
923
924 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
925 if (obj == NULL)
926 return -EBADF;
927 obj_priv = obj->driver_private;
928
929 /* Bounds check destination.
930 *
931 * XXX: This could use review for overflow issues...
932 */
933 if (args->offset > obj->size || args->size > obj->size ||
934 args->offset + args->size > obj->size) {
935 drm_gem_object_unreference(obj);
936 return -EINVAL;
937 }
938
939 /* We can only do the GTT pwrite on untiled buffers, as otherwise
940 * it would end up going through the fenced access, and we'll get
941 * different detiling behavior between reading and writing.
942 * pread/pwrite currently are reading and writing from the CPU
943 * perspective, requiring manual detiling by the client.
944 */
71acb5eb
DA		 945 if (obj_priv->phys_obj)
946 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
947 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
3de09aa3
EA		 948 dev->gtt_total != 0) {
949 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
950 if (ret == -EFAULT) {
951 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
952 file_priv);
953 }
280b713b
EA		 954 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
955 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
40123c1f
EA		 956 } else {
957 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
958 if (ret == -EFAULT) {
959 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
960 file_priv);
961 }
962 }
673a394b
EA		 963
964#if WATCH_PWRITE
965 if (ret)
966 DRM_INFO("pwrite failed %d\n", ret);
967#endif
968
969 drm_gem_object_unreference(obj);
970
971 return ret;
972}
973
974/**
2ef7eeaa
EA		 975 * Called when user space prepares to use an object with the CPU, either
976 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 977 */
978int
979i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
980 struct drm_file *file_priv)
981{
982 struct drm_i915_gem_set_domain *args = data;
983 struct drm_gem_object *obj;
2ef7eeaa
EA		 984 uint32_t read_domains = args->read_domains;
985 uint32_t write_domain = args->write_domain;
673a394b
EA		 986 int ret;
987
988 if (!(dev->driver->driver_features & DRIVER_GEM))
989 return -ENODEV;
990
2ef7eeaa 991 /* Only handle setting domains to types used by the CPU. */
21d509e3 992 if (write_domain & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 993 return -EINVAL;
994
21d509e3 995 if (read_domains & I915_GEM_GPU_DOMAINS)
2ef7eeaa
EA		 996 return -EINVAL;
997
998 /* Having something in the write domain implies it's in the read
999 * domain, and only that read domain. Enforce that in the request.
1000 */
1001 if (write_domain != 0 && read_domains != write_domain)
1002 return -EINVAL;
1003
673a394b
EA		 1004 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1005 if (obj == NULL)
1006 return -EBADF;
1007
1008 mutex_lock(&dev->struct_mutex);
1009#if WATCH_BUF
1010 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
2ef7eeaa 1011 obj, obj->size, read_domains, write_domain);
673a394b 1012#endif
2ef7eeaa
EA		 1013 if (read_domains & I915_GEM_DOMAIN_GTT) {
1014 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392
EA		 1015
1016 /* Silently promote "you're not bound, there was nothing to do"
1017 * to success, since the client was just asking us to
1018 * make sure everything was done.
1019 */
1020 if (ret == -EINVAL)
1021 ret = 0;
2ef7eeaa 1022 } else {
e47c68e9 1023 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 1024 }
1025
673a394b
EA		 1026 drm_gem_object_unreference(obj);
1027 mutex_unlock(&dev->struct_mutex);
1028 return ret;
1029}
1030
1031/**
1032 * Called when user space has done writes to this buffer
1033 */
1034int
1035i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1036 struct drm_file *file_priv)
1037{
1038 struct drm_i915_gem_sw_finish *args = data;
1039 struct drm_gem_object *obj;
1040 struct drm_i915_gem_object *obj_priv;
1041 int ret = 0;
1042
1043 if (!(dev->driver->driver_features & DRIVER_GEM))
1044 return -ENODEV;
1045
1046 mutex_lock(&dev->struct_mutex);
1047 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1048 if (obj == NULL) {
1049 mutex_unlock(&dev->struct_mutex);
1050 return -EBADF;
1051 }
1052
1053#if WATCH_BUF
1054 DRM_INFO("%s: sw_finish %d (%p %d)\n",
1055 __func__, args->handle, obj, obj->size);
1056#endif
1057 obj_priv = obj->driver_private;
1058
1059 /* Pinned buffers may be scanout, so flush the cache */
e47c68e9
EA		 1060 if (obj_priv->pin_count)
1061 i915_gem_object_flush_cpu_write_domain(obj);
1062
673a394b
EA		 1063 drm_gem_object_unreference(obj);
1064 mutex_unlock(&dev->struct_mutex);
1065 return ret;
1066}
1067
1068/**
1069 * Maps the contents of an object, returning the address it is mapped
1070 * into.
1071 *
1072 * While the mapping holds a reference on the contents of the object, it doesn't
1073 * imply a ref on the object itself.
1074 */
1075int
1076i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1077 struct drm_file *file_priv)
1078{
1079 struct drm_i915_gem_mmap *args = data;
1080 struct drm_gem_object *obj;
1081 loff_t offset;
1082 unsigned long addr;
1083
1084 if (!(dev->driver->driver_features & DRIVER_GEM))
1085 return -ENODEV;
1086
1087 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1088 if (obj == NULL)
1089 return -EBADF;
1090
1091 offset = args->offset;
1092
1093 down_write(&current->mm->mmap_sem);
1094 addr = do_mmap(obj->filp, 0, args->size,
1095 PROT_READ | PROT_WRITE, MAP_SHARED,
1096 args->offset);
1097 up_write(&current->mm->mmap_sem);
1098 mutex_lock(&dev->struct_mutex);
1099 drm_gem_object_unreference(obj);
1100 mutex_unlock(&dev->struct_mutex);
1101 if (IS_ERR((void *)addr))
1102 return addr;
1103
1104 args->addr_ptr = (uint64_t) addr;
1105
1106 return 0;
1107}
1108
de151cf6
JB		 1109/**
1110 * i915_gem_fault - fault a page into the GTT
1111 * vma: VMA in question
1112 * vmf: fault info
1113 *
1114 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1115 * from userspace. The fault handler takes care of binding the object to
1116 * the GTT (if needed), allocating and programming a fence register (again,
1117 * only if needed based on whether the old reg is still valid or the object
1118 * is tiled) and inserting a new PTE into the faulting process.
1119 *
1120 * Note that the faulting process may involve evicting existing objects
1121 * from the GTT and/or fence registers to make room. So performance may
1122 * suffer if the GTT working set is large or there are few fence registers
1123 * left.
1124 */
1125int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1126{
1127 struct drm_gem_object *obj = vma->vm_private_data;
1128 struct drm_device *dev = obj->dev;
1129 struct drm_i915_private *dev_priv = dev->dev_private;
1130 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1131 pgoff_t page_offset;
1132 unsigned long pfn;
1133 int ret = 0;
0f973f27 1134 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1135
1136 /* We don't use vmf->pgoff since that has the fake offset */
1137 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1138 PAGE_SHIFT;
1139
1140 /* Now bind it into the GTT if needed */
1141 mutex_lock(&dev->struct_mutex);
1142 if (!obj_priv->gtt_space) {
1143 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1144 if (ret) {
1145 mutex_unlock(&dev->struct_mutex);
1146 return VM_FAULT_SIGBUS;
1147 }
07f4f3e8
KH		 1148
1149 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1150 if (ret) {
1151 mutex_unlock(&dev->struct_mutex);
1152 return VM_FAULT_SIGBUS;
1153 }
1154
14b60391 1155 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
de151cf6
JB		 1156 }
1157
1158 /* Need a new fence register? */
1159 if (obj_priv->fence_reg == I915_FENCE_REG_NONE &&
d9ddcb96 1160 obj_priv->tiling_mode != I915_TILING_NONE) {
0f973f27 1161 ret = i915_gem_object_get_fence_reg(obj, write);
7d8d58b2
CW		 1162 if (ret) {
1163 mutex_unlock(&dev->struct_mutex);
d9ddcb96 1164 return VM_FAULT_SIGBUS;
7d8d58b2 1165 }
d9ddcb96 1166 }
de151cf6
JB		 1167
1168 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1169 page_offset;
1170
1171 /* Finally, remap it using the new GTT offset */
1172 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1173
1174 mutex_unlock(&dev->struct_mutex);
1175
1176 switch (ret) {
1177 case -ENOMEM:
1178 case -EAGAIN:
1179 return VM_FAULT_OOM;
1180 case -EFAULT:
959b887c 1181 case -EINVAL:
de151cf6
JB		 1182 return VM_FAULT_SIGBUS;
1183 default:
1184 return VM_FAULT_NOPAGE;
1185 }
1186}
1187
1188/**
1189 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1190 * @obj: obj in question
1191 *
1192 * GEM memory mapping works by handing back to userspace a fake mmap offset
1193 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1194 * up the object based on the offset and sets up the various memory mapping
1195 * structures.
1196 *
1197 * This routine allocates and attaches a fake offset for @obj.
1198 */
1199static int
1200i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1201{
1202 struct drm_device *dev = obj->dev;
1203 struct drm_gem_mm *mm = dev->mm_private;
1204 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1205 struct drm_map_list *list;
f77d390c 1206 struct drm_local_map *map;
de151cf6
JB		 1207 int ret = 0;
1208
1209 /* Set the object up for mmap'ing */
1210 list = &obj->map_list;
1211 list->map = drm_calloc(1, sizeof(struct drm_map_list),
1212 DRM_MEM_DRIVER);
1213 if (!list->map)
1214 return -ENOMEM;
1215
1216 map = list->map;
1217 map->type = _DRM_GEM;
1218 map->size = obj->size;
1219 map->handle = obj;
1220
1221 /* Get a DRM GEM mmap offset allocated... */
1222 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1223 obj->size / PAGE_SIZE, 0, 0);
1224 if (!list->file_offset_node) {
1225 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1226 ret = -ENOMEM;
1227 goto out_free_list;
1228 }
1229
1230 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1231 obj->size / PAGE_SIZE, 0);
1232 if (!list->file_offset_node) {
1233 ret = -ENOMEM;
1234 goto out_free_list;
1235 }
1236
1237 list->hash.key = list->file_offset_node->start;
1238 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1239 DRM_ERROR("failed to add to map hash\n");
1240 goto out_free_mm;
1241 }
1242
1243 /* By now we should be all set, any drm_mmap request on the offset
1244 * below will get to our mmap & fault handler */
1245 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1246
1247 return 0;
1248
1249out_free_mm:
1250 drm_mm_put_block(list->file_offset_node);
1251out_free_list:
1252 drm_free(list->map, sizeof(struct drm_map_list), DRM_MEM_DRIVER);
1253
1254 return ret;
1255}
1256
ab00b3e5
JB		 1257static void
1258i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1259{
1260 struct drm_device *dev = obj->dev;
1261 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1262 struct drm_gem_mm *mm = dev->mm_private;
1263 struct drm_map_list *list;
1264
1265 list = &obj->map_list;
1266 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1267
1268 if (list->file_offset_node) {
1269 drm_mm_put_block(list->file_offset_node);
1270 list->file_offset_node = NULL;
1271 }
1272
1273 if (list->map) {
1274 drm_free(list->map, sizeof(struct drm_map), DRM_MEM_DRIVER);
1275 list->map = NULL;
1276 }
1277
1278 obj_priv->mmap_offset = 0;
1279}
1280
de151cf6
JB		 1281/**
1282 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1283 * @obj: object to check
1284 *
1285 * Return the required GTT alignment for an object, taking into account
1286 * potential fence register mapping if needed.
1287 */
1288static uint32_t
1289i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1290{
1291 struct drm_device *dev = obj->dev;
1292 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1293 int start, i;
1294
1295 /*
1296 * Minimum alignment is 4k (GTT page size), but might be greater
1297 * if a fence register is needed for the object.
1298 */
1299 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1300 return 4096;
1301
1302 /*
1303 * Previous chips need to be aligned to the size of the smallest
1304 * fence register that can contain the object.
1305 */
1306 if (IS_I9XX(dev))
1307 start = 1024*1024;
1308 else
1309 start = 512*1024;
1310
1311 for (i = start; i < obj->size; i <<= 1)
1312 ;
1313
1314 return i;
1315}
1316
1317/**
1318 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1319 * @dev: DRM device
1320 * @data: GTT mapping ioctl data
1321 * @file_priv: GEM object info
1322 *
1323 * Simply returns the fake offset to userspace so it can mmap it.
1324 * The mmap call will end up in drm_gem_mmap(), which will set things
1325 * up so we can get faults in the handler above.
1326 *
1327 * The fault handler will take care of binding the object into the GTT
1328 * (since it may have been evicted to make room for something), allocating
1329 * a fence register, and mapping the appropriate aperture address into
1330 * userspace.
1331 */
1332int
1333i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1334 struct drm_file *file_priv)
1335{
1336 struct drm_i915_gem_mmap_gtt *args = data;
1337 struct drm_i915_private *dev_priv = dev->dev_private;
1338 struct drm_gem_object *obj;
1339 struct drm_i915_gem_object *obj_priv;
1340 int ret;
1341
1342 if (!(dev->driver->driver_features & DRIVER_GEM))
1343 return -ENODEV;
1344
1345 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1346 if (obj == NULL)
1347 return -EBADF;
1348
1349 mutex_lock(&dev->struct_mutex);
1350
1351 obj_priv = obj->driver_private;
1352
1353 if (!obj_priv->mmap_offset) {
1354 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1355 if (ret) {
1356 drm_gem_object_unreference(obj);
1357 mutex_unlock(&dev->struct_mutex);
de151cf6 1358 return ret;
13af1062 1359 }
de151cf6
JB		 1360 }
1361
1362 args->offset = obj_priv->mmap_offset;
1363
1364 obj_priv->gtt_alignment = i915_gem_get_gtt_alignment(obj);
1365
1366 /* Make sure the alignment is correct for fence regs etc */
1367 if (obj_priv->agp_mem &&
1368 (obj_priv->gtt_offset & (obj_priv->gtt_alignment - 1))) {
1369 drm_gem_object_unreference(obj);
1370 mutex_unlock(&dev->struct_mutex);
1371 return -EINVAL;
1372 }
1373
1374 /*
1375 * Pull it into the GTT so that we have a page list (makes the
1376 * initial fault faster and any subsequent flushing possible).
1377 */
1378 if (!obj_priv->agp_mem) {
1379 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1380 if (ret) {
1381 drm_gem_object_unreference(obj);
1382 mutex_unlock(&dev->struct_mutex);
1383 return ret;
1384 }
14b60391 1385 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
de151cf6
JB		 1386 }
1387
1388 drm_gem_object_unreference(obj);
1389 mutex_unlock(&dev->struct_mutex);
1390
1391 return 0;
1392}
1393
6911a9b8 1394void
856fa198 1395i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b
EA		 1396{
1397 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1398 int page_count = obj->size / PAGE_SIZE;
1399 int i;
1400
856fa198 1401 BUG_ON(obj_priv->pages_refcount == 0);
673a394b 1402
856fa198
EA		 1403 if (--obj_priv->pages_refcount != 0)
1404 return;
673a394b 1405
280b713b
EA		 1406 if (obj_priv->tiling_mode != I915_TILING_NONE)
1407 i915_gem_object_save_bit_17_swizzle(obj);
1408
673a394b 1409 for (i = 0; i < page_count; i++)
856fa198 1410 if (obj_priv->pages[i] != NULL) {
673a394b 1411 if (obj_priv->dirty)
856fa198
EA		 1412 set_page_dirty(obj_priv->pages[i]);
1413 mark_page_accessed(obj_priv->pages[i]);
1414 page_cache_release(obj_priv->pages[i]);
673a394b
EA		 1415 }
1416 obj_priv->dirty = 0;
1417
8e7d2b2c 1418 drm_free_large(obj_priv->pages);
856fa198 1419 obj_priv->pages = NULL;
673a394b
EA		 1420}
1421
1422static void
ce44b0ea 1423i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
673a394b
EA		 1424{
1425 struct drm_device *dev = obj->dev;
1426 drm_i915_private_t *dev_priv = dev->dev_private;
1427 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1428
1429 /* Add a reference if we're newly entering the active list. */
1430 if (!obj_priv->active) {
1431 drm_gem_object_reference(obj);
1432 obj_priv->active = 1;
1433 }
1434 /* Move from whatever list we were on to the tail of execution. */
5e118f41 1435 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1436 list_move_tail(&obj_priv->list,
1437 &dev_priv->mm.active_list);
5e118f41 1438 spin_unlock(&dev_priv->mm.active_list_lock);
ce44b0ea 1439 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1440}
1441
ce44b0ea
EA		 1442static void
1443i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1444{
1445 struct drm_device *dev = obj->dev;
1446 drm_i915_private_t *dev_priv = dev->dev_private;
1447 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1448
1449 BUG_ON(!obj_priv->active);
1450 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1451 obj_priv->last_rendering_seqno = 0;
1452}
673a394b
EA		 1453
1454static void
1455i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1456{
1457 struct drm_device *dev = obj->dev;
1458 drm_i915_private_t *dev_priv = dev->dev_private;
1459 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1460
1461 i915_verify_inactive(dev, __FILE__, __LINE__);
1462 if (obj_priv->pin_count != 0)
1463 list_del_init(&obj_priv->list);
1464 else
1465 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1466
ce44b0ea 1467 obj_priv->last_rendering_seqno = 0;
673a394b
EA		 1468 if (obj_priv->active) {
1469 obj_priv->active = 0;
1470 drm_gem_object_unreference(obj);
1471 }
1472 i915_verify_inactive(dev, __FILE__, __LINE__);
1473}
1474
1475/**
1476 * Creates a new sequence number, emitting a write of it to the status page
1477 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1478 *
1479 * Must be called with struct_lock held.
1480 *
1481 * Returned sequence numbers are nonzero on success.
1482 */
1483static uint32_t
b962442e
EA		 1484i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1485 uint32_t flush_domains)
673a394b
EA		 1486{
1487 drm_i915_private_t *dev_priv = dev->dev_private;
b962442e 1488 struct drm_i915_file_private *i915_file_priv = NULL;
673a394b
EA		 1489 struct drm_i915_gem_request *request;
1490 uint32_t seqno;
1491 int was_empty;
1492 RING_LOCALS;
1493
b962442e
EA		 1494 if (file_priv != NULL)
1495 i915_file_priv = file_priv->driver_priv;
1496
673a394b
EA		 1497 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
1498 if (request == NULL)
1499 return 0;
1500
1501 /* Grab the seqno we're going to make this request be, and bump the
1502 * next (skipping 0 so it can be the reserved no-seqno value).
1503 */
1504 seqno = dev_priv->mm.next_gem_seqno;
1505 dev_priv->mm.next_gem_seqno++;
1506 if (dev_priv->mm.next_gem_seqno == 0)
1507 dev_priv->mm.next_gem_seqno++;
1508
1509 BEGIN_LP_RING(4);
1510 OUT_RING(MI_STORE_DWORD_INDEX);
1511 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1512 OUT_RING(seqno);
1513
1514 OUT_RING(MI_USER_INTERRUPT);
1515 ADVANCE_LP_RING();
1516
1517 DRM_DEBUG("%d\n", seqno);
1518
1519 request->seqno = seqno;
1520 request->emitted_jiffies = jiffies;
673a394b
EA		 1521 was_empty = list_empty(&dev_priv->mm.request_list);
1522 list_add_tail(&request->list, &dev_priv->mm.request_list);
b962442e
EA		 1523 if (i915_file_priv) {
1524 list_add_tail(&request->client_list,
1525 &i915_file_priv->mm.request_list);
1526 } else {
1527 INIT_LIST_HEAD(&request->client_list);
1528 }
673a394b 1529
ce44b0ea
EA		 1530 /* Associate any objects on the flushing list matching the write
1531 * domain we're flushing with our flush.
1532 */
1533 if (flush_domains != 0) {
1534 struct drm_i915_gem_object *obj_priv, *next;
1535
1536 list_for_each_entry_safe(obj_priv, next,
1537 &dev_priv->mm.flushing_list, list) {
1538 struct drm_gem_object *obj = obj_priv->obj;
1539
1540 if ((obj->write_domain & flush_domains) ==
1541 obj->write_domain) {
1542 obj->write_domain = 0;
1543 i915_gem_object_move_to_active(obj, seqno);
1544 }
1545 }
1546
1547 }
1548
6dbe2772 1549 if (was_empty && !dev_priv->mm.suspended)
673a394b
EA		 1550 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
1551 return seqno;
1552}
1553
1554/**
1555 * Command execution barrier
1556 *
1557 * Ensures that all commands in the ring are finished
1558 * before signalling the CPU
1559 */
3043c60c 1560static uint32_t
673a394b
EA		 1561i915_retire_commands(struct drm_device *dev)
1562{
1563 drm_i915_private_t *dev_priv = dev->dev_private;
1564 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1565 uint32_t flush_domains = 0;
1566 RING_LOCALS;
1567
1568 /* The sampler always gets flushed on i965 (sigh) */
1569 if (IS_I965G(dev))
1570 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1571 BEGIN_LP_RING(2);
1572 OUT_RING(cmd);
1573 OUT_RING(0); /* noop */
1574 ADVANCE_LP_RING();
1575 return flush_domains;
1576}
1577
1578/**
1579 * Moves buffers associated only with the given active seqno from the active
1580 * to inactive list, potentially freeing them.
1581 */
1582static void
1583i915_gem_retire_request(struct drm_device *dev,
1584 struct drm_i915_gem_request *request)
1585{
1586 drm_i915_private_t *dev_priv = dev->dev_private;
1587
1588 /* Move any buffers on the active list that are no longer referenced
1589 * by the ringbuffer to the flushing/inactive lists as appropriate.
1590 */
5e118f41 1591 spin_lock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1592 while (!list_empty(&dev_priv->mm.active_list)) {
1593 struct drm_gem_object *obj;
1594 struct drm_i915_gem_object *obj_priv;
1595
1596 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1597 struct drm_i915_gem_object,
1598 list);
1599 obj = obj_priv->obj;
1600
1601 /* If the seqno being retired doesn't match the oldest in the
1602 * list, then the oldest in the list must still be newer than
1603 * this seqno.
1604 */
1605 if (obj_priv->last_rendering_seqno != request->seqno)
5e118f41 1606 goto out;
de151cf6 1607
673a394b
EA		 1608#if WATCH_LRU
1609 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1610 __func__, request->seqno, obj);
1611#endif
1612
ce44b0ea
EA		 1613 if (obj->write_domain != 0)
1614 i915_gem_object_move_to_flushing(obj);
68c84342
SL		 1615 else {
1616 /* Take a reference on the object so it won't be
1617 * freed while the spinlock is held. The list
1618 * protection for this spinlock is safe when breaking
1619 * the lock like this since the next thing we do
1620 * is just get the head of the list again.
1621 */
1622 drm_gem_object_reference(obj);
673a394b 1623 i915_gem_object_move_to_inactive(obj);
68c84342
SL		 1624 spin_unlock(&dev_priv->mm.active_list_lock);
1625 drm_gem_object_unreference(obj);
1626 spin_lock(&dev_priv->mm.active_list_lock);
1627 }
673a394b 1628 }
5e118f41
CW		 1629out:
1630 spin_unlock(&dev_priv->mm.active_list_lock);
673a394b
EA		 1631}
1632
1633/**
1634 * Returns true if seq1 is later than seq2.
1635 */
1636static int
1637i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1638{
1639 return (int32_t)(seq1 - seq2) >= 0;
1640}
1641
1642uint32_t
1643i915_get_gem_seqno(struct drm_device *dev)
1644{
1645 drm_i915_private_t *dev_priv = dev->dev_private;
1646
1647 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1648}
1649
1650/**
1651 * This function clears the request list as sequence numbers are passed.
1652 */
1653void
1654i915_gem_retire_requests(struct drm_device *dev)
1655{
1656 drm_i915_private_t *dev_priv = dev->dev_private;
1657 uint32_t seqno;
1658
6c0594a3
KW		 1659 if (!dev_priv->hw_status_page)
1660 return;
1661
673a394b
EA		 1662 seqno = i915_get_gem_seqno(dev);
1663
1664 while (!list_empty(&dev_priv->mm.request_list)) {
1665 struct drm_i915_gem_request *request;
1666 uint32_t retiring_seqno;
1667
1668 request = list_first_entry(&dev_priv->mm.request_list,
1669 struct drm_i915_gem_request,
1670 list);
1671 retiring_seqno = request->seqno;
1672
1673 if (i915_seqno_passed(seqno, retiring_seqno) ||
1674 dev_priv->mm.wedged) {
1675 i915_gem_retire_request(dev, request);
1676
1677 list_del(&request->list);
b962442e 1678 list_del(&request->client_list);
673a394b
EA		 1679 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
1680 } else
1681 break;
1682 }
1683}
1684
1685void
1686i915_gem_retire_work_handler(struct work_struct *work)
1687{
1688 drm_i915_private_t *dev_priv;
1689 struct drm_device *dev;
1690
1691 dev_priv = container_of(work, drm_i915_private_t,
1692 mm.retire_work.work);
1693 dev = dev_priv->dev;
1694
1695 mutex_lock(&dev->struct_mutex);
1696 i915_gem_retire_requests(dev);
6dbe2772
KP		 1697 if (!dev_priv->mm.suspended &&
1698 !list_empty(&dev_priv->mm.request_list))
673a394b
EA		 1699 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
1700 mutex_unlock(&dev->struct_mutex);
1701}
1702
1703/**
1704 * Waits for a sequence number to be signaled, and cleans up the
1705 * request and object lists appropriately for that event.
1706 */
3043c60c 1707static int
673a394b
EA		 1708i915_wait_request(struct drm_device *dev, uint32_t seqno)
1709{
1710 drm_i915_private_t *dev_priv = dev->dev_private;
802c7eb6 1711 u32 ier;
673a394b
EA		 1712 int ret = 0;
1713
1714 BUG_ON(seqno == 0);
1715
1716 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
036a4a7d
ZW		 1717 if (IS_IGDNG(dev))
1718 ier = I915_READ(DEIER) | I915_READ(GTIER);
1719 else
1720 ier = I915_READ(IER);
802c7eb6
JB		 1721 if (!ier) {
1722 DRM_ERROR("something (likely vbetool) disabled "
1723 "interrupts, re-enabling\n");
1724 i915_driver_irq_preinstall(dev);
1725 i915_driver_irq_postinstall(dev);
1726 }
1727
673a394b
EA		 1728 dev_priv->mm.waiting_gem_seqno = seqno;
1729 i915_user_irq_get(dev);
1730 ret = wait_event_interruptible(dev_priv->irq_queue,
1731 i915_seqno_passed(i915_get_gem_seqno(dev),
1732 seqno) ||
1733 dev_priv->mm.wedged);
1734 i915_user_irq_put(dev);
1735 dev_priv->mm.waiting_gem_seqno = 0;
1736 }
1737 if (dev_priv->mm.wedged)
1738 ret = -EIO;
1739
1740 if (ret && ret != -ERESTARTSYS)
1741 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1742 __func__, ret, seqno, i915_get_gem_seqno(dev));
1743
1744 /* Directly dispatch request retiring. While we have the work queue
1745 * to handle this, the waiter on a request often wants an associated
1746 * buffer to have made it to the inactive list, and we would need
1747 * a separate wait queue to handle that.
1748 */
1749 if (ret == 0)
1750 i915_gem_retire_requests(dev);
1751
1752 return ret;
1753}
1754
1755static void
1756i915_gem_flush(struct drm_device *dev,
1757 uint32_t invalidate_domains,
1758 uint32_t flush_domains)
1759{
1760 drm_i915_private_t *dev_priv = dev->dev_private;
1761 uint32_t cmd;
1762 RING_LOCALS;
1763
1764#if WATCH_EXEC
1765 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1766 invalidate_domains, flush_domains);
1767#endif
1768
1769 if (flush_domains & I915_GEM_DOMAIN_CPU)
1770 drm_agp_chipset_flush(dev);
1771
21d509e3 1772 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
673a394b
EA		 1773 /*
1774 * read/write caches:
1775 *
1776 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1777 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1778 * also flushed at 2d versus 3d pipeline switches.
1779 *
1780 * read-only caches:
1781 *
1782 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1783 * MI_READ_FLUSH is set, and is always flushed on 965.
1784 *
1785 * I915_GEM_DOMAIN_COMMAND may not exist?
1786 *
1787 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1788 * invalidated when MI_EXE_FLUSH is set.
1789 *
1790 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1791 * invalidated with every MI_FLUSH.
1792 *
1793 * TLBs:
1794 *
1795 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1796 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1797 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1798 * are flushed at any MI_FLUSH.
1799 */
1800
1801 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1802 if ((invalidate_domains|flush_domains) &
1803 I915_GEM_DOMAIN_RENDER)
1804 cmd &= ~MI_NO_WRITE_FLUSH;
1805 if (!IS_I965G(dev)) {
1806 /*
1807 * On the 965, the sampler cache always gets flushed
1808 * and this bit is reserved.
1809 */
1810 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1811 cmd |= MI_READ_FLUSH;
1812 }
1813 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1814 cmd |= MI_EXE_FLUSH;
1815
1816#if WATCH_EXEC
1817 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1818#endif
1819 BEGIN_LP_RING(2);
1820 OUT_RING(cmd);
1821 OUT_RING(0); /* noop */
1822 ADVANCE_LP_RING();
1823 }
1824}
1825
1826/**
1827 * Ensures that all rendering to the object has completed and the object is
1828 * safe to unbind from the GTT or access from the CPU.
1829 */
1830static int
1831i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1832{
1833 struct drm_device *dev = obj->dev;
1834 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1835 int ret;
1836
e47c68e9
EA		 1837 /* This function only exists to support waiting for existing rendering,
1838 * not for emitting required flushes.
673a394b 1839 */
e47c68e9 1840 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 1841
1842 /* If there is rendering queued on the buffer being evicted, wait for
1843 * it.
1844 */
1845 if (obj_priv->active) {
1846#if WATCH_BUF
1847 DRM_INFO("%s: object %p wait for seqno %08x\n",
1848 __func__, obj, obj_priv->last_rendering_seqno);
1849#endif
1850 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1851 if (ret != 0)
1852 return ret;
1853 }
1854
1855 return 0;
1856}
1857
1858/**
1859 * Unbinds an object from the GTT aperture.
1860 */
0f973f27 1861int
673a394b
EA		 1862i915_gem_object_unbind(struct drm_gem_object *obj)
1863{
1864 struct drm_device *dev = obj->dev;
1865 struct drm_i915_gem_object *obj_priv = obj->driver_private;
de151cf6 1866 loff_t offset;
673a394b
EA		 1867 int ret = 0;
1868
1869#if WATCH_BUF
1870 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1871 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1872#endif
1873 if (obj_priv->gtt_space == NULL)
1874 return 0;
1875
1876 if (obj_priv->pin_count != 0) {
1877 DRM_ERROR("Attempting to unbind pinned buffer\n");
1878 return -EINVAL;
1879 }
1880
673a394b
EA		 1881 /* Move the object to the CPU domain to ensure that
1882 * any possible CPU writes while it's not in the GTT
1883 * are flushed when we go to remap it. This will
1884 * also ensure that all pending GPU writes are finished
1885 * before we unbind.
1886 */
e47c68e9 1887 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
673a394b 1888 if (ret) {
e47c68e9
EA		 1889 if (ret != -ERESTARTSYS)
1890 DRM_ERROR("set_domain failed: %d\n", ret);
673a394b
EA		 1891 return ret;
1892 }
1893
1894 if (obj_priv->agp_mem != NULL) {
1895 drm_unbind_agp(obj_priv->agp_mem);
1896 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1897 obj_priv->agp_mem = NULL;
1898 }
1899
1900 BUG_ON(obj_priv->active);
1901
de151cf6
JB		 1902 /* blow away mappings if mapped through GTT */
1903 offset = ((loff_t) obj->map_list.hash.key) << PAGE_SHIFT;
79e53945
JB		 1904 if (dev->dev_mapping)
1905 unmap_mapping_range(dev->dev_mapping, offset, obj->size, 1);
de151cf6
JB		 1906
1907 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1908 i915_gem_clear_fence_reg(obj);
1909
856fa198 1910 i915_gem_object_put_pages(obj);
673a394b
EA		 1911
1912 if (obj_priv->gtt_space) {
1913 atomic_dec(&dev->gtt_count);
1914 atomic_sub(obj->size, &dev->gtt_memory);
1915
1916 drm_mm_put_block(obj_priv->gtt_space);
1917 obj_priv->gtt_space = NULL;
1918 }
1919
1920 /* Remove ourselves from the LRU list if present. */
1921 if (!list_empty(&obj_priv->list))
1922 list_del_init(&obj_priv->list);
1923
1924 return 0;
1925}
1926
1927static int
1928i915_gem_evict_something(struct drm_device *dev)
1929{
1930 drm_i915_private_t *dev_priv = dev->dev_private;
1931 struct drm_gem_object *obj;
1932 struct drm_i915_gem_object *obj_priv;
1933 int ret = 0;
1934
1935 for (;;) {
1936 /* If there's an inactive buffer available now, grab it
1937 * and be done.
1938 */
1939 if (!list_empty(&dev_priv->mm.inactive_list)) {
1940 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
1941 struct drm_i915_gem_object,
1942 list);
1943 obj = obj_priv->obj;
1944 BUG_ON(obj_priv->pin_count != 0);
1945#if WATCH_LRU
1946 DRM_INFO("%s: evicting %p\n", __func__, obj);
1947#endif
1948 BUG_ON(obj_priv->active);
1949
1950 /* Wait on the rendering and unbind the buffer. */
1951 ret = i915_gem_object_unbind(obj);
1952 break;
1953 }
1954
1955 /* If we didn't get anything, but the ring is still processing
1956 * things, wait for one of those things to finish and hopefully
1957 * leave us a buffer to evict.
1958 */
1959 if (!list_empty(&dev_priv->mm.request_list)) {
1960 struct drm_i915_gem_request *request;
1961
1962 request = list_first_entry(&dev_priv->mm.request_list,
1963 struct drm_i915_gem_request,
1964 list);
1965
1966 ret = i915_wait_request(dev, request->seqno);
1967 if (ret)
1968 break;
1969
1970 /* if waiting caused an object to become inactive,
1971 * then loop around and wait for it. Otherwise, we
1972 * assume that waiting freed and unbound something,
1973 * so there should now be some space in the GTT
1974 */
1975 if (!list_empty(&dev_priv->mm.inactive_list))
1976 continue;
1977 break;
1978 }
1979
1980 /* If we didn't have anything on the request list but there
1981 * are buffers awaiting a flush, emit one and try again.
1982 * When we wait on it, those buffers waiting for that flush
1983 * will get moved to inactive.
1984 */
1985 if (!list_empty(&dev_priv->mm.flushing_list)) {
1986 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1987 struct drm_i915_gem_object,
1988 list);
1989 obj = obj_priv->obj;
1990
1991 i915_gem_flush(dev,
1992 obj->write_domain,
1993 obj->write_domain);
b962442e 1994 i915_add_request(dev, NULL, obj->write_domain);
673a394b
EA		 1995
1996 obj = NULL;
1997 continue;
1998 }
1999
2000 DRM_ERROR("inactive empty %d request empty %d "
2001 "flushing empty %d\n",
2002 list_empty(&dev_priv->mm.inactive_list),
2003 list_empty(&dev_priv->mm.request_list),
2004 list_empty(&dev_priv->mm.flushing_list));
2005 /* If we didn't do any of the above, there's nothing to be done
2006 * and we just can't fit it in.
2007 */
2008 return -ENOMEM;
2009 }
2010 return ret;
2011}
2012
ac94a962
KP		 2013static int
2014i915_gem_evict_everything(struct drm_device *dev)
2015{
2016 int ret;
2017
2018 for (;;) {
2019 ret = i915_gem_evict_something(dev);
2020 if (ret != 0)
2021 break;
2022 }
15c35334
OA		 2023 if (ret == -ENOMEM)
2024 return 0;
ac94a962
KP		 2025 return ret;
2026}
2027
6911a9b8 2028int
856fa198 2029i915_gem_object_get_pages(struct drm_gem_object *obj)
673a394b
EA		 2030{
2031 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2032 int page_count, i;
2033 struct address_space *mapping;
2034 struct inode *inode;
2035 struct page *page;
2036 int ret;
2037
856fa198 2038 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 2039 return 0;
2040
2041 /* Get the list of pages out of our struct file. They'll be pinned
2042 * at this point until we release them.
2043 */
2044 page_count = obj->size / PAGE_SIZE;
856fa198 2045 BUG_ON(obj_priv->pages != NULL);
8e7d2b2c 2046 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
856fa198 2047 if (obj_priv->pages == NULL) {
673a394b 2048 DRM_ERROR("Faled to allocate page list\n");
856fa198 2049 obj_priv->pages_refcount--;
673a394b
EA		 2050 return -ENOMEM;
2051 }
2052
2053 inode = obj->filp->f_path.dentry->d_inode;
2054 mapping = inode->i_mapping;
2055 for (i = 0; i < page_count; i++) {
2056 page = read_mapping_page(mapping, i, NULL);
2057 if (IS_ERR(page)) {
2058 ret = PTR_ERR(page);
2059 DRM_ERROR("read_mapping_page failed: %d\n", ret);
856fa198 2060 i915_gem_object_put_pages(obj);
673a394b
EA		 2061 return ret;
2062 }
856fa198 2063 obj_priv->pages[i] = page;
673a394b 2064 }
280b713b
EA		 2065
2066 if (obj_priv->tiling_mode != I915_TILING_NONE)
2067 i915_gem_object_do_bit_17_swizzle(obj);
2068
673a394b
EA		 2069 return 0;
2070}
2071
de151cf6
JB		 2072static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2073{
2074 struct drm_gem_object *obj = reg->obj;
2075 struct drm_device *dev = obj->dev;
2076 drm_i915_private_t *dev_priv = dev->dev_private;
2077 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2078 int regnum = obj_priv->fence_reg;
2079 uint64_t val;
2080
2081 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2082 0xfffff000) << 32;
2083 val |= obj_priv->gtt_offset & 0xfffff000;
2084 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2085 if (obj_priv->tiling_mode == I915_TILING_Y)
2086 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2087 val |= I965_FENCE_REG_VALID;
2088
2089 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2090}
2091
2092static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2093{
2094 struct drm_gem_object *obj = reg->obj;
2095 struct drm_device *dev = obj->dev;
2096 drm_i915_private_t *dev_priv = dev->dev_private;
2097 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2098 int regnum = obj_priv->fence_reg;
0f973f27 2099 int tile_width;
dc529a4f 2100 uint32_t fence_reg, val;
de151cf6
JB		 2101 uint32_t pitch_val;
2102
2103 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2104 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 2105 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 2106 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 2107 return;
2108 }
2109
0f973f27
JB		 2110 if (obj_priv->tiling_mode == I915_TILING_Y &&
2111 HAS_128_BYTE_Y_TILING(dev))
2112 tile_width = 128;
de151cf6 2113 else
0f973f27
JB		 2114 tile_width = 512;
2115
2116 /* Note: pitch better be a power of two tile widths */
2117 pitch_val = obj_priv->stride / tile_width;
2118 pitch_val = ffs(pitch_val) - 1;
de151cf6
JB		 2119
2120 val = obj_priv->gtt_offset;
2121 if (obj_priv->tiling_mode == I915_TILING_Y)
2122 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2123 val |= I915_FENCE_SIZE_BITS(obj->size);
2124 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2125 val |= I830_FENCE_REG_VALID;
2126
dc529a4f
EA		 2127 if (regnum < 8)
2128 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2129 else
2130 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2131 I915_WRITE(fence_reg, val);
de151cf6
JB		 2132}
2133
2134static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2135{
2136 struct drm_gem_object *obj = reg->obj;
2137 struct drm_device *dev = obj->dev;
2138 drm_i915_private_t *dev_priv = dev->dev_private;
2139 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2140 int regnum = obj_priv->fence_reg;
2141 uint32_t val;
2142 uint32_t pitch_val;
8d7773a3 2143 uint32_t fence_size_bits;
de151cf6 2144
8d7773a3 2145 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
de151cf6 2146 (obj_priv->gtt_offset & (obj->size - 1))) {
8d7773a3 2147 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
0f973f27 2148 __func__, obj_priv->gtt_offset);
de151cf6
JB		 2149 return;
2150 }
2151
e76a16de
EA		 2152 pitch_val = obj_priv->stride / 128;
2153 pitch_val = ffs(pitch_val) - 1;
2154 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2155
de151cf6
JB		 2156 val = obj_priv->gtt_offset;
2157 if (obj_priv->tiling_mode == I915_TILING_Y)
2158 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
8d7773a3
DV		 2159 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2160 WARN_ON(fence_size_bits & ~0x00000f00);
2161 val |= fence_size_bits;
de151cf6
JB		 2162 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2163 val |= I830_FENCE_REG_VALID;
2164
2165 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2166
2167}
2168
2169/**
2170 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2171 * @obj: object to map through a fence reg
0f973f27 2172 * @write: object is about to be written
de151cf6
JB		 2173 *
2174 * When mapping objects through the GTT, userspace wants to be able to write
2175 * to them without having to worry about swizzling if the object is tiled.
2176 *
2177 * This function walks the fence regs looking for a free one for @obj,
2178 * stealing one if it can't find any.
2179 *
2180 * It then sets up the reg based on the object's properties: address, pitch
2181 * and tiling format.
2182 */
d9ddcb96 2183static int
0f973f27 2184i915_gem_object_get_fence_reg(struct drm_gem_object *obj, bool write)
de151cf6
JB		 2185{
2186 struct drm_device *dev = obj->dev;
79e53945 2187 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6
JB		 2188 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2189 struct drm_i915_fence_reg *reg = NULL;
fc7170ba
CW		 2190 struct drm_i915_gem_object *old_obj_priv = NULL;
2191 int i, ret, avail;
de151cf6
JB		 2192
2193 switch (obj_priv->tiling_mode) {
2194 case I915_TILING_NONE:
2195 WARN(1, "allocating a fence for non-tiled object?\n");
2196 break;
2197 case I915_TILING_X:
0f973f27
JB		 2198 if (!obj_priv->stride)
2199 return -EINVAL;
2200 WARN((obj_priv->stride & (512 - 1)),
2201 "object 0x%08x is X tiled but has non-512B pitch\n",
2202 obj_priv->gtt_offset);
de151cf6
JB		 2203 break;
2204 case I915_TILING_Y:
0f973f27
JB		 2205 if (!obj_priv->stride)
2206 return -EINVAL;
2207 WARN((obj_priv->stride & (128 - 1)),
2208 "object 0x%08x is Y tiled but has non-128B pitch\n",
2209 obj_priv->gtt_offset);
de151cf6
JB		 2210 break;
2211 }
2212
2213 /* First try to find a free reg */
9b2412f9 2214try_again:
fc7170ba 2215 avail = 0;
de151cf6
JB		 2216 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2217 reg = &dev_priv->fence_regs[i];
2218 if (!reg->obj)
2219 break;
fc7170ba
CW		 2220
2221 old_obj_priv = reg->obj->driver_private;
2222 if (!old_obj_priv->pin_count)
2223 avail++;
de151cf6
JB		 2224 }
2225
2226 /* None available, try to steal one or wait for a user to finish */
2227 if (i == dev_priv->num_fence_regs) {
d7619c4b 2228 uint32_t seqno = dev_priv->mm.next_gem_seqno;
de151cf6
JB		 2229 loff_t offset;
2230
fc7170ba
CW		 2231 if (avail == 0)
2232 return -ENOMEM;
2233
de151cf6
JB		 2234 for (i = dev_priv->fence_reg_start;
2235 i < dev_priv->num_fence_regs; i++) {
d7619c4b
CW		 2236 uint32_t this_seqno;
2237
de151cf6
JB		 2238 reg = &dev_priv->fence_regs[i];
2239 old_obj_priv = reg->obj->driver_private;
d7619c4b
CW		 2240
2241 if (old_obj_priv->pin_count)
2242 continue;
2243
2244 /* i915 uses fences for GPU access to tiled buffers */
2245 if (IS_I965G(dev) || !old_obj_priv->active)
de151cf6 2246 break;
d7619c4b
CW		 2247
2248 /* find the seqno of the first available fence */
2249 this_seqno = old_obj_priv->last_rendering_seqno;
2250 if (this_seqno != 0 &&
2251 reg->obj->write_domain == 0 &&
2252 i915_seqno_passed(seqno, this_seqno))
2253 seqno = this_seqno;
de151cf6
JB		 2254 }
2255
2256 /*
2257 * Now things get ugly... we have to wait for one of the
2258 * objects to finish before trying again.
2259 */
2260 if (i == dev_priv->num_fence_regs) {
d7619c4b
CW		 2261 if (seqno == dev_priv->mm.next_gem_seqno) {
2262 i915_gem_flush(dev,
2263 I915_GEM_GPU_DOMAINS,
2264 I915_GEM_GPU_DOMAINS);
b962442e 2265 seqno = i915_add_request(dev, NULL,
d7619c4b
CW		 2266 I915_GEM_GPU_DOMAINS);
2267 if (seqno == 0)
2268 return -ENOMEM;
de151cf6 2269 }
d7619c4b
CW		 2270
2271 ret = i915_wait_request(dev, seqno);
2272 if (ret)
2273 return ret;
de151cf6
JB		 2274 goto try_again;
2275 }
2276
2277 /*
2278 * Zap this virtual mapping so we can set up a fence again
2279 * for this object next time we need it.
2280 */
2281 offset = ((loff_t) reg->obj->map_list.hash.key) << PAGE_SHIFT;
79e53945
JB		 2282 if (dev->dev_mapping)
2283 unmap_mapping_range(dev->dev_mapping, offset,
2284 reg->obj->size, 1);
de151cf6
JB		 2285 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
2286 }
2287
2288 obj_priv->fence_reg = i;
2289 reg->obj = obj;
2290
2291 if (IS_I965G(dev))
2292 i965_write_fence_reg(reg);
2293 else if (IS_I9XX(dev))
2294 i915_write_fence_reg(reg);
2295 else
2296 i830_write_fence_reg(reg);
d9ddcb96
EA		 2297
2298 return 0;
de151cf6
JB		 2299}
2300
2301/**
2302 * i915_gem_clear_fence_reg - clear out fence register info
2303 * @obj: object to clear
2304 *
2305 * Zeroes out the fence register itself and clears out the associated
2306 * data structures in dev_priv and obj_priv.
2307 */
2308static void
2309i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2310{
2311 struct drm_device *dev = obj->dev;
79e53945 2312 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 2313 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2314
2315 if (IS_I965G(dev))
2316 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
dc529a4f
EA		 2317 else {
2318 uint32_t fence_reg;
2319
2320 if (obj_priv->fence_reg < 8)
2321 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2322 else
2323 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2324 8) * 4;
2325
2326 I915_WRITE(fence_reg, 0);
2327 }
de151cf6
JB		 2328
2329 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2330 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2331}
2332
673a394b
EA		 2333/**
2334 * Finds free space in the GTT aperture and binds the object there.
2335 */
2336static int
2337i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2338{
2339 struct drm_device *dev = obj->dev;
2340 drm_i915_private_t *dev_priv = dev->dev_private;
2341 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2342 struct drm_mm_node *free_space;
2343 int page_count, ret;
2344
9bb2d6f9
EA		 2345 if (dev_priv->mm.suspended)
2346 return -EBUSY;
673a394b 2347 if (alignment == 0)
0f973f27 2348 alignment = i915_gem_get_gtt_alignment(obj);
8d7773a3 2349 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
673a394b
EA		 2350 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2351 return -EINVAL;
2352 }
2353
2354 search_free:
2355 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2356 obj->size, alignment, 0);
2357 if (free_space != NULL) {
2358 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2359 alignment);
2360 if (obj_priv->gtt_space != NULL) {
2361 obj_priv->gtt_space->private = obj;
2362 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2363 }
2364 }
2365 if (obj_priv->gtt_space == NULL) {
5e118f41
CW		 2366 bool lists_empty;
2367
673a394b
EA		 2368 /* If the gtt is empty and we're still having trouble
2369 * fitting our object in, we're out of memory.
2370 */
2371#if WATCH_LRU
2372 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2373#endif
5e118f41
CW		 2374 spin_lock(&dev_priv->mm.active_list_lock);
2375 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2376 list_empty(&dev_priv->mm.flushing_list) &&
2377 list_empty(&dev_priv->mm.active_list));
2378 spin_unlock(&dev_priv->mm.active_list_lock);
2379 if (lists_empty) {
673a394b
EA		 2380 DRM_ERROR("GTT full, but LRU list empty\n");
2381 return -ENOMEM;
2382 }
2383
2384 ret = i915_gem_evict_something(dev);
2385 if (ret != 0) {
ac94a962
KP		 2386 if (ret != -ERESTARTSYS)
2387 DRM_ERROR("Failed to evict a buffer %d\n", ret);
673a394b
EA		 2388 return ret;
2389 }
2390 goto search_free;
2391 }
2392
2393#if WATCH_BUF
2394 DRM_INFO("Binding object of size %d at 0x%08x\n",
2395 obj->size, obj_priv->gtt_offset);
2396#endif
856fa198 2397 ret = i915_gem_object_get_pages(obj);
673a394b
EA		 2398 if (ret) {
2399 drm_mm_put_block(obj_priv->gtt_space);
2400 obj_priv->gtt_space = NULL;
2401 return ret;
2402 }
2403
2404 page_count = obj->size / PAGE_SIZE;
2405 /* Create an AGP memory structure pointing at our pages, and bind it
2406 * into the GTT.
2407 */
2408 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2409 obj_priv->pages,
673a394b 2410 page_count,
ba1eb1d8
KP		 2411 obj_priv->gtt_offset,
2412 obj_priv->agp_type);
673a394b 2413 if (obj_priv->agp_mem == NULL) {
856fa198 2414 i915_gem_object_put_pages(obj);
673a394b
EA		 2415 drm_mm_put_block(obj_priv->gtt_space);
2416 obj_priv->gtt_space = NULL;
2417 return -ENOMEM;
2418 }
2419 atomic_inc(&dev->gtt_count);
2420 atomic_add(obj->size, &dev->gtt_memory);
2421
2422 /* Assert that the object is not currently in any GPU domain. As it
2423 * wasn't in the GTT, there shouldn't be any way it could have been in
2424 * a GPU cache
2425 */
21d509e3
CW		 2426 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2427 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
673a394b
EA		 2428
2429 return 0;
2430}
2431
2432void
2433i915_gem_clflush_object(struct drm_gem_object *obj)
2434{
2435 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2436
2437 /* If we don't have a page list set up, then we're not pinned
2438 * to GPU, and we can ignore the cache flush because it'll happen
2439 * again at bind time.
2440 */
856fa198 2441 if (obj_priv->pages == NULL)
673a394b
EA		 2442 return;
2443
cfa16a0d
EA		 2444 /* XXX: The 865 in particular appears to be weird in how it handles
2445 * cache flushing. We haven't figured it out, but the
2446 * clflush+agp_chipset_flush doesn't appear to successfully get the
2447 * data visible to the PGU, while wbinvd + agp_chipset_flush does.
2448 */
2449 if (IS_I865G(obj->dev)) {
2450 wbinvd();
2451 return;
2452 }
2453
856fa198 2454 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2455}
2456
e47c68e9
EA		 2457/** Flushes any GPU write domain for the object if it's dirty. */
2458static void
2459i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2460{
2461 struct drm_device *dev = obj->dev;
2462 uint32_t seqno;
2463
2464 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2465 return;
2466
2467 /* Queue the GPU write cache flushing we need. */
2468 i915_gem_flush(dev, 0, obj->write_domain);
b962442e 2469 seqno = i915_add_request(dev, NULL, obj->write_domain);
e47c68e9
EA		 2470 obj->write_domain = 0;
2471 i915_gem_object_move_to_active(obj, seqno);
2472}
2473
2474/** Flushes the GTT write domain for the object if it's dirty. */
2475static void
2476i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2477{
2478 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2479 return;
2480
2481 /* No actual flushing is required for the GTT write domain. Writes
2482 * to it immediately go to main memory as far as we know, so there's
2483 * no chipset flush. It also doesn't land in render cache.
2484 */
2485 obj->write_domain = 0;
2486}
2487
2488/** Flushes the CPU write domain for the object if it's dirty. */
2489static void
2490i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2491{
2492 struct drm_device *dev = obj->dev;
2493
2494 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2495 return;
2496
2497 i915_gem_clflush_object(obj);
2498 drm_agp_chipset_flush(dev);
2499 obj->write_domain = 0;
2500}
2501
2ef7eeaa
EA		 2502/**
2503 * Moves a single object to the GTT read, and possibly write domain.
2504 *
2505 * This function returns when the move is complete, including waiting on
2506 * flushes to occur.
2507 */
79e53945 2508int
2ef7eeaa
EA		 2509i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2510{
2ef7eeaa 2511 struct drm_i915_gem_object *obj_priv = obj->driver_private;
e47c68e9 2512 int ret;
2ef7eeaa 2513
02354392
EA		 2514 /* Not valid to be called on unbound objects. */
2515 if (obj_priv->gtt_space == NULL)
2516 return -EINVAL;
2517
e47c68e9
EA		 2518 i915_gem_object_flush_gpu_write_domain(obj);
2519 /* Wait on any GPU rendering and flushing to occur. */
2520 ret = i915_gem_object_wait_rendering(obj);
2521 if (ret != 0)
2522 return ret;
2523
2524 /* If we're writing through the GTT domain, then CPU and GPU caches
2525 * will need to be invalidated at next use.
2ef7eeaa 2526 */
e47c68e9
EA		 2527 if (write)
2528 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2ef7eeaa 2529
e47c68e9 2530 i915_gem_object_flush_cpu_write_domain(obj);
2ef7eeaa 2531
e47c68e9
EA		 2532 /* It should now be out of any other write domains, and we can update
2533 * the domain values for our changes.
2534 */
2535 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2536 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2537 if (write) {
2538 obj->write_domain = I915_GEM_DOMAIN_GTT;
2539 obj_priv->dirty = 1;
2ef7eeaa
EA		 2540 }
2541
e47c68e9
EA		 2542 return 0;
2543}
2544
2545/**
2546 * Moves a single object to the CPU read, and possibly write domain.
2547 *
2548 * This function returns when the move is complete, including waiting on
2549 * flushes to occur.
2550 */
2551static int
2552i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2553{
e47c68e9
EA		 2554 int ret;
2555
2556 i915_gem_object_flush_gpu_write_domain(obj);
2ef7eeaa 2557 /* Wait on any GPU rendering and flushing to occur. */
e47c68e9
EA		 2558 ret = i915_gem_object_wait_rendering(obj);
2559 if (ret != 0)
2560 return ret;
2ef7eeaa 2561
e47c68e9 2562 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2563
e47c68e9
EA		 2564 /* If we have a partially-valid cache of the object in the CPU,
2565 * finish invalidating it and free the per-page flags.
2ef7eeaa 2566 */
e47c68e9 2567 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2568
e47c68e9
EA		 2569 /* Flush the CPU cache if it's still invalid. */
2570 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2571 i915_gem_clflush_object(obj);
2ef7eeaa 2572
e47c68e9 2573 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2574 }
2575
2576 /* It should now be out of any other write domains, and we can update
2577 * the domain values for our changes.
2578 */
e47c68e9
EA		 2579 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2580
2581 /* If we're writing through the CPU, then the GPU read domains will
2582 * need to be invalidated at next use.
2583 */
2584 if (write) {
2585 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2586 obj->write_domain = I915_GEM_DOMAIN_CPU;
2587 }
2ef7eeaa
EA		 2588
2589 return 0;
2590}
2591
673a394b
EA		 2592/*
2593 * Set the next domain for the specified object. This
2594 * may not actually perform the necessary flushing/invaliding though,
2595 * as that may want to be batched with other set_domain operations
2596 *
2597 * This is (we hope) the only really tricky part of gem. The goal
2598 * is fairly simple -- track which caches hold bits of the object
2599 * and make sure they remain coherent. A few concrete examples may
2600 * help to explain how it works. For shorthand, we use the notation
2601 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2602 * a pair of read and write domain masks.
2603 *
2604 * Case 1: the batch buffer
2605 *
2606 * 1. Allocated
2607 * 2. Written by CPU
2608 * 3. Mapped to GTT
2609 * 4. Read by GPU
2610 * 5. Unmapped from GTT
2611 * 6. Freed
2612 *
2613 * Let's take these a step at a time
2614 *
2615 * 1. Allocated
2616 * Pages allocated from the kernel may still have
2617 * cache contents, so we set them to (CPU, CPU) always.
2618 * 2. Written by CPU (using pwrite)
2619 * The pwrite function calls set_domain (CPU, CPU) and
2620 * this function does nothing (as nothing changes)
2621 * 3. Mapped by GTT
2622 * This function asserts that the object is not
2623 * currently in any GPU-based read or write domains
2624 * 4. Read by GPU
2625 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2626 * As write_domain is zero, this function adds in the
2627 * current read domains (CPU+COMMAND, 0).
2628 * flush_domains is set to CPU.
2629 * invalidate_domains is set to COMMAND
2630 * clflush is run to get data out of the CPU caches
2631 * then i915_dev_set_domain calls i915_gem_flush to
2632 * emit an MI_FLUSH and drm_agp_chipset_flush
2633 * 5. Unmapped from GTT
2634 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2635 * flush_domains and invalidate_domains end up both zero
2636 * so no flushing/invalidating happens
2637 * 6. Freed
2638 * yay, done
2639 *
2640 * Case 2: The shared render buffer
2641 *
2642 * 1. Allocated
2643 * 2. Mapped to GTT
2644 * 3. Read/written by GPU
2645 * 4. set_domain to (CPU,CPU)
2646 * 5. Read/written by CPU
2647 * 6. Read/written by GPU
2648 *
2649 * 1. Allocated
2650 * Same as last example, (CPU, CPU)
2651 * 2. Mapped to GTT
2652 * Nothing changes (assertions find that it is not in the GPU)
2653 * 3. Read/written by GPU
2654 * execbuffer calls set_domain (RENDER, RENDER)
2655 * flush_domains gets CPU
2656 * invalidate_domains gets GPU
2657 * clflush (obj)
2658 * MI_FLUSH and drm_agp_chipset_flush
2659 * 4. set_domain (CPU, CPU)
2660 * flush_domains gets GPU
2661 * invalidate_domains gets CPU
2662 * wait_rendering (obj) to make sure all drawing is complete.
2663 * This will include an MI_FLUSH to get the data from GPU
2664 * to memory
2665 * clflush (obj) to invalidate the CPU cache
2666 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2667 * 5. Read/written by CPU
2668 * cache lines are loaded and dirtied
2669 * 6. Read written by GPU
2670 * Same as last GPU access
2671 *
2672 * Case 3: The constant buffer
2673 *
2674 * 1. Allocated
2675 * 2. Written by CPU
2676 * 3. Read by GPU
2677 * 4. Updated (written) by CPU again
2678 * 5. Read by GPU
2679 *
2680 * 1. Allocated
2681 * (CPU, CPU)
2682 * 2. Written by CPU
2683 * (CPU, CPU)
2684 * 3. Read by GPU
2685 * (CPU+RENDER, 0)
2686 * flush_domains = CPU
2687 * invalidate_domains = RENDER
2688 * clflush (obj)
2689 * MI_FLUSH
2690 * drm_agp_chipset_flush
2691 * 4. Updated (written) by CPU again
2692 * (CPU, CPU)
2693 * flush_domains = 0 (no previous write domain)
2694 * invalidate_domains = 0 (no new read domains)
2695 * 5. Read by GPU
2696 * (CPU+RENDER, 0)
2697 * flush_domains = CPU
2698 * invalidate_domains = RENDER
2699 * clflush (obj)
2700 * MI_FLUSH
2701 * drm_agp_chipset_flush
2702 */
c0d90829 2703static void
8b0e378a 2704i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 2705{
2706 struct drm_device *dev = obj->dev;
2707 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2708 uint32_t invalidate_domains = 0;
2709 uint32_t flush_domains = 0;
e47c68e9 2710
8b0e378a
EA		 2711 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2712 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b
EA		 2713
2714#if WATCH_BUF
2715 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2716 __func__, obj,
8b0e378a
EA		 2717 obj->read_domains, obj->pending_read_domains,
2718 obj->write_domain, obj->pending_write_domain);
673a394b
EA		 2719#endif
2720 /*
2721 * If the object isn't moving to a new write domain,
2722 * let the object stay in multiple read domains
2723 */
8b0e378a
EA		 2724 if (obj->pending_write_domain == 0)
2725 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 2726 else
2727 obj_priv->dirty = 1;
2728
2729 /*
2730 * Flush the current write domain if
2731 * the new read domains don't match. Invalidate
2732 * any read domains which differ from the old
2733 * write domain
2734 */
8b0e378a
EA		 2735 if (obj->write_domain &&
2736 obj->write_domain != obj->pending_read_domains) {
673a394b 2737 flush_domains |= obj->write_domain;
8b0e378a
EA		 2738 invalidate_domains |=
2739 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 2740 }
2741 /*
2742 * Invalidate any read caches which may have
2743 * stale data. That is, any new read domains.
2744 */
8b0e378a 2745 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
673a394b
EA		 2746 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
2747#if WATCH_BUF
2748 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
2749 __func__, flush_domains, invalidate_domains);
2750#endif
673a394b
EA		 2751 i915_gem_clflush_object(obj);
2752 }
2753
efbeed96
EA		 2754 /* The actual obj->write_domain will be updated with
2755 * pending_write_domain after we emit the accumulated flush for all
2756 * of our domain changes in execbuffers (which clears objects'
2757 * write_domains). So if we have a current write domain that we
2758 * aren't changing, set pending_write_domain to that.
2759 */
2760 if (flush_domains == 0 && obj->pending_write_domain == 0)
2761 obj->pending_write_domain = obj->write_domain;
8b0e378a 2762 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 2763
2764 dev->invalidate_domains |= invalidate_domains;
2765 dev->flush_domains |= flush_domains;
2766#if WATCH_BUF
2767 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
2768 __func__,
2769 obj->read_domains, obj->write_domain,
2770 dev->invalidate_domains, dev->flush_domains);
2771#endif
673a394b
EA		 2772}
2773
2774/**
e47c68e9 2775 * Moves the object from a partially CPU read to a full one.
673a394b 2776 *
e47c68e9
EA		 2777 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
2778 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 2779 */
e47c68e9
EA		 2780static void
2781i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b
EA		 2782{
2783 struct drm_i915_gem_object *obj_priv = obj->driver_private;
673a394b 2784
e47c68e9
EA		 2785 if (!obj_priv->page_cpu_valid)
2786 return;
2787
2788 /* If we're partially in the CPU read domain, finish moving it in.
2789 */
2790 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
2791 int i;
2792
2793 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
2794 if (obj_priv->page_cpu_valid[i])
2795 continue;
856fa198 2796 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 2797 }
e47c68e9
EA		 2798 }
2799
2800 /* Free the page_cpu_valid mappings which are now stale, whether
2801 * or not we've got I915_GEM_DOMAIN_CPU.
2802 */
2803 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
2804 DRM_MEM_DRIVER);
2805 obj_priv->page_cpu_valid = NULL;
2806}
2807
2808/**
2809 * Set the CPU read domain on a range of the object.
2810 *
2811 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
2812 * not entirely valid. The page_cpu_valid member of the object flags which
2813 * pages have been flushed, and will be respected by
2814 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
2815 * of the whole object.
2816 *
2817 * This function returns when the move is complete, including waiting on
2818 * flushes to occur.
2819 */
2820static int
2821i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
2822 uint64_t offset, uint64_t size)
2823{
2824 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2825 int i, ret;
673a394b 2826
e47c68e9
EA		 2827 if (offset == 0 && size == obj->size)
2828 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 2829
e47c68e9
EA		 2830 i915_gem_object_flush_gpu_write_domain(obj);
2831 /* Wait on any GPU rendering and flushing to occur. */
6a47baa6 2832 ret = i915_gem_object_wait_rendering(obj);
e47c68e9 2833 if (ret != 0)
6a47baa6 2834 return ret;
e47c68e9
EA		 2835 i915_gem_object_flush_gtt_write_domain(obj);
2836
2837 /* If we're already fully in the CPU read domain, we're done. */
2838 if (obj_priv->page_cpu_valid == NULL &&
2839 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
2840 return 0;
673a394b 2841
e47c68e9
EA		 2842 /* Otherwise, create/clear the per-page CPU read domain flag if we're
2843 * newly adding I915_GEM_DOMAIN_CPU
2844 */
673a394b
EA		 2845 if (obj_priv->page_cpu_valid == NULL) {
2846 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
2847 DRM_MEM_DRIVER);
e47c68e9
EA		 2848 if (obj_priv->page_cpu_valid == NULL)
2849 return -ENOMEM;
2850 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
2851 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 2852
2853 /* Flush the cache on any pages that are still invalid from the CPU's
2854 * perspective.
2855 */
e47c68e9
EA		 2856 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
2857 i++) {
673a394b
EA		 2858 if (obj_priv->page_cpu_valid[i])
2859 continue;
2860
856fa198 2861 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 2862
2863 obj_priv->page_cpu_valid[i] = 1;
2864 }
2865
e47c68e9
EA		 2866 /* It should now be out of any other write domains, and we can update
2867 * the domain values for our changes.
2868 */
2869 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2870
2871 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2872
673a394b
EA		 2873 return 0;
2874}
2875
673a394b
EA		 2876/**
2877 * Pin an object to the GTT and evaluate the relocations landing in it.
2878 */
2879static int
2880i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
2881 struct drm_file *file_priv,
40a5f0de
EA		 2882 struct drm_i915_gem_exec_object *entry,
2883 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 2884{
2885 struct drm_device *dev = obj->dev;
0839ccb8 2886 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 2887 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2888 int i, ret;
0839ccb8 2889 void __iomem *reloc_page;
673a394b
EA		 2890
2891 /* Choose the GTT offset for our buffer and put it there. */
2892 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
2893 if (ret)
2894 return ret;
2895
2896 entry->offset = obj_priv->gtt_offset;
2897
673a394b
EA		 2898 /* Apply the relocations, using the GTT aperture to avoid cache
2899 * flushing requirements.
2900 */
2901 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 2902 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 2903 struct drm_gem_object *target_obj;
2904 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 2905 uint32_t reloc_val, reloc_offset;
2906 uint32_t __iomem *reloc_entry;
673a394b 2907
673a394b 2908 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 2909 reloc->target_handle);
673a394b
EA		 2910 if (target_obj == NULL) {
2911 i915_gem_object_unpin(obj);
2912 return -EBADF;
2913 }
2914 target_obj_priv = target_obj->driver_private;
2915
2916 /* The target buffer should have appeared before us in the
2917 * exec_object list, so it should have a GTT space bound by now.
2918 */
2919 if (target_obj_priv->gtt_space == NULL) {
2920 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 2921 reloc->target_handle);
673a394b
EA		 2922 drm_gem_object_unreference(target_obj);
2923 i915_gem_object_unpin(obj);
2924 return -EINVAL;
2925 }
2926
40a5f0de 2927 if (reloc->offset > obj->size - 4) {
673a394b
EA		 2928 DRM_ERROR("Relocation beyond object bounds: "
2929 "obj %p target %d offset %d size %d.\n",
40a5f0de
EA		 2930 obj, reloc->target_handle,
2931 (int) reloc->offset, (int) obj->size);
673a394b
EA		 2932 drm_gem_object_unreference(target_obj);
2933 i915_gem_object_unpin(obj);
2934 return -EINVAL;
2935 }
40a5f0de 2936 if (reloc->offset & 3) {
673a394b
EA		 2937 DRM_ERROR("Relocation not 4-byte aligned: "
2938 "obj %p target %d offset %d.\n",
40a5f0de
EA		 2939 obj, reloc->target_handle,
2940 (int) reloc->offset);
673a394b
EA		 2941 drm_gem_object_unreference(target_obj);
2942 i915_gem_object_unpin(obj);
2943 return -EINVAL;
2944 }
2945
40a5f0de
EA		 2946 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
2947 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 2948 DRM_ERROR("reloc with read/write CPU domains: "
2949 "obj %p target %d offset %d "
2950 "read %08x write %08x",
40a5f0de
EA		 2951 obj, reloc->target_handle,
2952 (int) reloc->offset,
2953 reloc->read_domains,
2954 reloc->write_domain);
491152b8
CW		 2955 drm_gem_object_unreference(target_obj);
2956 i915_gem_object_unpin(obj);
e47c68e9
EA		 2957 return -EINVAL;
2958 }
2959
40a5f0de
EA		 2960 if (reloc->write_domain && target_obj->pending_write_domain &&
2961 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 2962 DRM_ERROR("Write domain conflict: "
2963 "obj %p target %d offset %d "
2964 "new %08x old %08x\n",
40a5f0de
EA		 2965 obj, reloc->target_handle,
2966 (int) reloc->offset,
2967 reloc->write_domain,
673a394b
EA		 2968 target_obj->pending_write_domain);
2969 drm_gem_object_unreference(target_obj);
2970 i915_gem_object_unpin(obj);
2971 return -EINVAL;
2972 }
2973
2974#if WATCH_RELOC
2975 DRM_INFO("%s: obj %p offset %08x target %d "
2976 "read %08x write %08x gtt %08x "
2977 "presumed %08x delta %08x\n",
2978 __func__,
2979 obj,
40a5f0de
EA		 2980 (int) reloc->offset,
2981 (int) reloc->target_handle,
2982 (int) reloc->read_domains,
2983 (int) reloc->write_domain,
673a394b 2984 (int) target_obj_priv->gtt_offset,
40a5f0de
EA		 2985 (int) reloc->presumed_offset,
2986 reloc->delta);
673a394b
EA		 2987#endif
2988
40a5f0de
EA		 2989 target_obj->pending_read_domains |= reloc->read_domains;
2990 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 2991
2992 /* If the relocation already has the right value in it, no
2993 * more work needs to be done.
2994 */
40a5f0de 2995 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 2996 drm_gem_object_unreference(target_obj);
2997 continue;
2998 }
2999
2ef7eeaa
EA		 3000 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3001 if (ret != 0) {
3002 drm_gem_object_unreference(target_obj);
3003 i915_gem_object_unpin(obj);
3004 return -EINVAL;
673a394b
EA		 3005 }
3006
3007 /* Map the page containing the relocation we're going to
3008 * perform.
3009 */
40a5f0de 3010 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 3011 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3012 (reloc_offset &
3013 ~(PAGE_SIZE - 1)));
3043c60c 3014 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 3015 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 3016 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b
EA		 3017
3018#if WATCH_BUF
3019 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
40a5f0de 3020 obj, (unsigned int) reloc->offset,
673a394b
EA		 3021 readl(reloc_entry), reloc_val);
3022#endif
3023 writel(reloc_val, reloc_entry);
0839ccb8 3024 io_mapping_unmap_atomic(reloc_page);
673a394b 3025
40a5f0de
EA		 3026 /* The updated presumed offset for this entry will be
3027 * copied back out to the user.
673a394b 3028 */
40a5f0de 3029 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 3030
3031 drm_gem_object_unreference(target_obj);
3032 }
3033
673a394b
EA		 3034#if WATCH_BUF
3035 if (0)
3036 i915_gem_dump_object(obj, 128, __func__, ~0);
3037#endif
3038 return 0;
3039}
3040
3041/** Dispatch a batchbuffer to the ring
3042 */
3043static int
3044i915_dispatch_gem_execbuffer(struct drm_device *dev,
3045 struct drm_i915_gem_execbuffer *exec,
201361a5 3046 struct drm_clip_rect *cliprects,
673a394b
EA		 3047 uint64_t exec_offset)
3048{
3049 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3050 int nbox = exec->num_cliprects;
3051 int i = 0, count;
83d60795 3052 uint32_t exec_start, exec_len;
673a394b
EA		 3053 RING_LOCALS;
3054
3055 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3056 exec_len = (uint32_t) exec->batch_len;
3057
673a394b
EA		 3058 count = nbox ? nbox : 1;
3059
3060 for (i = 0; i < count; i++) {
3061 if (i < nbox) {
201361a5 3062 int ret = i915_emit_box(dev, cliprects, i,
673a394b
EA		 3063 exec->DR1, exec->DR4);
3064 if (ret)
3065 return ret;
3066 }
3067
3068 if (IS_I830(dev) || IS_845G(dev)) {
3069 BEGIN_LP_RING(4);
3070 OUT_RING(MI_BATCH_BUFFER);
3071 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3072 OUT_RING(exec_start + exec_len - 4);
3073 OUT_RING(0);
3074 ADVANCE_LP_RING();
3075 } else {
3076 BEGIN_LP_RING(2);
3077 if (IS_I965G(dev)) {
3078 OUT_RING(MI_BATCH_BUFFER_START |
3079 (2 << 6) |
3080 MI_BATCH_NON_SECURE_I965);
3081 OUT_RING(exec_start);
3082 } else {
3083 OUT_RING(MI_BATCH_BUFFER_START |
3084 (2 << 6));
3085 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3086 }
3087 ADVANCE_LP_RING();
3088 }
3089 }
3090
3091 /* XXX breadcrumb */
3092 return 0;
3093}
3094
3095/* Throttle our rendering by waiting until the ring has completed our requests
3096 * emitted over 20 msec ago.
3097 *
b962442e
EA		 3098 * Note that if we were to use the current jiffies each time around the loop,
3099 * we wouldn't escape the function with any frames outstanding if the time to
3100 * render a frame was over 20ms.
3101 *
673a394b
EA		 3102 * This should get us reasonable parallelism between CPU and GPU but also
3103 * relatively low latency when blocking on a particular request to finish.
3104 */
3105static int
3106i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3107{
3108 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3109 int ret = 0;
b962442e 3110 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
673a394b
EA		 3111
3112 mutex_lock(&dev->struct_mutex);
b962442e
EA		 3113 while (!list_empty(&i915_file_priv->mm.request_list)) {
3114 struct drm_i915_gem_request *request;
3115
3116 request = list_first_entry(&i915_file_priv->mm.request_list,
3117 struct drm_i915_gem_request,
3118 client_list);
3119
3120 if (time_after_eq(request->emitted_jiffies, recent_enough))
3121 break;
3122
3123 ret = i915_wait_request(dev, request->seqno);
3124 if (ret != 0)
3125 break;
3126 }
673a394b 3127 mutex_unlock(&dev->struct_mutex);
b962442e 3128
673a394b
EA		 3129 return ret;
3130}
3131
40a5f0de
EA		 3132static int
3133i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list,
3134 uint32_t buffer_count,
3135 struct drm_i915_gem_relocation_entry **relocs)
3136{
3137 uint32_t reloc_count = 0, reloc_index = 0, i;
3138 int ret;
3139
3140 *relocs = NULL;
3141 for (i = 0; i < buffer_count; i++) {
3142 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3143 return -EINVAL;
3144 reloc_count += exec_list[i].relocation_count;
3145 }
3146
8e7d2b2c 3147 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
40a5f0de
EA		 3148 if (*relocs == NULL)
3149 return -ENOMEM;
3150
3151 for (i = 0; i < buffer_count; i++) {
3152 struct drm_i915_gem_relocation_entry __user *user_relocs;
3153
3154 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3155
3156 ret = copy_from_user(&(*relocs)[reloc_index],
3157 user_relocs,
3158 exec_list[i].relocation_count *
3159 sizeof(**relocs));
3160 if (ret != 0) {
8e7d2b2c 3161 drm_free_large(*relocs);
40a5f0de 3162 *relocs = NULL;
2bc43b5c 3163 return -EFAULT;
40a5f0de
EA		 3164 }
3165
3166 reloc_index += exec_list[i].relocation_count;
3167 }
3168
2bc43b5c 3169 return 0;
40a5f0de
EA		 3170}
3171
3172static int
3173i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list,
3174 uint32_t buffer_count,
3175 struct drm_i915_gem_relocation_entry *relocs)
3176{
3177 uint32_t reloc_count = 0, i;
2bc43b5c 3178 int ret = 0;
40a5f0de
EA		 3179
3180 for (i = 0; i < buffer_count; i++) {
3181 struct drm_i915_gem_relocation_entry __user *user_relocs;
2bc43b5c 3182 int unwritten;
40a5f0de
EA		 3183
3184 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3185
2bc43b5c
FM		 3186 unwritten = copy_to_user(user_relocs,
3187 &relocs[reloc_count],
3188 exec_list[i].relocation_count *
3189 sizeof(*relocs));
3190
3191 if (unwritten) {
3192 ret = -EFAULT;
3193 goto err;
40a5f0de
EA		 3194 }
3195
3196 reloc_count += exec_list[i].relocation_count;
3197 }
3198
2bc43b5c 3199err:
8e7d2b2c 3200 drm_free_large(relocs);
40a5f0de
EA		 3201
3202 return ret;
3203}
3204
83d60795
CW		 3205static int
3206i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer *exec,
3207 uint64_t exec_offset)
3208{
3209 uint32_t exec_start, exec_len;
3210
3211 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3212 exec_len = (uint32_t) exec->batch_len;
3213
3214 if ((exec_start | exec_len) & 0x7)
3215 return -EINVAL;
3216
3217 if (!exec_start)
3218 return -EINVAL;
3219
3220 return 0;
3221}
3222
673a394b
EA		 3223int
3224i915_gem_execbuffer(struct drm_device *dev, void *data,
3225 struct drm_file *file_priv)
3226{
3227 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 3228 struct drm_i915_gem_execbuffer *args = data;
3229 struct drm_i915_gem_exec_object *exec_list = NULL;
3230 struct drm_gem_object **object_list = NULL;
3231 struct drm_gem_object *batch_obj;
b70d11da 3232 struct drm_i915_gem_object *obj_priv;
201361a5 3233 struct drm_clip_rect *cliprects = NULL;
40a5f0de
EA		 3234 struct drm_i915_gem_relocation_entry *relocs;
3235 int ret, ret2, i, pinned = 0;
673a394b 3236 uint64_t exec_offset;
40a5f0de 3237 uint32_t seqno, flush_domains, reloc_index;
ac94a962 3238 int pin_tries;
673a394b
EA		 3239
3240#if WATCH_EXEC
3241 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3242 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3243#endif
3244
4f481ed2
EA		 3245 if (args->buffer_count < 1) {
3246 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3247 return -EINVAL;
3248 }
673a394b 3249 /* Copy in the exec list from userland */
8e7d2b2c
JB		 3250 exec_list = drm_calloc_large(sizeof(*exec_list), args->buffer_count);
3251 object_list = drm_calloc_large(sizeof(*object_list), args->buffer_count);
673a394b
EA		 3252 if (exec_list == NULL || object_list == NULL) {
3253 DRM_ERROR("Failed to allocate exec or object list "
3254 "for %d buffers\n",
3255 args->buffer_count);
3256 ret = -ENOMEM;
3257 goto pre_mutex_err;
3258 }
3259 ret = copy_from_user(exec_list,
3260 (struct drm_i915_relocation_entry __user *)
3261 (uintptr_t) args->buffers_ptr,
3262 sizeof(*exec_list) * args->buffer_count);
3263 if (ret != 0) {
3264 DRM_ERROR("copy %d exec entries failed %d\n",
3265 args->buffer_count, ret);
3266 goto pre_mutex_err;
3267 }
3268
201361a5
EA		 3269 if (args->num_cliprects != 0) {
3270 cliprects = drm_calloc(args->num_cliprects, sizeof(*cliprects),
3271 DRM_MEM_DRIVER);
3272 if (cliprects == NULL)
3273 goto pre_mutex_err;
3274
3275 ret = copy_from_user(cliprects,
3276 (struct drm_clip_rect __user *)
3277 (uintptr_t) args->cliprects_ptr,
3278 sizeof(*cliprects) * args->num_cliprects);
3279 if (ret != 0) {
3280 DRM_ERROR("copy %d cliprects failed: %d\n",
3281 args->num_cliprects, ret);
3282 goto pre_mutex_err;
3283 }
3284 }
3285
40a5f0de
EA		 3286 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3287 &relocs);
3288 if (ret != 0)
3289 goto pre_mutex_err;
3290
673a394b
EA		 3291 mutex_lock(&dev->struct_mutex);
3292
3293 i915_verify_inactive(dev, __FILE__, __LINE__);
3294
3295 if (dev_priv->mm.wedged) {
3296 DRM_ERROR("Execbuf while wedged\n");
3297 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3298 ret = -EIO;
3299 goto pre_mutex_err;
673a394b
EA		 3300 }
3301
3302 if (dev_priv->mm.suspended) {
3303 DRM_ERROR("Execbuf while VT-switched.\n");
3304 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3305 ret = -EBUSY;
3306 goto pre_mutex_err;
673a394b
EA		 3307 }
3308
ac94a962 3309 /* Look up object handles */
673a394b
EA		 3310 for (i = 0; i < args->buffer_count; i++) {
3311 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3312 exec_list[i].handle);
3313 if (object_list[i] == NULL) {
3314 DRM_ERROR("Invalid object handle %d at index %d\n",
3315 exec_list[i].handle, i);
3316 ret = -EBADF;
3317 goto err;
3318 }
b70d11da
KH		 3319
3320 obj_priv = object_list[i]->driver_private;
3321 if (obj_priv->in_execbuffer) {
3322 DRM_ERROR("Object %p appears more than once in object list\n",
3323 object_list[i]);
3324 ret = -EBADF;
3325 goto err;
3326 }
3327 obj_priv->in_execbuffer = true;
ac94a962 3328 }
673a394b 3329
ac94a962
KP		 3330 /* Pin and relocate */
3331 for (pin_tries = 0; ; pin_tries++) {
3332 ret = 0;
40a5f0de
EA		 3333 reloc_index = 0;
3334
ac94a962
KP		 3335 for (i = 0; i < args->buffer_count; i++) {
3336 object_list[i]->pending_read_domains = 0;
3337 object_list[i]->pending_write_domain = 0;
3338 ret = i915_gem_object_pin_and_relocate(object_list[i],
3339 file_priv,
40a5f0de
EA		 3340 &exec_list[i],
3341 &relocs[reloc_index]);
ac94a962
KP		 3342 if (ret)
3343 break;
3344 pinned = i + 1;
40a5f0de 3345 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3346 }
3347 /* success */
3348 if (ret == 0)
3349 break;
3350
3351 /* error other than GTT full, or we've already tried again */
3352 if (ret != -ENOMEM || pin_tries >= 1) {
f1acec93
EA		 3353 if (ret != -ERESTARTSYS)
3354 DRM_ERROR("Failed to pin buffers %d\n", ret);
673a394b
EA		 3355 goto err;
3356 }
ac94a962
KP		 3357
3358 /* unpin all of our buffers */
3359 for (i = 0; i < pinned; i++)
3360 i915_gem_object_unpin(object_list[i]);
b1177636 3361 pinned = 0;
ac94a962
KP		 3362
3363 /* evict everyone we can from the aperture */
3364 ret = i915_gem_evict_everything(dev);
3365 if (ret)
3366 goto err;
673a394b
EA		 3367 }
3368
3369 /* Set the pending read domains for the batch buffer to COMMAND */
3370 batch_obj = object_list[args->buffer_count-1];
5f26a2c7
CW		 3371 if (batch_obj->pending_write_domain) {
3372 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3373 ret = -EINVAL;
3374 goto err;
3375 }
3376 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
673a394b 3377
83d60795
CW		 3378 /* Sanity check the batch buffer, prior to moving objects */
3379 exec_offset = exec_list[args->buffer_count - 1].offset;
3380 ret = i915_gem_check_execbuffer (args, exec_offset);
3381 if (ret != 0) {
3382 DRM_ERROR("execbuf with invalid offset/length\n");
3383 goto err;
3384 }
3385
673a394b
EA		 3386 i915_verify_inactive(dev, __FILE__, __LINE__);
3387
646f0f6e
KP		 3388 /* Zero the global flush/invalidate flags. These
3389 * will be modified as new domains are computed
3390 * for each object
3391 */
3392 dev->invalidate_domains = 0;
3393 dev->flush_domains = 0;
3394
673a394b
EA		 3395 for (i = 0; i < args->buffer_count; i++) {
3396 struct drm_gem_object *obj = object_list[i];
673a394b 3397
646f0f6e 3398 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3399 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3400 }
3401
3402 i915_verify_inactive(dev, __FILE__, __LINE__);
3403
646f0f6e
KP		 3404 if (dev->invalidate_domains | dev->flush_domains) {
3405#if WATCH_EXEC
3406 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3407 __func__,
3408 dev->invalidate_domains,
3409 dev->flush_domains);
3410#endif
3411 i915_gem_flush(dev,
3412 dev->invalidate_domains,
3413 dev->flush_domains);
3414 if (dev->flush_domains)
b962442e
EA		 3415 (void)i915_add_request(dev, file_priv,
3416 dev->flush_domains);
646f0f6e 3417 }
673a394b 3418
efbeed96
EA		 3419 for (i = 0; i < args->buffer_count; i++) {
3420 struct drm_gem_object *obj = object_list[i];
3421
3422 obj->write_domain = obj->pending_write_domain;
3423 }
3424
673a394b
EA		 3425 i915_verify_inactive(dev, __FILE__, __LINE__);
3426
3427#if WATCH_COHERENCY
3428 for (i = 0; i < args->buffer_count; i++) {
3429 i915_gem_object_check_coherency(object_list[i],
3430 exec_list[i].handle);
3431 }
3432#endif
3433
673a394b 3434#if WATCH_EXEC
6911a9b8 3435 i915_gem_dump_object(batch_obj,
673a394b
EA		 3436 args->batch_len,
3437 __func__,
3438 ~0);
3439#endif
3440
673a394b 3441 /* Exec the batchbuffer */
201361a5 3442 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
673a394b
EA		 3443 if (ret) {
3444 DRM_ERROR("dispatch failed %d\n", ret);
3445 goto err;
3446 }
3447
3448 /*
3449 * Ensure that the commands in the batch buffer are
3450 * finished before the interrupt fires
3451 */
3452 flush_domains = i915_retire_commands(dev);
3453
3454 i915_verify_inactive(dev, __FILE__, __LINE__);
3455
3456 /*
3457 * Get a seqno representing the execution of the current buffer,
3458 * which we can wait on. We would like to mitigate these interrupts,
3459 * likely by only creating seqnos occasionally (so that we have
3460 * *some* interrupts representing completion of buffers that we can
3461 * wait on when trying to clear up gtt space).
3462 */
b962442e 3463 seqno = i915_add_request(dev, file_priv, flush_domains);
673a394b 3464 BUG_ON(seqno == 0);
673a394b
EA		 3465 for (i = 0; i < args->buffer_count; i++) {
3466 struct drm_gem_object *obj = object_list[i];
673a394b 3467
ce44b0ea 3468 i915_gem_object_move_to_active(obj, seqno);
673a394b
EA		 3469#if WATCH_LRU
3470 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3471#endif
3472 }
3473#if WATCH_LRU
3474 i915_dump_lru(dev, __func__);
3475#endif
3476
3477 i915_verify_inactive(dev, __FILE__, __LINE__);
3478
673a394b 3479err:
aad87dff
JL		 3480 for (i = 0; i < pinned; i++)
3481 i915_gem_object_unpin(object_list[i]);
3482
b70d11da
KH		 3483 for (i = 0; i < args->buffer_count; i++) {
3484 if (object_list[i]) {
3485 obj_priv = object_list[i]->driver_private;
3486 obj_priv->in_execbuffer = false;
3487 }
aad87dff 3488 drm_gem_object_unreference(object_list[i]);
b70d11da 3489 }
673a394b 3490
673a394b
EA		 3491 mutex_unlock(&dev->struct_mutex);
3492
a35f2e2b
RD		 3493 if (!ret) {
3494 /* Copy the new buffer offsets back to the user's exec list. */
3495 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3496 (uintptr_t) args->buffers_ptr,
3497 exec_list,
3498 sizeof(*exec_list) * args->buffer_count);
2bc43b5c
FM		 3499 if (ret) {
3500 ret = -EFAULT;
a35f2e2b
RD		 3501 DRM_ERROR("failed to copy %d exec entries "
3502 "back to user (%d)\n",
3503 args->buffer_count, ret);
2bc43b5c 3504 }
a35f2e2b
RD		 3505 }
3506
40a5f0de
EA		 3507 /* Copy the updated relocations out regardless of current error
3508 * state. Failure to update the relocs would mean that the next
3509 * time userland calls execbuf, it would do so with presumed offset
3510 * state that didn't match the actual object state.
3511 */
3512 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3513 relocs);
3514 if (ret2 != 0) {
3515 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3516
3517 if (ret == 0)
3518 ret = ret2;
3519 }
3520
673a394b 3521pre_mutex_err:
8e7d2b2c
JB		 3522 drm_free_large(object_list);
3523 drm_free_large(exec_list);
201361a5
EA		 3524 drm_free(cliprects, sizeof(*cliprects) * args->num_cliprects,
3525 DRM_MEM_DRIVER);
673a394b
EA		 3526
3527 return ret;
3528}
3529
3530int
3531i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
3532{
3533 struct drm_device *dev = obj->dev;
3534 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3535 int ret;
3536
3537 i915_verify_inactive(dev, __FILE__, __LINE__);
3538 if (obj_priv->gtt_space == NULL) {
3539 ret = i915_gem_object_bind_to_gtt(obj, alignment);
3540 if (ret != 0) {
9bb2d6f9 3541 if (ret != -EBUSY && ret != -ERESTARTSYS)
0fce81e3 3542 DRM_ERROR("Failure to bind: %d\n", ret);
673a394b
EA		 3543 return ret;
3544 }
22c344e9
CW		 3545 }
3546 /*
3547 * Pre-965 chips need a fence register set up in order to
3548 * properly handle tiled surfaces.
3549 */
3550 if (!IS_I965G(dev) &&
3551 obj_priv->fence_reg == I915_FENCE_REG_NONE &&
3552 obj_priv->tiling_mode != I915_TILING_NONE) {
3553 ret = i915_gem_object_get_fence_reg(obj, true);
3554 if (ret != 0) {
3555 if (ret != -EBUSY && ret != -ERESTARTSYS)
3556 DRM_ERROR("Failure to install fence: %d\n",
3557 ret);
3558 return ret;
3559 }
673a394b
EA		 3560 }
3561 obj_priv->pin_count++;
3562
3563 /* If the object is not active and not pending a flush,
3564 * remove it from the inactive list
3565 */
3566 if (obj_priv->pin_count == 1) {
3567 atomic_inc(&dev->pin_count);
3568 atomic_add(obj->size, &dev->pin_memory);
3569 if (!obj_priv->active &&
21d509e3 3570 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
673a394b
EA		 3571 !list_empty(&obj_priv->list))
3572 list_del_init(&obj_priv->list);
3573 }
3574 i915_verify_inactive(dev, __FILE__, __LINE__);
3575
3576 return 0;
3577}
3578
3579void
3580i915_gem_object_unpin(struct drm_gem_object *obj)
3581{
3582 struct drm_device *dev = obj->dev;
3583 drm_i915_private_t *dev_priv = dev->dev_private;
3584 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3585
3586 i915_verify_inactive(dev, __FILE__, __LINE__);
3587 obj_priv->pin_count--;
3588 BUG_ON(obj_priv->pin_count < 0);
3589 BUG_ON(obj_priv->gtt_space == NULL);
3590
3591 /* If the object is no longer pinned, and is
3592 * neither active nor being flushed, then stick it on
3593 * the inactive list
3594 */
3595 if (obj_priv->pin_count == 0) {
3596 if (!obj_priv->active &&
21d509e3 3597 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
673a394b
EA		 3598 list_move_tail(&obj_priv->list,
3599 &dev_priv->mm.inactive_list);
3600 atomic_dec(&dev->pin_count);
3601 atomic_sub(obj->size, &dev->pin_memory);
3602 }
3603 i915_verify_inactive(dev, __FILE__, __LINE__);
3604}
3605
3606int
3607i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3608 struct drm_file *file_priv)
3609{
3610 struct drm_i915_gem_pin *args = data;
3611 struct drm_gem_object *obj;
3612 struct drm_i915_gem_object *obj_priv;
3613 int ret;
3614
3615 mutex_lock(&dev->struct_mutex);
3616
3617 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3618 if (obj == NULL) {
3619 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
3620 args->handle);
3621 mutex_unlock(&dev->struct_mutex);
3622 return -EBADF;
3623 }
3624 obj_priv = obj->driver_private;
3625
79e53945
JB		 3626 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
3627 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3628 args->handle);
96dec61d 3629 drm_gem_object_unreference(obj);
673a394b 3630 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 3631 return -EINVAL;
3632 }
3633
3634 obj_priv->user_pin_count++;
3635 obj_priv->pin_filp = file_priv;
3636 if (obj_priv->user_pin_count == 1) {
3637 ret = i915_gem_object_pin(obj, args->alignment);
3638 if (ret != 0) {
3639 drm_gem_object_unreference(obj);
3640 mutex_unlock(&dev->struct_mutex);
3641 return ret;
3642 }
673a394b
EA		 3643 }
3644
3645 /* XXX - flush the CPU caches for pinned objects
3646 * as the X server doesn't manage domains yet
3647 */
e47c68e9 3648 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 3649 args->offset = obj_priv->gtt_offset;
3650 drm_gem_object_unreference(obj);
3651 mutex_unlock(&dev->struct_mutex);
3652
3653 return 0;
3654}
3655
3656int
3657i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3658 struct drm_file *file_priv)
3659{
3660 struct drm_i915_gem_pin *args = data;
3661 struct drm_gem_object *obj;
79e53945 3662 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 3663
3664 mutex_lock(&dev->struct_mutex);
3665
3666 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3667 if (obj == NULL) {
3668 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
3669 args->handle);
3670 mutex_unlock(&dev->struct_mutex);
3671 return -EBADF;
3672 }
3673
79e53945
JB		 3674 obj_priv = obj->driver_private;
3675 if (obj_priv->pin_filp != file_priv) {
3676 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3677 args->handle);
3678 drm_gem_object_unreference(obj);
3679 mutex_unlock(&dev->struct_mutex);
3680 return -EINVAL;
3681 }
3682 obj_priv->user_pin_count--;
3683 if (obj_priv->user_pin_count == 0) {
3684 obj_priv->pin_filp = NULL;
3685 i915_gem_object_unpin(obj);
3686 }
673a394b
EA		 3687
3688 drm_gem_object_unreference(obj);
3689 mutex_unlock(&dev->struct_mutex);
3690 return 0;
3691}
3692
3693int
3694i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3695 struct drm_file *file_priv)
3696{
3697 struct drm_i915_gem_busy *args = data;
3698 struct drm_gem_object *obj;
3699 struct drm_i915_gem_object *obj_priv;
3700
673a394b
EA		 3701 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3702 if (obj == NULL) {
3703 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
3704 args->handle);
673a394b
EA		 3705 return -EBADF;
3706 }
3707
b1ce786c 3708 mutex_lock(&dev->struct_mutex);
f21289b3
EA		 3709 /* Update the active list for the hardware's current position.
3710 * Otherwise this only updates on a delayed timer or when irqs are
3711 * actually unmasked, and our working set ends up being larger than
3712 * required.
3713 */
3714 i915_gem_retire_requests(dev);
3715
673a394b 3716 obj_priv = obj->driver_private;
c4de0a5d
EA		 3717 /* Don't count being on the flushing list against the object being
3718 * done. Otherwise, a buffer left on the flushing list but not getting
3719 * flushed (because nobody's flushing that domain) won't ever return
3720 * unbusy and get reused by libdrm's bo cache. The other expected
3721 * consumer of this interface, OpenGL's occlusion queries, also specs
3722 * that the objects get unbusy "eventually" without any interference.
3723 */
3724 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
673a394b
EA		 3725
3726 drm_gem_object_unreference(obj);
3727 mutex_unlock(&dev->struct_mutex);
3728 return 0;
3729}
3730
3731int
3732i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3733 struct drm_file *file_priv)
3734{
3735 return i915_gem_ring_throttle(dev, file_priv);
3736}
3737
3738int i915_gem_init_object(struct drm_gem_object *obj)
3739{
3740 struct drm_i915_gem_object *obj_priv;
3741
3742 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
3743 if (obj_priv == NULL)
3744 return -ENOMEM;
3745
3746 /*
3747 * We've just allocated pages from the kernel,
3748 * so they've just been written by the CPU with
3749 * zeros. They'll need to be clflushed before we
3750 * use them with the GPU.
3751 */
3752 obj->write_domain = I915_GEM_DOMAIN_CPU;
3753 obj->read_domains = I915_GEM_DOMAIN_CPU;
3754
ba1eb1d8
KP		 3755 obj_priv->agp_type = AGP_USER_MEMORY;
3756
673a394b
EA		 3757 obj->driver_private = obj_priv;
3758 obj_priv->obj = obj;
de151cf6 3759 obj_priv->fence_reg = I915_FENCE_REG_NONE;
673a394b 3760 INIT_LIST_HEAD(&obj_priv->list);
de151cf6 3761
673a394b
EA		 3762 return 0;
3763}
3764
3765void i915_gem_free_object(struct drm_gem_object *obj)
3766{
de151cf6 3767 struct drm_device *dev = obj->dev;
673a394b
EA		 3768 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3769
3770 while (obj_priv->pin_count > 0)
3771 i915_gem_object_unpin(obj);
3772
71acb5eb
DA		 3773 if (obj_priv->phys_obj)
3774 i915_gem_detach_phys_object(dev, obj);
3775
673a394b
EA		 3776 i915_gem_object_unbind(obj);
3777
ab00b3e5 3778 i915_gem_free_mmap_offset(obj);
de151cf6 3779
673a394b 3780 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
280b713b 3781 kfree(obj_priv->bit_17);
673a394b
EA		 3782 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
3783}
3784
673a394b
EA		 3785/** Unbinds all objects that are on the given buffer list. */
3786static int
3787i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
3788{
3789 struct drm_gem_object *obj;
3790 struct drm_i915_gem_object *obj_priv;
3791 int ret;
3792
3793 while (!list_empty(head)) {
3794 obj_priv = list_first_entry(head,
3795 struct drm_i915_gem_object,
3796 list);
3797 obj = obj_priv->obj;
3798
3799 if (obj_priv->pin_count != 0) {
3800 DRM_ERROR("Pinned object in unbind list\n");
3801 mutex_unlock(&dev->struct_mutex);
3802 return -EINVAL;
3803 }
3804
3805 ret = i915_gem_object_unbind(obj);
3806 if (ret != 0) {
3807 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
3808 ret);
3809 mutex_unlock(&dev->struct_mutex);
3810 return ret;
3811 }
3812 }
3813
3814
3815 return 0;
3816}
3817
5669fcac 3818int
673a394b
EA		 3819i915_gem_idle(struct drm_device *dev)
3820{
3821 drm_i915_private_t *dev_priv = dev->dev_private;
3822 uint32_t seqno, cur_seqno, last_seqno;
3823 int stuck, ret;
3824
6dbe2772
KP		 3825 mutex_lock(&dev->struct_mutex);
3826
3827 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
3828 mutex_unlock(&dev->struct_mutex);
673a394b 3829 return 0;
6dbe2772 3830 }
673a394b
EA		 3831
3832 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3833 * We need to replace this with a semaphore, or something.
3834 */
3835 dev_priv->mm.suspended = 1;
3836
6dbe2772
KP		 3837 /* Cancel the retire work handler, wait for it to finish if running
3838 */
3839 mutex_unlock(&dev->struct_mutex);
3840 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3841 mutex_lock(&dev->struct_mutex);
3842
673a394b
EA		 3843 i915_kernel_lost_context(dev);
3844
3845 /* Flush the GPU along with all non-CPU write domains
3846 */
21d509e3
CW		 3847 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
3848 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
673a394b
EA		 3849
3850 if (seqno == 0) {
3851 mutex_unlock(&dev->struct_mutex);
3852 return -ENOMEM;
3853 }
3854
3855 dev_priv->mm.waiting_gem_seqno = seqno;
3856 last_seqno = 0;
3857 stuck = 0;
3858 for (;;) {
3859 cur_seqno = i915_get_gem_seqno(dev);
3860 if (i915_seqno_passed(cur_seqno, seqno))
3861 break;
3862 if (last_seqno == cur_seqno) {
3863 if (stuck++ > 100) {
3864 DRM_ERROR("hardware wedged\n");
3865 dev_priv->mm.wedged = 1;
3866 DRM_WAKEUP(&dev_priv->irq_queue);
3867 break;
3868 }
3869 }
3870 msleep(10);
3871 last_seqno = cur_seqno;
3872 }
3873 dev_priv->mm.waiting_gem_seqno = 0;
3874
3875 i915_gem_retire_requests(dev);
3876
5e118f41 3877 spin_lock(&dev_priv->mm.active_list_lock);
28dfe52a
EA		 3878 if (!dev_priv->mm.wedged) {
3879 /* Active and flushing should now be empty as we've
3880 * waited for a sequence higher than any pending execbuffer
3881 */
3882 WARN_ON(!list_empty(&dev_priv->mm.active_list));
3883 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
3884 /* Request should now be empty as we've also waited
3885 * for the last request in the list
3886 */
3887 WARN_ON(!list_empty(&dev_priv->mm.request_list));
3888 }
673a394b 3889
28dfe52a
EA		 3890 /* Empty the active and flushing lists to inactive. If there's
3891 * anything left at this point, it means that we're wedged and
3892 * nothing good's going to happen by leaving them there. So strip
3893 * the GPU domains and just stuff them onto inactive.
673a394b 3894 */
28dfe52a
EA		 3895 while (!list_empty(&dev_priv->mm.active_list)) {
3896 struct drm_i915_gem_object *obj_priv;
673a394b 3897
28dfe52a
EA		 3898 obj_priv = list_first_entry(&dev_priv->mm.active_list,
3899 struct drm_i915_gem_object,
3900 list);
3901 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3902 i915_gem_object_move_to_inactive(obj_priv->obj);
3903 }
5e118f41 3904 spin_unlock(&dev_priv->mm.active_list_lock);
28dfe52a
EA		 3905
3906 while (!list_empty(&dev_priv->mm.flushing_list)) {
3907 struct drm_i915_gem_object *obj_priv;
3908
151903d5 3909 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
28dfe52a
EA		 3910 struct drm_i915_gem_object,
3911 list);
3912 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3913 i915_gem_object_move_to_inactive(obj_priv->obj);
3914 }
3915
3916
3917 /* Move all inactive buffers out of the GTT. */
673a394b 3918 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
28dfe52a 3919 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
6dbe2772
KP		 3920 if (ret) {
3921 mutex_unlock(&dev->struct_mutex);
673a394b 3922 return ret;
6dbe2772 3923 }
673a394b 3924
6dbe2772
KP		 3925 i915_gem_cleanup_ringbuffer(dev);
3926 mutex_unlock(&dev->struct_mutex);
3927
673a394b
EA		 3928 return 0;
3929}
3930
3931static int
3932i915_gem_init_hws(struct drm_device *dev)
3933{
3934 drm_i915_private_t *dev_priv = dev->dev_private;
3935 struct drm_gem_object *obj;
3936 struct drm_i915_gem_object *obj_priv;
3937 int ret;
3938
3939 /* If we need a physical address for the status page, it's already
3940 * initialized at driver load time.
3941 */
3942 if (!I915_NEED_GFX_HWS(dev))
3943 return 0;
3944
3945 obj = drm_gem_object_alloc(dev, 4096);
3946 if (obj == NULL) {
3947 DRM_ERROR("Failed to allocate status page\n");
3948 return -ENOMEM;
3949 }
3950 obj_priv = obj->driver_private;
ba1eb1d8 3951 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
673a394b
EA		 3952
3953 ret = i915_gem_object_pin(obj, 4096);
3954 if (ret != 0) {
3955 drm_gem_object_unreference(obj);
3956 return ret;
3957 }
3958
3959 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
673a394b 3960
856fa198 3961 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
ba1eb1d8 3962 if (dev_priv->hw_status_page == NULL) {
673a394b
EA		 3963 DRM_ERROR("Failed to map status page.\n");
3964 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
3eb2ee77 3965 i915_gem_object_unpin(obj);
673a394b
EA		 3966 drm_gem_object_unreference(obj);
3967 return -EINVAL;
3968 }
3969 dev_priv->hws_obj = obj;
673a394b
EA		 3970 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
3971 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
ba1eb1d8 3972 I915_READ(HWS_PGA); /* posting read */
673a394b
EA		 3973 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
3974
3975 return 0;
3976}
3977
85a7bb98
CW		 3978static void
3979i915_gem_cleanup_hws(struct drm_device *dev)
3980{
3981 drm_i915_private_t *dev_priv = dev->dev_private;
bab2d1f6
CW		 3982 struct drm_gem_object *obj;
3983 struct drm_i915_gem_object *obj_priv;
85a7bb98
CW		 3984
3985 if (dev_priv->hws_obj == NULL)
3986 return;
3987
bab2d1f6
CW		 3988 obj = dev_priv->hws_obj;
3989 obj_priv = obj->driver_private;
3990
856fa198 3991 kunmap(obj_priv->pages[0]);
85a7bb98
CW		 3992 i915_gem_object_unpin(obj);
3993 drm_gem_object_unreference(obj);
3994 dev_priv->hws_obj = NULL;
bab2d1f6 3995
85a7bb98
CW		 3996 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
3997 dev_priv->hw_status_page = NULL;
3998
3999 /* Write high address into HWS_PGA when disabling. */
4000 I915_WRITE(HWS_PGA, 0x1ffff000);
4001}
4002
79e53945 4003int
673a394b
EA		 4004i915_gem_init_ringbuffer(struct drm_device *dev)
4005{
4006 drm_i915_private_t *dev_priv = dev->dev_private;
4007 struct drm_gem_object *obj;
4008 struct drm_i915_gem_object *obj_priv;
79e53945 4009 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
673a394b 4010 int ret;
50aa253d 4011 u32 head;
673a394b
EA		 4012
4013 ret = i915_gem_init_hws(dev);
4014 if (ret != 0)
4015 return ret;
4016
4017 obj = drm_gem_object_alloc(dev, 128 * 1024);
4018 if (obj == NULL) {
4019 DRM_ERROR("Failed to allocate ringbuffer\n");
85a7bb98 4020 i915_gem_cleanup_hws(dev);
673a394b
EA		 4021 return -ENOMEM;
4022 }
4023 obj_priv = obj->driver_private;
4024
4025 ret = i915_gem_object_pin(obj, 4096);
4026 if (ret != 0) {
4027 drm_gem_object_unreference(obj);
85a7bb98 4028 i915_gem_cleanup_hws(dev);
673a394b
EA		 4029 return ret;
4030 }
4031
4032 /* Set up the kernel mapping for the ring. */
79e53945
JB		 4033 ring->Size = obj->size;
4034 ring->tail_mask = obj->size - 1;
673a394b 4035
79e53945
JB		 4036 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4037 ring->map.size = obj->size;
4038 ring->map.type = 0;
4039 ring->map.flags = 0;
4040 ring->map.mtrr = 0;
673a394b 4041
79e53945
JB		 4042 drm_core_ioremap_wc(&ring->map, dev);
4043 if (ring->map.handle == NULL) {
673a394b
EA		 4044 DRM_ERROR("Failed to map ringbuffer.\n");
4045 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
47ed185a 4046 i915_gem_object_unpin(obj);
673a394b 4047 drm_gem_object_unreference(obj);
85a7bb98 4048 i915_gem_cleanup_hws(dev);
673a394b
EA		 4049 return -EINVAL;
4050 }
79e53945
JB		 4051 ring->ring_obj = obj;
4052 ring->virtual_start = ring->map.handle;
673a394b
EA		 4053
4054 /* Stop the ring if it's running. */
4055 I915_WRITE(PRB0_CTL, 0);
673a394b 4056 I915_WRITE(PRB0_TAIL, 0);
50aa253d 4057 I915_WRITE(PRB0_HEAD, 0);
673a394b
EA		 4058
4059 /* Initialize the ring. */
4060 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
50aa253d
KP		 4061 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4062
4063 /* G45 ring initialization fails to reset head to zero */
4064 if (head != 0) {
4065 DRM_ERROR("Ring head not reset to zero "
4066 "ctl %08x head %08x tail %08x start %08x\n",
4067 I915_READ(PRB0_CTL),
4068 I915_READ(PRB0_HEAD),
4069 I915_READ(PRB0_TAIL),
4070 I915_READ(PRB0_START));
4071 I915_WRITE(PRB0_HEAD, 0);
4072
4073 DRM_ERROR("Ring head forced to zero "
4074 "ctl %08x head %08x tail %08x start %08x\n",
4075 I915_READ(PRB0_CTL),
4076 I915_READ(PRB0_HEAD),
4077 I915_READ(PRB0_TAIL),
4078 I915_READ(PRB0_START));
4079 }
4080
673a394b
EA		 4081 I915_WRITE(PRB0_CTL,
4082 ((obj->size - 4096) & RING_NR_PAGES) |
4083 RING_NO_REPORT |
4084 RING_VALID);
4085
50aa253d
KP		 4086 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4087
4088 /* If the head is still not zero, the ring is dead */
4089 if (head != 0) {
4090 DRM_ERROR("Ring initialization failed "
4091 "ctl %08x head %08x tail %08x start %08x\n",
4092 I915_READ(PRB0_CTL),
4093 I915_READ(PRB0_HEAD),
4094 I915_READ(PRB0_TAIL),
4095 I915_READ(PRB0_START));
4096 return -EIO;
4097 }
4098
673a394b 4099 /* Update our cache of the ring state */
79e53945
JB		 4100 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4101 i915_kernel_lost_context(dev);
4102 else {
4103 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4104 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4105 ring->space = ring->head - (ring->tail + 8);
4106 if (ring->space < 0)
4107 ring->space += ring->Size;
4108 }
673a394b
EA		 4109
4110 return 0;
4111}
4112
79e53945 4113void
673a394b
EA		 4114i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4115{
4116 drm_i915_private_t *dev_priv = dev->dev_private;
4117
4118 if (dev_priv->ring.ring_obj == NULL)
4119 return;
4120
4121 drm_core_ioremapfree(&dev_priv->ring.map, dev);
4122
4123 i915_gem_object_unpin(dev_priv->ring.ring_obj);
4124 drm_gem_object_unreference(dev_priv->ring.ring_obj);
4125 dev_priv->ring.ring_obj = NULL;
4126 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4127
85a7bb98 4128 i915_gem_cleanup_hws(dev);
673a394b
EA		 4129}
4130
4131int
4132i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4133 struct drm_file *file_priv)
4134{
4135 drm_i915_private_t *dev_priv = dev->dev_private;
4136 int ret;
4137
79e53945
JB		 4138 if (drm_core_check_feature(dev, DRIVER_MODESET))
4139 return 0;
4140
673a394b
EA		 4141 if (dev_priv->mm.wedged) {
4142 DRM_ERROR("Reenabling wedged hardware, good luck\n");
4143 dev_priv->mm.wedged = 0;
4144 }
4145
673a394b 4146 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 4147 dev_priv->mm.suspended = 0;
4148
4149 ret = i915_gem_init_ringbuffer(dev);
d816f6ac
WF		 4150 if (ret != 0) {
4151 mutex_unlock(&dev->struct_mutex);
9bb2d6f9 4152 return ret;
d816f6ac 4153 }
9bb2d6f9 4154
5e118f41 4155 spin_lock(&dev_priv->mm.active_list_lock);
673a394b 4156 BUG_ON(!list_empty(&dev_priv->mm.active_list));
5e118f41
CW		 4157 spin_unlock(&dev_priv->mm.active_list_lock);
4158
673a394b
EA		 4159 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4160 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4161 BUG_ON(!list_empty(&dev_priv->mm.request_list));
673a394b 4162 mutex_unlock(&dev->struct_mutex);
dbb19d30
KH		 4163
4164 drm_irq_install(dev);
4165
673a394b
EA		 4166 return 0;
4167}
4168
4169int
4170i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4171 struct drm_file *file_priv)
4172{
4173 int ret;
4174
79e53945
JB		 4175 if (drm_core_check_feature(dev, DRIVER_MODESET))
4176 return 0;
4177
673a394b 4178 ret = i915_gem_idle(dev);
dbb19d30
KH		 4179 drm_irq_uninstall(dev);
4180
6dbe2772 4181 return ret;
673a394b
EA		 4182}
4183
4184void
4185i915_gem_lastclose(struct drm_device *dev)
4186{
4187 int ret;
673a394b 4188
e806b495
EA		 4189 if (drm_core_check_feature(dev, DRIVER_MODESET))
4190 return;
4191
6dbe2772
KP		 4192 ret = i915_gem_idle(dev);
4193 if (ret)
4194 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 4195}
4196
4197void
4198i915_gem_load(struct drm_device *dev)
4199{
4200 drm_i915_private_t *dev_priv = dev->dev_private;
4201
5e118f41 4202 spin_lock_init(&dev_priv->mm.active_list_lock);
673a394b
EA		 4203 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4204 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
4205 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4206 INIT_LIST_HEAD(&dev_priv->mm.request_list);
4207 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4208 i915_gem_retire_work_handler);
4209 dev_priv->mm.next_gem_seqno = 1;
4210
de151cf6
JB		 4211 /* Old X drivers will take 0-2 for front, back, depth buffers */
4212 dev_priv->fence_reg_start = 3;
4213
0f973f27 4214 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4215 dev_priv->num_fence_regs = 16;
4216 else
4217 dev_priv->num_fence_regs = 8;
4218
673a394b
EA		 4219 i915_gem_detect_bit_6_swizzle(dev);
4220}
71acb5eb
DA		 4221
4222/*
4223 * Create a physically contiguous memory object for this object
4224 * e.g. for cursor + overlay regs
4225 */
4226int i915_gem_init_phys_object(struct drm_device *dev,
4227 int id, int size)
4228{
4229 drm_i915_private_t *dev_priv = dev->dev_private;
4230 struct drm_i915_gem_phys_object *phys_obj;
4231 int ret;
4232
4233 if (dev_priv->mm.phys_objs[id - 1] || !size)
4234 return 0;
4235
4236 phys_obj = drm_calloc(1, sizeof(struct drm_i915_gem_phys_object), DRM_MEM_DRIVER);
4237 if (!phys_obj)
4238 return -ENOMEM;
4239
4240 phys_obj->id = id;
4241
4242 phys_obj->handle = drm_pci_alloc(dev, size, 0, 0xffffffff);
4243 if (!phys_obj->handle) {
4244 ret = -ENOMEM;
4245 goto kfree_obj;
4246 }
4247#ifdef CONFIG_X86
4248 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4249#endif
4250
4251 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4252
4253 return 0;
4254kfree_obj:
4255 drm_free(phys_obj, sizeof(struct drm_i915_gem_phys_object), DRM_MEM_DRIVER);
4256 return ret;
4257}
4258
4259void i915_gem_free_phys_object(struct drm_device *dev, int id)
4260{
4261 drm_i915_private_t *dev_priv = dev->dev_private;
4262 struct drm_i915_gem_phys_object *phys_obj;
4263
4264 if (!dev_priv->mm.phys_objs[id - 1])
4265 return;
4266
4267 phys_obj = dev_priv->mm.phys_objs[id - 1];
4268 if (phys_obj->cur_obj) {
4269 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4270 }
4271
4272#ifdef CONFIG_X86
4273 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4274#endif
4275 drm_pci_free(dev, phys_obj->handle);
4276 kfree(phys_obj);
4277 dev_priv->mm.phys_objs[id - 1] = NULL;
4278}
4279
4280void i915_gem_free_all_phys_object(struct drm_device *dev)
4281{
4282 int i;
4283
260883c8 4284 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4285 i915_gem_free_phys_object(dev, i);
4286}
4287
4288void i915_gem_detach_phys_object(struct drm_device *dev,
4289 struct drm_gem_object *obj)
4290{
4291 struct drm_i915_gem_object *obj_priv;
4292 int i;
4293 int ret;
4294 int page_count;
4295
4296 obj_priv = obj->driver_private;
4297 if (!obj_priv->phys_obj)
4298 return;
4299
856fa198 4300 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4301 if (ret)
4302 goto out;
4303
4304 page_count = obj->size / PAGE_SIZE;
4305
4306 for (i = 0; i < page_count; i++) {
856fa198 4307 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4308 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4309
4310 memcpy(dst, src, PAGE_SIZE);
4311 kunmap_atomic(dst, KM_USER0);
4312 }
856fa198 4313 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb
DA		 4314 drm_agp_chipset_flush(dev);
4315out:
4316 obj_priv->phys_obj->cur_obj = NULL;
4317 obj_priv->phys_obj = NULL;
4318}
4319
4320int
4321i915_gem_attach_phys_object(struct drm_device *dev,
4322 struct drm_gem_object *obj, int id)
4323{
4324 drm_i915_private_t *dev_priv = dev->dev_private;
4325 struct drm_i915_gem_object *obj_priv;
4326 int ret = 0;
4327 int page_count;
4328 int i;
4329
4330 if (id > I915_MAX_PHYS_OBJECT)
4331 return -EINVAL;
4332
4333 obj_priv = obj->driver_private;
4334
4335 if (obj_priv->phys_obj) {
4336 if (obj_priv->phys_obj->id == id)
4337 return 0;
4338 i915_gem_detach_phys_object(dev, obj);
4339 }
4340
4341
4342 /* create a new object */
4343 if (!dev_priv->mm.phys_objs[id - 1]) {
4344 ret = i915_gem_init_phys_object(dev, id,
4345 obj->size);
4346 if (ret) {
aeb565df 4347 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4348 goto out;
4349 }
4350 }
4351
4352 /* bind to the object */
4353 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4354 obj_priv->phys_obj->cur_obj = obj;
4355
856fa198 4356 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4357 if (ret) {
4358 DRM_ERROR("failed to get page list\n");
4359 goto out;
4360 }
4361
4362 page_count = obj->size / PAGE_SIZE;
4363
4364 for (i = 0; i < page_count; i++) {
856fa198 4365 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4366 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4367
4368 memcpy(dst, src, PAGE_SIZE);
4369 kunmap_atomic(src, KM_USER0);
4370 }
4371
4372 return 0;
4373out:
4374 return ret;
4375}
4376
4377static int
4378i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4379 struct drm_i915_gem_pwrite *args,
4380 struct drm_file *file_priv)
4381{
4382 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4383 void *obj_addr;
4384 int ret;
4385 char __user *user_data;
4386
4387 user_data = (char __user *) (uintptr_t) args->data_ptr;
4388 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4389
e08fb4f6 4390 DRM_DEBUG("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4391 ret = copy_from_user(obj_addr, user_data, args->size);
4392 if (ret)
4393 return -EFAULT;
4394
4395 drm_agp_chipset_flush(dev);
4396 return 0;
4397}
b962442e
EA		 4398
4399void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4400{
4401 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4402
4403 /* Clean up our request list when the client is going away, so that
4404 * later retire_requests won't dereference our soon-to-be-gone
4405 * file_priv.
4406 */
4407 mutex_lock(&dev->struct_mutex);
4408 while (!list_empty(&i915_file_priv->mm.request_list))
4409 list_del_init(i915_file_priv->mm.request_list.next);
4410 mutex_unlock(&dev->struct_mutex);
4411}
Clone of davem's net-next-2.6 tree