]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/gpu/drm/i915/i915_gem.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
drm: merge Linux master into HEAD
[net-next-2.6.git] / drivers / gpu / drm / i915 / i915_gem.c
CommitLineData
673a394b
EA		 1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32#include <linux/swap.h>
79e53945 33#include <linux/pci.h>
673a394b 34
28dfe52a
EA		 35#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
36
e47c68e9
EA		 37static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
38static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
39static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
e47c68e9
EA		 40static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
41 int write);
42static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
43 uint64_t offset,
44 uint64_t size);
45static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
856fa198
EA		 46static int i915_gem_object_get_pages(struct drm_gem_object *obj);
47static void i915_gem_object_put_pages(struct drm_gem_object *obj);
673a394b 48static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
de151cf6
JB		 49static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
50 unsigned alignment);
0f973f27 51static int i915_gem_object_get_fence_reg(struct drm_gem_object *obj, bool write);
de151cf6
JB		 52static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
53static int i915_gem_evict_something(struct drm_device *dev);
71acb5eb
DA		 54static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
55 struct drm_i915_gem_pwrite *args,
56 struct drm_file *file_priv);
673a394b 57
79e53945
JB		 58int i915_gem_do_init(struct drm_device *dev, unsigned long start,
59 unsigned long end)
673a394b
EA		 60{
61 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 62
79e53945
JB		 63 if (start >= end ||
64 (start & (PAGE_SIZE - 1)) != 0 ||
65 (end & (PAGE_SIZE - 1)) != 0) {
673a394b
EA		 66 return -EINVAL;
67 }
68
79e53945
JB		 69 drm_mm_init(&dev_priv->mm.gtt_space, start,
70 end - start);
673a394b 71
79e53945
JB		 72 dev->gtt_total = (uint32_t) (end - start);
73
74 return 0;
75}
673a394b 76
79e53945
JB		 77int
78i915_gem_init_ioctl(struct drm_device *dev, void *data,
79 struct drm_file *file_priv)
80{
81 struct drm_i915_gem_init *args = data;
82 int ret;
83
84 mutex_lock(&dev->struct_mutex);
85 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
673a394b
EA		 86 mutex_unlock(&dev->struct_mutex);
87
79e53945 88 return ret;
673a394b
EA		 89}
90
5a125c3c
EA		 91int
92i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
93 struct drm_file *file_priv)
94{
5a125c3c 95 struct drm_i915_gem_get_aperture *args = data;
5a125c3c
EA		 96
97 if (!(dev->driver->driver_features & DRIVER_GEM))
98 return -ENODEV;
99
100 args->aper_size = dev->gtt_total;
2678d9d6
KP		 101 args->aper_available_size = (args->aper_size -
102 atomic_read(&dev->pin_memory));
5a125c3c
EA		 103
104 return 0;
105}
106
673a394b
EA		 107
108/**
109 * Creates a new mm object and returns a handle to it.
110 */
111int
112i915_gem_create_ioctl(struct drm_device *dev, void *data,
113 struct drm_file *file_priv)
114{
115 struct drm_i915_gem_create *args = data;
116 struct drm_gem_object *obj;
117 int handle, ret;
118
119 args->size = roundup(args->size, PAGE_SIZE);
120
121 /* Allocate the new object */
122 obj = drm_gem_object_alloc(dev, args->size);
123 if (obj == NULL)
124 return -ENOMEM;
125
126 ret = drm_gem_handle_create(file_priv, obj, &handle);
127 mutex_lock(&dev->struct_mutex);
128 drm_gem_object_handle_unreference(obj);
129 mutex_unlock(&dev->struct_mutex);
130
131 if (ret)
132 return ret;
133
134 args->handle = handle;
135
136 return 0;
137}
138
eb01459f
EA		 139static inline int
140fast_shmem_read(struct page **pages,
141 loff_t page_base, int page_offset,
142 char __user *data,
143 int length)
144{
145 char __iomem *vaddr;
146 int ret;
147
148 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
149 if (vaddr == NULL)
150 return -ENOMEM;
151 ret = __copy_to_user_inatomic(data, vaddr + page_offset, length);
152 kunmap_atomic(vaddr, KM_USER0);
153
154 return ret;
155}
156
40123c1f
EA		 157static inline int
158slow_shmem_copy(struct page *dst_page,
159 int dst_offset,
160 struct page *src_page,
161 int src_offset,
162 int length)
163{
164 char *dst_vaddr, *src_vaddr;
165
166 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
167 if (dst_vaddr == NULL)
168 return -ENOMEM;
169
170 src_vaddr = kmap_atomic(src_page, KM_USER1);
171 if (src_vaddr == NULL) {
172 kunmap_atomic(dst_vaddr, KM_USER0);
173 return -ENOMEM;
174 }
175
176 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
177
178 kunmap_atomic(src_vaddr, KM_USER1);
179 kunmap_atomic(dst_vaddr, KM_USER0);
180
181 return 0;
182}
183
eb01459f
EA		 184/**
185 * This is the fast shmem pread path, which attempts to copy_from_user directly
186 * from the backing pages of the object to the user's address space. On a
187 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
188 */
189static int
190i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
191 struct drm_i915_gem_pread *args,
192 struct drm_file *file_priv)
193{
194 struct drm_i915_gem_object *obj_priv = obj->driver_private;
195 ssize_t remain;
196 loff_t offset, page_base;
197 char __user *user_data;
198 int page_offset, page_length;
199 int ret;
200
201 user_data = (char __user *) (uintptr_t) args->data_ptr;
202 remain = args->size;
203
204 mutex_lock(&dev->struct_mutex);
205
206 ret = i915_gem_object_get_pages(obj);
207 if (ret != 0)
208 goto fail_unlock;
209
210 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
211 args->size);
212 if (ret != 0)
213 goto fail_put_pages;
214
215 obj_priv = obj->driver_private;
216 offset = args->offset;
217
218 while (remain > 0) {
219 /* Operation in this page
220 *
221 * page_base = page offset within aperture
222 * page_offset = offset within page
223 * page_length = bytes to copy for this page
224 */
225 page_base = (offset & ~(PAGE_SIZE-1));
226 page_offset = offset & (PAGE_SIZE-1);
227 page_length = remain;
228 if ((page_offset + remain) > PAGE_SIZE)
229 page_length = PAGE_SIZE - page_offset;
230
231 ret = fast_shmem_read(obj_priv->pages,
232 page_base, page_offset,
233 user_data, page_length);
234 if (ret)
235 goto fail_put_pages;
236
237 remain -= page_length;
238 user_data += page_length;
239 offset += page_length;
240 }
241
242fail_put_pages:
243 i915_gem_object_put_pages(obj);
244fail_unlock:
245 mutex_unlock(&dev->struct_mutex);
246
247 return ret;
248}
249
250/**
251 * This is the fallback shmem pread path, which allocates temporary storage
252 * in kernel space to copy_to_user into outside of the struct_mutex, so we
253 * can copy out of the object's backing pages while holding the struct mutex
254 * and not take page faults.
255 */
256static int
257i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
258 struct drm_i915_gem_pread *args,
259 struct drm_file *file_priv)
260{
261 struct drm_i915_gem_object *obj_priv = obj->driver_private;
262 struct mm_struct *mm = current->mm;
263 struct page **user_pages;
264 ssize_t remain;
265 loff_t offset, pinned_pages, i;
266 loff_t first_data_page, last_data_page, num_pages;
267 int shmem_page_index, shmem_page_offset;
268 int data_page_index, data_page_offset;
269 int page_length;
270 int ret;
271 uint64_t data_ptr = args->data_ptr;
272
273 remain = args->size;
274
275 /* Pin the user pages containing the data. We can't fault while
276 * holding the struct mutex, yet we want to hold it while
277 * dereferencing the user data.
278 */
279 first_data_page = data_ptr / PAGE_SIZE;
280 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
281 num_pages = last_data_page - first_data_page + 1;
282
283 user_pages = kcalloc(num_pages, sizeof(struct page *), GFP_KERNEL);
284 if (user_pages == NULL)
285 return -ENOMEM;
286
287 down_read(&mm->mmap_sem);
288 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
289 num_pages, 0, 0, user_pages, NULL);
290 up_read(&mm->mmap_sem);
291 if (pinned_pages < num_pages) {
292 ret = -EFAULT;
293 goto fail_put_user_pages;
294 }
295
296 mutex_lock(&dev->struct_mutex);
297
298 ret = i915_gem_object_get_pages(obj);
299 if (ret != 0)
300 goto fail_unlock;
301
302 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
303 args->size);
304 if (ret != 0)
305 goto fail_put_pages;
306
307 obj_priv = obj->driver_private;
308 offset = args->offset;
309
310 while (remain > 0) {
311 /* Operation in this page
312 *
313 * shmem_page_index = page number within shmem file
314 * shmem_page_offset = offset within page in shmem file
315 * data_page_index = page number in get_user_pages return
316 * data_page_offset = offset with data_page_index page.
317 * page_length = bytes to copy for this page
318 */
319 shmem_page_index = offset / PAGE_SIZE;
320 shmem_page_offset = offset & ~PAGE_MASK;
321 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
322 data_page_offset = data_ptr & ~PAGE_MASK;
323
324 page_length = remain;
325 if ((shmem_page_offset + page_length) > PAGE_SIZE)
326 page_length = PAGE_SIZE - shmem_page_offset;
327 if ((data_page_offset + page_length) > PAGE_SIZE)
328 page_length = PAGE_SIZE - data_page_offset;
329
330 ret = slow_shmem_copy(user_pages[data_page_index],
331 data_page_offset,
332 obj_priv->pages[shmem_page_index],
333 shmem_page_offset,
334 page_length);
335 if (ret)
336 goto fail_put_pages;
337
338 remain -= page_length;
339 data_ptr += page_length;
340 offset += page_length;
341 }
342
343fail_put_pages:
344 i915_gem_object_put_pages(obj);
345fail_unlock:
346 mutex_unlock(&dev->struct_mutex);
347fail_put_user_pages:
348 for (i = 0; i < pinned_pages; i++) {
349 SetPageDirty(user_pages[i]);
350 page_cache_release(user_pages[i]);
351 }
352 kfree(user_pages);
353
354 return ret;
355}
356
673a394b
EA		 357/**
358 * Reads data from the object referenced by handle.
359 *
360 * On error, the contents of *data are undefined.
361 */
362int
363i915_gem_pread_ioctl(struct drm_device *dev, void *data,
364 struct drm_file *file_priv)
365{
366 struct drm_i915_gem_pread *args = data;
367 struct drm_gem_object *obj;
368 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 369 int ret;
370
371 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
372 if (obj == NULL)
373 return -EBADF;
374 obj_priv = obj->driver_private;
375
376 /* Bounds check source.
377 *
378 * XXX: This could use review for overflow issues...
379 */
380 if (args->offset > obj->size || args->size > obj->size ||
381 args->offset + args->size > obj->size) {
382 drm_gem_object_unreference(obj);
383 return -EINVAL;
384 }
385
eb01459f
EA		 386 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
387 if (ret != 0)
388 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
673a394b
EA		 389
390 drm_gem_object_unreference(obj);
673a394b 391
eb01459f 392 return ret;
673a394b
EA		 393}
394
0839ccb8
KP		 395/* This is the fast write path which cannot handle
396 * page faults in the source data
9b7530cc 397 */
0839ccb8
KP		 398
399static inline int
400fast_user_write(struct io_mapping *mapping,
401 loff_t page_base, int page_offset,
402 char __user *user_data,
403 int length)
9b7530cc 404{
9b7530cc 405 char *vaddr_atomic;
0839ccb8 406 unsigned long unwritten;
9b7530cc 407
0839ccb8
KP		 408 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
409 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
410 user_data, length);
411 io_mapping_unmap_atomic(vaddr_atomic);
412 if (unwritten)
413 return -EFAULT;
414 return 0;
415}
416
417/* Here's the write path which can sleep for
418 * page faults
419 */
420
421static inline int
3de09aa3
EA		 422slow_kernel_write(struct io_mapping *mapping,
423 loff_t gtt_base, int gtt_offset,
424 struct page *user_page, int user_offset,
425 int length)
0839ccb8 426{
3de09aa3 427 char *src_vaddr, *dst_vaddr;
0839ccb8
KP		 428 unsigned long unwritten;
429
3de09aa3
EA		 430 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
431 src_vaddr = kmap_atomic(user_page, KM_USER1);
432 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
433 src_vaddr + user_offset,
434 length);
435 kunmap_atomic(src_vaddr, KM_USER1);
436 io_mapping_unmap_atomic(dst_vaddr);
0839ccb8
KP		 437 if (unwritten)
438 return -EFAULT;
9b7530cc 439 return 0;
9b7530cc
LT		 440}
441
40123c1f
EA		 442static inline int
443fast_shmem_write(struct page **pages,
444 loff_t page_base, int page_offset,
445 char __user *data,
446 int length)
447{
448 char __iomem *vaddr;
449
450 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
451 if (vaddr == NULL)
452 return -ENOMEM;
453 __copy_from_user_inatomic(vaddr + page_offset, data, length);
454 kunmap_atomic(vaddr, KM_USER0);
455
456 return 0;
457}
458
3de09aa3
EA		 459/**
460 * This is the fast pwrite path, where we copy the data directly from the
461 * user into the GTT, uncached.
462 */
673a394b 463static int
3de09aa3
EA		 464i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
465 struct drm_i915_gem_pwrite *args,
466 struct drm_file *file_priv)
673a394b
EA		 467{
468 struct drm_i915_gem_object *obj_priv = obj->driver_private;
0839ccb8 469 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b 470 ssize_t remain;
0839ccb8 471 loff_t offset, page_base;
673a394b 472 char __user *user_data;
0839ccb8
KP		 473 int page_offset, page_length;
474 int ret;
673a394b
EA		 475
476 user_data = (char __user *) (uintptr_t) args->data_ptr;
477 remain = args->size;
478 if (!access_ok(VERIFY_READ, user_data, remain))
479 return -EFAULT;
480
481
482 mutex_lock(&dev->struct_mutex);
483 ret = i915_gem_object_pin(obj, 0);
484 if (ret) {
485 mutex_unlock(&dev->struct_mutex);
486 return ret;
487 }
2ef7eeaa 488 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
673a394b
EA		 489 if (ret)
490 goto fail;
491
492 obj_priv = obj->driver_private;
493 offset = obj_priv->gtt_offset + args->offset;
673a394b
EA		 494
495 while (remain > 0) {
496 /* Operation in this page
497 *
0839ccb8
KP		 498 * page_base = page offset within aperture
499 * page_offset = offset within page
500 * page_length = bytes to copy for this page
673a394b 501 */
0839ccb8
KP		 502 page_base = (offset & ~(PAGE_SIZE-1));
503 page_offset = offset & (PAGE_SIZE-1);
504 page_length = remain;
505 if ((page_offset + remain) > PAGE_SIZE)
506 page_length = PAGE_SIZE - page_offset;
507
508 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
509 page_offset, user_data, page_length);
510
511 /* If we get a fault while copying data, then (presumably) our
3de09aa3
EA		 512 * source page isn't available. Return the error and we'll
513 * retry in the slow path.
0839ccb8 514 */
3de09aa3
EA		 515 if (ret)
516 goto fail;
673a394b 517
0839ccb8
KP		 518 remain -= page_length;
519 user_data += page_length;
520 offset += page_length;
673a394b 521 }
673a394b
EA		 522
523fail:
524 i915_gem_object_unpin(obj);
525 mutex_unlock(&dev->struct_mutex);
526
527 return ret;
528}
529
3de09aa3
EA		 530/**
531 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
532 * the memory and maps it using kmap_atomic for copying.
533 *
534 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
535 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
536 */
3043c60c 537static int
3de09aa3
EA		 538i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
539 struct drm_i915_gem_pwrite *args,
540 struct drm_file *file_priv)
673a394b 541{
3de09aa3
EA		 542 struct drm_i915_gem_object *obj_priv = obj->driver_private;
543 drm_i915_private_t *dev_priv = dev->dev_private;
544 ssize_t remain;
545 loff_t gtt_page_base, offset;
546 loff_t first_data_page, last_data_page, num_pages;
547 loff_t pinned_pages, i;
548 struct page **user_pages;
549 struct mm_struct *mm = current->mm;
550 int gtt_page_offset, data_page_offset, data_page_index, page_length;
673a394b 551 int ret;
3de09aa3
EA		 552 uint64_t data_ptr = args->data_ptr;
553
554 remain = args->size;
555
556 /* Pin the user pages containing the data. We can't fault while
557 * holding the struct mutex, and all of the pwrite implementations
558 * want to hold it while dereferencing the user data.
559 */
560 first_data_page = data_ptr / PAGE_SIZE;
561 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
562 num_pages = last_data_page - first_data_page + 1;
563
564 user_pages = kcalloc(num_pages, sizeof(struct page *), GFP_KERNEL);
565 if (user_pages == NULL)
566 return -ENOMEM;
567
568 down_read(&mm->mmap_sem);
569 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
570 num_pages, 0, 0, user_pages, NULL);
571 up_read(&mm->mmap_sem);
572 if (pinned_pages < num_pages) {
573 ret = -EFAULT;
574 goto out_unpin_pages;
575 }
673a394b
EA		 576
577 mutex_lock(&dev->struct_mutex);
3de09aa3
EA		 578 ret = i915_gem_object_pin(obj, 0);
579 if (ret)
580 goto out_unlock;
581
582 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
583 if (ret)
584 goto out_unpin_object;
585
586 obj_priv = obj->driver_private;
587 offset = obj_priv->gtt_offset + args->offset;
588
589 while (remain > 0) {
590 /* Operation in this page
591 *
592 * gtt_page_base = page offset within aperture
593 * gtt_page_offset = offset within page in aperture
594 * data_page_index = page number in get_user_pages return
595 * data_page_offset = offset with data_page_index page.
596 * page_length = bytes to copy for this page
597 */
598 gtt_page_base = offset & PAGE_MASK;
599 gtt_page_offset = offset & ~PAGE_MASK;
600 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
601 data_page_offset = data_ptr & ~PAGE_MASK;
602
603 page_length = remain;
604 if ((gtt_page_offset + page_length) > PAGE_SIZE)
605 page_length = PAGE_SIZE - gtt_page_offset;
606 if ((data_page_offset + page_length) > PAGE_SIZE)
607 page_length = PAGE_SIZE - data_page_offset;
608
609 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
610 gtt_page_base, gtt_page_offset,
611 user_pages[data_page_index],
612 data_page_offset,
613 page_length);
614
615 /* If we get a fault while copying data, then (presumably) our
616 * source page isn't available. Return the error and we'll
617 * retry in the slow path.
618 */
619 if (ret)
620 goto out_unpin_object;
621
622 remain -= page_length;
623 offset += page_length;
624 data_ptr += page_length;
625 }
626
627out_unpin_object:
628 i915_gem_object_unpin(obj);
629out_unlock:
630 mutex_unlock(&dev->struct_mutex);
631out_unpin_pages:
632 for (i = 0; i < pinned_pages; i++)
633 page_cache_release(user_pages[i]);
634 kfree(user_pages);
635
636 return ret;
637}
638
40123c1f
EA		 639/**
640 * This is the fast shmem pwrite path, which attempts to directly
641 * copy_from_user into the kmapped pages backing the object.
642 */
3043c60c 643static int
40123c1f
EA		 644i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
645 struct drm_i915_gem_pwrite *args,
646 struct drm_file *file_priv)
673a394b 647{
40123c1f
EA		 648 struct drm_i915_gem_object *obj_priv = obj->driver_private;
649 ssize_t remain;
650 loff_t offset, page_base;
651 char __user *user_data;
652 int page_offset, page_length;
673a394b 653 int ret;
40123c1f
EA		 654
655 user_data = (char __user *) (uintptr_t) args->data_ptr;
656 remain = args->size;
673a394b
EA		 657
658 mutex_lock(&dev->struct_mutex);
659
40123c1f
EA		 660 ret = i915_gem_object_get_pages(obj);
661 if (ret != 0)
662 goto fail_unlock;
673a394b 663
e47c68e9 664 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
40123c1f
EA		 665 if (ret != 0)
666 goto fail_put_pages;
667
668 obj_priv = obj->driver_private;
669 offset = args->offset;
670 obj_priv->dirty = 1;
671
672 while (remain > 0) {
673 /* Operation in this page
674 *
675 * page_base = page offset within aperture
676 * page_offset = offset within page
677 * page_length = bytes to copy for this page
678 */
679 page_base = (offset & ~(PAGE_SIZE-1));
680 page_offset = offset & (PAGE_SIZE-1);
681 page_length = remain;
682 if ((page_offset + remain) > PAGE_SIZE)
683 page_length = PAGE_SIZE - page_offset;
684
685 ret = fast_shmem_write(obj_priv->pages,
686 page_base, page_offset,
687 user_data, page_length);
688 if (ret)
689 goto fail_put_pages;
690
691 remain -= page_length;
692 user_data += page_length;
693 offset += page_length;
694 }
695
696fail_put_pages:
697 i915_gem_object_put_pages(obj);
698fail_unlock:
699 mutex_unlock(&dev->struct_mutex);
700
701 return ret;
702}
703
704/**
705 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
706 * the memory and maps it using kmap_atomic for copying.
707 *
708 * This avoids taking mmap_sem for faulting on the user's address while the
709 * struct_mutex is held.
710 */
711static int
712i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
713 struct drm_i915_gem_pwrite *args,
714 struct drm_file *file_priv)
715{
716 struct drm_i915_gem_object *obj_priv = obj->driver_private;
717 struct mm_struct *mm = current->mm;
718 struct page **user_pages;
719 ssize_t remain;
720 loff_t offset, pinned_pages, i;
721 loff_t first_data_page, last_data_page, num_pages;
722 int shmem_page_index, shmem_page_offset;
723 int data_page_index, data_page_offset;
724 int page_length;
725 int ret;
726 uint64_t data_ptr = args->data_ptr;
727
728 remain = args->size;
729
730 /* Pin the user pages containing the data. We can't fault while
731 * holding the struct mutex, and all of the pwrite implementations
732 * want to hold it while dereferencing the user data.
733 */
734 first_data_page = data_ptr / PAGE_SIZE;
735 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
736 num_pages = last_data_page - first_data_page + 1;
737
738 user_pages = kcalloc(num_pages, sizeof(struct page *), GFP_KERNEL);
739 if (user_pages == NULL)
740 return -ENOMEM;
741
742 down_read(&mm->mmap_sem);
743 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
744 num_pages, 0, 0, user_pages, NULL);
745 up_read(&mm->mmap_sem);
746 if (pinned_pages < num_pages) {
747 ret = -EFAULT;
748 goto fail_put_user_pages;
673a394b
EA		 749 }
750
40123c1f
EA		 751 mutex_lock(&dev->struct_mutex);
752
753 ret = i915_gem_object_get_pages(obj);
754 if (ret != 0)
755 goto fail_unlock;
756
757 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
758 if (ret != 0)
759 goto fail_put_pages;
760
761 obj_priv = obj->driver_private;
673a394b 762 offset = args->offset;
40123c1f 763 obj_priv->dirty = 1;
673a394b 764
40123c1f
EA		 765 while (remain > 0) {
766 /* Operation in this page
767 *
768 * shmem_page_index = page number within shmem file
769 * shmem_page_offset = offset within page in shmem file
770 * data_page_index = page number in get_user_pages return
771 * data_page_offset = offset with data_page_index page.
772 * page_length = bytes to copy for this page
773 */
774 shmem_page_index = offset / PAGE_SIZE;
775 shmem_page_offset = offset & ~PAGE_MASK;
776 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
777 data_page_offset = data_ptr & ~PAGE_MASK;
778
779 page_length = remain;
780 if ((shmem_page_offset + page_length) > PAGE_SIZE)
781 page_length = PAGE_SIZE - shmem_page_offset;
782 if ((data_page_offset + page_length) > PAGE_SIZE)
783 page_length = PAGE_SIZE - data_page_offset;
784
785 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
786 shmem_page_offset,
787 user_pages[data_page_index],
788 data_page_offset,
789 page_length);
790 if (ret)
791 goto fail_put_pages;
792
793 remain -= page_length;
794 data_ptr += page_length;
795 offset += page_length;
673a394b
EA		 796 }
797
40123c1f
EA		 798fail_put_pages:
799 i915_gem_object_put_pages(obj);
800fail_unlock:
673a394b 801 mutex_unlock(&dev->struct_mutex);
40123c1f
EA		 802fail_put_user_pages:
803 for (i = 0; i < pinned_pages; i++)
804 page_cache_release(user_pages[i]);
805 kfree(user_pages);
673a394b 806
40123c1f 807 return ret;
673a394b
EA		 808}
809
810/**
811 * Writes data to the object referenced by handle.
812 *
813 * On error, the contents of the buffer that were to be modified are undefined.
814 */
815int
816i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
817 struct drm_file *file_priv)
818{
819 struct drm_i915_gem_pwrite *args = data;
820 struct drm_gem_object *obj;
821 struct drm_i915_gem_object *obj_priv;
822 int ret = 0;
823
824 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
825 if (obj == NULL)
826 return -EBADF;
827 obj_priv = obj->driver_private;
828
829 /* Bounds check destination.
830 *
831 * XXX: This could use review for overflow issues...
832 */
833 if (args->offset > obj->size || args->size > obj->size ||
834 args->offset + args->size > obj->size) {
835 drm_gem_object_unreference(obj);
836 return -EINVAL;
837 }
838
839 /* We can only do the GTT pwrite on untiled buffers, as otherwise
840 * it would end up going through the fenced access, and we'll get
841 * different detiling behavior between reading and writing.
842 * pread/pwrite currently are reading and writing from the CPU
843 * perspective, requiring manual detiling by the client.
844 */
71acb5eb
DA		 845 if (obj_priv->phys_obj)
846 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
847 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
3de09aa3
EA		 848 dev->gtt_total != 0) {
849 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
850 if (ret == -EFAULT) {
851 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
852 file_priv);
853 }
40123c1f
EA		 854 } else {
855 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
856 if (ret == -EFAULT) {
857 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
858 file_priv);
859 }
860 }
673a394b
EA		 861
862#if WATCH_PWRITE
863 if (ret)
864 DRM_INFO("pwrite failed %d\n", ret);
865#endif
866
867 drm_gem_object_unreference(obj);
868
869 return ret;
870}
871
872/**
2ef7eeaa
EA		 873 * Called when user space prepares to use an object with the CPU, either
874 * through the mmap ioctl's mapping or a GTT mapping.
673a394b
EA		 875 */
876int
877i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
878 struct drm_file *file_priv)
879{
880 struct drm_i915_gem_set_domain *args = data;
881 struct drm_gem_object *obj;
2ef7eeaa
EA		 882 uint32_t read_domains = args->read_domains;
883 uint32_t write_domain = args->write_domain;
673a394b
EA		 884 int ret;
885
886 if (!(dev->driver->driver_features & DRIVER_GEM))
887 return -ENODEV;
888
2ef7eeaa
EA		 889 /* Only handle setting domains to types used by the CPU. */
890 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
891 return -EINVAL;
892
893 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
894 return -EINVAL;
895
896 /* Having something in the write domain implies it's in the read
897 * domain, and only that read domain. Enforce that in the request.
898 */
899 if (write_domain != 0 && read_domains != write_domain)
900 return -EINVAL;
901
673a394b
EA		 902 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
903 if (obj == NULL)
904 return -EBADF;
905
906 mutex_lock(&dev->struct_mutex);
907#if WATCH_BUF
908 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
2ef7eeaa 909 obj, obj->size, read_domains, write_domain);
673a394b 910#endif
2ef7eeaa
EA		 911 if (read_domains & I915_GEM_DOMAIN_GTT) {
912 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
02354392
EA		 913
914 /* Silently promote "you're not bound, there was nothing to do"
915 * to success, since the client was just asking us to
916 * make sure everything was done.
917 */
918 if (ret == -EINVAL)
919 ret = 0;
2ef7eeaa 920 } else {
e47c68e9 921 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
2ef7eeaa
EA		 922 }
923
673a394b
EA		 924 drm_gem_object_unreference(obj);
925 mutex_unlock(&dev->struct_mutex);
926 return ret;
927}
928
929/**
930 * Called when user space has done writes to this buffer
931 */
932int
933i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
934 struct drm_file *file_priv)
935{
936 struct drm_i915_gem_sw_finish *args = data;
937 struct drm_gem_object *obj;
938 struct drm_i915_gem_object *obj_priv;
939 int ret = 0;
940
941 if (!(dev->driver->driver_features & DRIVER_GEM))
942 return -ENODEV;
943
944 mutex_lock(&dev->struct_mutex);
945 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
946 if (obj == NULL) {
947 mutex_unlock(&dev->struct_mutex);
948 return -EBADF;
949 }
950
951#if WATCH_BUF
952 DRM_INFO("%s: sw_finish %d (%p %d)\n",
953 __func__, args->handle, obj, obj->size);
954#endif
955 obj_priv = obj->driver_private;
956
957 /* Pinned buffers may be scanout, so flush the cache */
e47c68e9
EA		 958 if (obj_priv->pin_count)
959 i915_gem_object_flush_cpu_write_domain(obj);
960
673a394b
EA		 961 drm_gem_object_unreference(obj);
962 mutex_unlock(&dev->struct_mutex);
963 return ret;
964}
965
966/**
967 * Maps the contents of an object, returning the address it is mapped
968 * into.
969 *
970 * While the mapping holds a reference on the contents of the object, it doesn't
971 * imply a ref on the object itself.
972 */
973int
974i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
975 struct drm_file *file_priv)
976{
977 struct drm_i915_gem_mmap *args = data;
978 struct drm_gem_object *obj;
979 loff_t offset;
980 unsigned long addr;
981
982 if (!(dev->driver->driver_features & DRIVER_GEM))
983 return -ENODEV;
984
985 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
986 if (obj == NULL)
987 return -EBADF;
988
989 offset = args->offset;
990
991 down_write(&current->mm->mmap_sem);
992 addr = do_mmap(obj->filp, 0, args->size,
993 PROT_READ | PROT_WRITE, MAP_SHARED,
994 args->offset);
995 up_write(&current->mm->mmap_sem);
996 mutex_lock(&dev->struct_mutex);
997 drm_gem_object_unreference(obj);
998 mutex_unlock(&dev->struct_mutex);
999 if (IS_ERR((void *)addr))
1000 return addr;
1001
1002 args->addr_ptr = (uint64_t) addr;
1003
1004 return 0;
1005}
1006
de151cf6
JB		 1007/**
1008 * i915_gem_fault - fault a page into the GTT
1009 * vma: VMA in question
1010 * vmf: fault info
1011 *
1012 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1013 * from userspace. The fault handler takes care of binding the object to
1014 * the GTT (if needed), allocating and programming a fence register (again,
1015 * only if needed based on whether the old reg is still valid or the object
1016 * is tiled) and inserting a new PTE into the faulting process.
1017 *
1018 * Note that the faulting process may involve evicting existing objects
1019 * from the GTT and/or fence registers to make room. So performance may
1020 * suffer if the GTT working set is large or there are few fence registers
1021 * left.
1022 */
1023int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1024{
1025 struct drm_gem_object *obj = vma->vm_private_data;
1026 struct drm_device *dev = obj->dev;
1027 struct drm_i915_private *dev_priv = dev->dev_private;
1028 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1029 pgoff_t page_offset;
1030 unsigned long pfn;
1031 int ret = 0;
0f973f27 1032 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
de151cf6
JB		 1033
1034 /* We don't use vmf->pgoff since that has the fake offset */
1035 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1036 PAGE_SHIFT;
1037
1038 /* Now bind it into the GTT if needed */
1039 mutex_lock(&dev->struct_mutex);
1040 if (!obj_priv->gtt_space) {
1041 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1042 if (ret) {
1043 mutex_unlock(&dev->struct_mutex);
1044 return VM_FAULT_SIGBUS;
1045 }
1046 list_add(&obj_priv->list, &dev_priv->mm.inactive_list);
1047 }
1048
1049 /* Need a new fence register? */
1050 if (obj_priv->fence_reg == I915_FENCE_REG_NONE &&
d9ddcb96 1051 obj_priv->tiling_mode != I915_TILING_NONE) {
0f973f27 1052 ret = i915_gem_object_get_fence_reg(obj, write);
7d8d58b2
CW		 1053 if (ret) {
1054 mutex_unlock(&dev->struct_mutex);
d9ddcb96 1055 return VM_FAULT_SIGBUS;
7d8d58b2 1056 }
d9ddcb96 1057 }
de151cf6
JB		 1058
1059 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1060 page_offset;
1061
1062 /* Finally, remap it using the new GTT offset */
1063 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1064
1065 mutex_unlock(&dev->struct_mutex);
1066
1067 switch (ret) {
1068 case -ENOMEM:
1069 case -EAGAIN:
1070 return VM_FAULT_OOM;
1071 case -EFAULT:
de151cf6
JB		 1072 return VM_FAULT_SIGBUS;
1073 default:
1074 return VM_FAULT_NOPAGE;
1075 }
1076}
1077
1078/**
1079 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1080 * @obj: obj in question
1081 *
1082 * GEM memory mapping works by handing back to userspace a fake mmap offset
1083 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1084 * up the object based on the offset and sets up the various memory mapping
1085 * structures.
1086 *
1087 * This routine allocates and attaches a fake offset for @obj.
1088 */
1089static int
1090i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1091{
1092 struct drm_device *dev = obj->dev;
1093 struct drm_gem_mm *mm = dev->mm_private;
1094 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1095 struct drm_map_list *list;
f77d390c 1096 struct drm_local_map *map;
de151cf6
JB		 1097 int ret = 0;
1098
1099 /* Set the object up for mmap'ing */
1100 list = &obj->map_list;
1101 list->map = drm_calloc(1, sizeof(struct drm_map_list),
1102 DRM_MEM_DRIVER);
1103 if (!list->map)
1104 return -ENOMEM;
1105
1106 map = list->map;
1107 map->type = _DRM_GEM;
1108 map->size = obj->size;
1109 map->handle = obj;
1110
1111 /* Get a DRM GEM mmap offset allocated... */
1112 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1113 obj->size / PAGE_SIZE, 0, 0);
1114 if (!list->file_offset_node) {
1115 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1116 ret = -ENOMEM;
1117 goto out_free_list;
1118 }
1119
1120 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1121 obj->size / PAGE_SIZE, 0);
1122 if (!list->file_offset_node) {
1123 ret = -ENOMEM;
1124 goto out_free_list;
1125 }
1126
1127 list->hash.key = list->file_offset_node->start;
1128 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1129 DRM_ERROR("failed to add to map hash\n");
1130 goto out_free_mm;
1131 }
1132
1133 /* By now we should be all set, any drm_mmap request on the offset
1134 * below will get to our mmap & fault handler */
1135 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1136
1137 return 0;
1138
1139out_free_mm:
1140 drm_mm_put_block(list->file_offset_node);
1141out_free_list:
1142 drm_free(list->map, sizeof(struct drm_map_list), DRM_MEM_DRIVER);
1143
1144 return ret;
1145}
1146
ab00b3e5
JB		 1147static void
1148i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1149{
1150 struct drm_device *dev = obj->dev;
1151 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1152 struct drm_gem_mm *mm = dev->mm_private;
1153 struct drm_map_list *list;
1154
1155 list = &obj->map_list;
1156 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1157
1158 if (list->file_offset_node) {
1159 drm_mm_put_block(list->file_offset_node);
1160 list->file_offset_node = NULL;
1161 }
1162
1163 if (list->map) {
1164 drm_free(list->map, sizeof(struct drm_map), DRM_MEM_DRIVER);
1165 list->map = NULL;
1166 }
1167
1168 obj_priv->mmap_offset = 0;
1169}
1170
de151cf6
JB		 1171/**
1172 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1173 * @obj: object to check
1174 *
1175 * Return the required GTT alignment for an object, taking into account
1176 * potential fence register mapping if needed.
1177 */
1178static uint32_t
1179i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1180{
1181 struct drm_device *dev = obj->dev;
1182 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1183 int start, i;
1184
1185 /*
1186 * Minimum alignment is 4k (GTT page size), but might be greater
1187 * if a fence register is needed for the object.
1188 */
1189 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1190 return 4096;
1191
1192 /*
1193 * Previous chips need to be aligned to the size of the smallest
1194 * fence register that can contain the object.
1195 */
1196 if (IS_I9XX(dev))
1197 start = 1024*1024;
1198 else
1199 start = 512*1024;
1200
1201 for (i = start; i < obj->size; i <<= 1)
1202 ;
1203
1204 return i;
1205}
1206
1207/**
1208 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1209 * @dev: DRM device
1210 * @data: GTT mapping ioctl data
1211 * @file_priv: GEM object info
1212 *
1213 * Simply returns the fake offset to userspace so it can mmap it.
1214 * The mmap call will end up in drm_gem_mmap(), which will set things
1215 * up so we can get faults in the handler above.
1216 *
1217 * The fault handler will take care of binding the object into the GTT
1218 * (since it may have been evicted to make room for something), allocating
1219 * a fence register, and mapping the appropriate aperture address into
1220 * userspace.
1221 */
1222int
1223i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1224 struct drm_file *file_priv)
1225{
1226 struct drm_i915_gem_mmap_gtt *args = data;
1227 struct drm_i915_private *dev_priv = dev->dev_private;
1228 struct drm_gem_object *obj;
1229 struct drm_i915_gem_object *obj_priv;
1230 int ret;
1231
1232 if (!(dev->driver->driver_features & DRIVER_GEM))
1233 return -ENODEV;
1234
1235 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1236 if (obj == NULL)
1237 return -EBADF;
1238
1239 mutex_lock(&dev->struct_mutex);
1240
1241 obj_priv = obj->driver_private;
1242
1243 if (!obj_priv->mmap_offset) {
1244 ret = i915_gem_create_mmap_offset(obj);
13af1062
CW		 1245 if (ret) {
1246 drm_gem_object_unreference(obj);
1247 mutex_unlock(&dev->struct_mutex);
de151cf6 1248 return ret;
13af1062 1249 }
de151cf6
JB		 1250 }
1251
1252 args->offset = obj_priv->mmap_offset;
1253
1254 obj_priv->gtt_alignment = i915_gem_get_gtt_alignment(obj);
1255
1256 /* Make sure the alignment is correct for fence regs etc */
1257 if (obj_priv->agp_mem &&
1258 (obj_priv->gtt_offset & (obj_priv->gtt_alignment - 1))) {
1259 drm_gem_object_unreference(obj);
1260 mutex_unlock(&dev->struct_mutex);
1261 return -EINVAL;
1262 }
1263
1264 /*
1265 * Pull it into the GTT so that we have a page list (makes the
1266 * initial fault faster and any subsequent flushing possible).
1267 */
1268 if (!obj_priv->agp_mem) {
1269 ret = i915_gem_object_bind_to_gtt(obj, obj_priv->gtt_alignment);
1270 if (ret) {
1271 drm_gem_object_unreference(obj);
1272 mutex_unlock(&dev->struct_mutex);
1273 return ret;
1274 }
1275 list_add(&obj_priv->list, &dev_priv->mm.inactive_list);
1276 }
1277
1278 drm_gem_object_unreference(obj);
1279 mutex_unlock(&dev->struct_mutex);
1280
1281 return 0;
1282}
1283
673a394b 1284static void
856fa198 1285i915_gem_object_put_pages(struct drm_gem_object *obj)
673a394b
EA		 1286{
1287 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1288 int page_count = obj->size / PAGE_SIZE;
1289 int i;
1290
856fa198 1291 BUG_ON(obj_priv->pages_refcount == 0);
673a394b 1292
856fa198
EA		 1293 if (--obj_priv->pages_refcount != 0)
1294 return;
673a394b
EA		 1295
1296 for (i = 0; i < page_count; i++)
856fa198 1297 if (obj_priv->pages[i] != NULL) {
673a394b 1298 if (obj_priv->dirty)
856fa198
EA		 1299 set_page_dirty(obj_priv->pages[i]);
1300 mark_page_accessed(obj_priv->pages[i]);
1301 page_cache_release(obj_priv->pages[i]);
673a394b
EA		 1302 }
1303 obj_priv->dirty = 0;
1304
856fa198 1305 drm_free(obj_priv->pages,
673a394b
EA		 1306 page_count * sizeof(struct page *),
1307 DRM_MEM_DRIVER);
856fa198 1308 obj_priv->pages = NULL;
673a394b
EA		 1309}
1310
1311static void
ce44b0ea 1312i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
673a394b
EA		 1313{
1314 struct drm_device *dev = obj->dev;
1315 drm_i915_private_t *dev_priv = dev->dev_private;
1316 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1317
1318 /* Add a reference if we're newly entering the active list. */
1319 if (!obj_priv->active) {
1320 drm_gem_object_reference(obj);
1321 obj_priv->active = 1;
1322 }
1323 /* Move from whatever list we were on to the tail of execution. */
1324 list_move_tail(&obj_priv->list,
1325 &dev_priv->mm.active_list);
ce44b0ea 1326 obj_priv->last_rendering_seqno = seqno;
673a394b
EA		 1327}
1328
ce44b0ea
EA		 1329static void
1330i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1331{
1332 struct drm_device *dev = obj->dev;
1333 drm_i915_private_t *dev_priv = dev->dev_private;
1334 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1335
1336 BUG_ON(!obj_priv->active);
1337 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1338 obj_priv->last_rendering_seqno = 0;
1339}
673a394b
EA		 1340
1341static void
1342i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1343{
1344 struct drm_device *dev = obj->dev;
1345 drm_i915_private_t *dev_priv = dev->dev_private;
1346 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1347
1348 i915_verify_inactive(dev, __FILE__, __LINE__);
1349 if (obj_priv->pin_count != 0)
1350 list_del_init(&obj_priv->list);
1351 else
1352 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1353
ce44b0ea 1354 obj_priv->last_rendering_seqno = 0;
673a394b
EA		 1355 if (obj_priv->active) {
1356 obj_priv->active = 0;
1357 drm_gem_object_unreference(obj);
1358 }
1359 i915_verify_inactive(dev, __FILE__, __LINE__);
1360}
1361
1362/**
1363 * Creates a new sequence number, emitting a write of it to the status page
1364 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1365 *
1366 * Must be called with struct_lock held.
1367 *
1368 * Returned sequence numbers are nonzero on success.
1369 */
1370static uint32_t
1371i915_add_request(struct drm_device *dev, uint32_t flush_domains)
1372{
1373 drm_i915_private_t *dev_priv = dev->dev_private;
1374 struct drm_i915_gem_request *request;
1375 uint32_t seqno;
1376 int was_empty;
1377 RING_LOCALS;
1378
1379 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
1380 if (request == NULL)
1381 return 0;
1382
1383 /* Grab the seqno we're going to make this request be, and bump the
1384 * next (skipping 0 so it can be the reserved no-seqno value).
1385 */
1386 seqno = dev_priv->mm.next_gem_seqno;
1387 dev_priv->mm.next_gem_seqno++;
1388 if (dev_priv->mm.next_gem_seqno == 0)
1389 dev_priv->mm.next_gem_seqno++;
1390
1391 BEGIN_LP_RING(4);
1392 OUT_RING(MI_STORE_DWORD_INDEX);
1393 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1394 OUT_RING(seqno);
1395
1396 OUT_RING(MI_USER_INTERRUPT);
1397 ADVANCE_LP_RING();
1398
1399 DRM_DEBUG("%d\n", seqno);
1400
1401 request->seqno = seqno;
1402 request->emitted_jiffies = jiffies;
673a394b
EA		 1403 was_empty = list_empty(&dev_priv->mm.request_list);
1404 list_add_tail(&request->list, &dev_priv->mm.request_list);
1405
ce44b0ea
EA		 1406 /* Associate any objects on the flushing list matching the write
1407 * domain we're flushing with our flush.
1408 */
1409 if (flush_domains != 0) {
1410 struct drm_i915_gem_object *obj_priv, *next;
1411
1412 list_for_each_entry_safe(obj_priv, next,
1413 &dev_priv->mm.flushing_list, list) {
1414 struct drm_gem_object *obj = obj_priv->obj;
1415
1416 if ((obj->write_domain & flush_domains) ==
1417 obj->write_domain) {
1418 obj->write_domain = 0;
1419 i915_gem_object_move_to_active(obj, seqno);
1420 }
1421 }
1422
1423 }
1424
6dbe2772 1425 if (was_empty && !dev_priv->mm.suspended)
673a394b
EA		 1426 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
1427 return seqno;
1428}
1429
1430/**
1431 * Command execution barrier
1432 *
1433 * Ensures that all commands in the ring are finished
1434 * before signalling the CPU
1435 */
3043c60c 1436static uint32_t
673a394b
EA		 1437i915_retire_commands(struct drm_device *dev)
1438{
1439 drm_i915_private_t *dev_priv = dev->dev_private;
1440 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1441 uint32_t flush_domains = 0;
1442 RING_LOCALS;
1443
1444 /* The sampler always gets flushed on i965 (sigh) */
1445 if (IS_I965G(dev))
1446 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1447 BEGIN_LP_RING(2);
1448 OUT_RING(cmd);
1449 OUT_RING(0); /* noop */
1450 ADVANCE_LP_RING();
1451 return flush_domains;
1452}
1453
1454/**
1455 * Moves buffers associated only with the given active seqno from the active
1456 * to inactive list, potentially freeing them.
1457 */
1458static void
1459i915_gem_retire_request(struct drm_device *dev,
1460 struct drm_i915_gem_request *request)
1461{
1462 drm_i915_private_t *dev_priv = dev->dev_private;
1463
1464 /* Move any buffers on the active list that are no longer referenced
1465 * by the ringbuffer to the flushing/inactive lists as appropriate.
1466 */
1467 while (!list_empty(&dev_priv->mm.active_list)) {
1468 struct drm_gem_object *obj;
1469 struct drm_i915_gem_object *obj_priv;
1470
1471 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1472 struct drm_i915_gem_object,
1473 list);
1474 obj = obj_priv->obj;
1475
1476 /* If the seqno being retired doesn't match the oldest in the
1477 * list, then the oldest in the list must still be newer than
1478 * this seqno.
1479 */
1480 if (obj_priv->last_rendering_seqno != request->seqno)
1481 return;
de151cf6 1482
673a394b
EA		 1483#if WATCH_LRU
1484 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1485 __func__, request->seqno, obj);
1486#endif
1487
ce44b0ea
EA		 1488 if (obj->write_domain != 0)
1489 i915_gem_object_move_to_flushing(obj);
1490 else
673a394b 1491 i915_gem_object_move_to_inactive(obj);
673a394b
EA		 1492 }
1493}
1494
1495/**
1496 * Returns true if seq1 is later than seq2.
1497 */
1498static int
1499i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1500{
1501 return (int32_t)(seq1 - seq2) >= 0;
1502}
1503
1504uint32_t
1505i915_get_gem_seqno(struct drm_device *dev)
1506{
1507 drm_i915_private_t *dev_priv = dev->dev_private;
1508
1509 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1510}
1511
1512/**
1513 * This function clears the request list as sequence numbers are passed.
1514 */
1515void
1516i915_gem_retire_requests(struct drm_device *dev)
1517{
1518 drm_i915_private_t *dev_priv = dev->dev_private;
1519 uint32_t seqno;
1520
6c0594a3
KW		 1521 if (!dev_priv->hw_status_page)
1522 return;
1523
673a394b
EA		 1524 seqno = i915_get_gem_seqno(dev);
1525
1526 while (!list_empty(&dev_priv->mm.request_list)) {
1527 struct drm_i915_gem_request *request;
1528 uint32_t retiring_seqno;
1529
1530 request = list_first_entry(&dev_priv->mm.request_list,
1531 struct drm_i915_gem_request,
1532 list);
1533 retiring_seqno = request->seqno;
1534
1535 if (i915_seqno_passed(seqno, retiring_seqno) ||
1536 dev_priv->mm.wedged) {
1537 i915_gem_retire_request(dev, request);
1538
1539 list_del(&request->list);
1540 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
1541 } else
1542 break;
1543 }
1544}
1545
1546void
1547i915_gem_retire_work_handler(struct work_struct *work)
1548{
1549 drm_i915_private_t *dev_priv;
1550 struct drm_device *dev;
1551
1552 dev_priv = container_of(work, drm_i915_private_t,
1553 mm.retire_work.work);
1554 dev = dev_priv->dev;
1555
1556 mutex_lock(&dev->struct_mutex);
1557 i915_gem_retire_requests(dev);
6dbe2772
KP		 1558 if (!dev_priv->mm.suspended &&
1559 !list_empty(&dev_priv->mm.request_list))
673a394b
EA		 1560 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
1561 mutex_unlock(&dev->struct_mutex);
1562}
1563
1564/**
1565 * Waits for a sequence number to be signaled, and cleans up the
1566 * request and object lists appropriately for that event.
1567 */
3043c60c 1568static int
673a394b
EA		 1569i915_wait_request(struct drm_device *dev, uint32_t seqno)
1570{
1571 drm_i915_private_t *dev_priv = dev->dev_private;
1572 int ret = 0;
1573
1574 BUG_ON(seqno == 0);
1575
1576 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
1577 dev_priv->mm.waiting_gem_seqno = seqno;
1578 i915_user_irq_get(dev);
1579 ret = wait_event_interruptible(dev_priv->irq_queue,
1580 i915_seqno_passed(i915_get_gem_seqno(dev),
1581 seqno) ||
1582 dev_priv->mm.wedged);
1583 i915_user_irq_put(dev);
1584 dev_priv->mm.waiting_gem_seqno = 0;
1585 }
1586 if (dev_priv->mm.wedged)
1587 ret = -EIO;
1588
1589 if (ret && ret != -ERESTARTSYS)
1590 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1591 __func__, ret, seqno, i915_get_gem_seqno(dev));
1592
1593 /* Directly dispatch request retiring. While we have the work queue
1594 * to handle this, the waiter on a request often wants an associated
1595 * buffer to have made it to the inactive list, and we would need
1596 * a separate wait queue to handle that.
1597 */
1598 if (ret == 0)
1599 i915_gem_retire_requests(dev);
1600
1601 return ret;
1602}
1603
1604static void
1605i915_gem_flush(struct drm_device *dev,
1606 uint32_t invalidate_domains,
1607 uint32_t flush_domains)
1608{
1609 drm_i915_private_t *dev_priv = dev->dev_private;
1610 uint32_t cmd;
1611 RING_LOCALS;
1612
1613#if WATCH_EXEC
1614 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1615 invalidate_domains, flush_domains);
1616#endif
1617
1618 if (flush_domains & I915_GEM_DOMAIN_CPU)
1619 drm_agp_chipset_flush(dev);
1620
1621 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
1622 I915_GEM_DOMAIN_GTT)) {
1623 /*
1624 * read/write caches:
1625 *
1626 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1627 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1628 * also flushed at 2d versus 3d pipeline switches.
1629 *
1630 * read-only caches:
1631 *
1632 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1633 * MI_READ_FLUSH is set, and is always flushed on 965.
1634 *
1635 * I915_GEM_DOMAIN_COMMAND may not exist?
1636 *
1637 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1638 * invalidated when MI_EXE_FLUSH is set.
1639 *
1640 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1641 * invalidated with every MI_FLUSH.
1642 *
1643 * TLBs:
1644 *
1645 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1646 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1647 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1648 * are flushed at any MI_FLUSH.
1649 */
1650
1651 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1652 if ((invalidate_domains|flush_domains) &
1653 I915_GEM_DOMAIN_RENDER)
1654 cmd &= ~MI_NO_WRITE_FLUSH;
1655 if (!IS_I965G(dev)) {
1656 /*
1657 * On the 965, the sampler cache always gets flushed
1658 * and this bit is reserved.
1659 */
1660 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1661 cmd |= MI_READ_FLUSH;
1662 }
1663 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1664 cmd |= MI_EXE_FLUSH;
1665
1666#if WATCH_EXEC
1667 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1668#endif
1669 BEGIN_LP_RING(2);
1670 OUT_RING(cmd);
1671 OUT_RING(0); /* noop */
1672 ADVANCE_LP_RING();
1673 }
1674}
1675
1676/**
1677 * Ensures that all rendering to the object has completed and the object is
1678 * safe to unbind from the GTT or access from the CPU.
1679 */
1680static int
1681i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1682{
1683 struct drm_device *dev = obj->dev;
1684 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1685 int ret;
1686
e47c68e9
EA		 1687 /* This function only exists to support waiting for existing rendering,
1688 * not for emitting required flushes.
673a394b 1689 */
e47c68e9 1690 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
673a394b
EA		 1691
1692 /* If there is rendering queued on the buffer being evicted, wait for
1693 * it.
1694 */
1695 if (obj_priv->active) {
1696#if WATCH_BUF
1697 DRM_INFO("%s: object %p wait for seqno %08x\n",
1698 __func__, obj, obj_priv->last_rendering_seqno);
1699#endif
1700 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1701 if (ret != 0)
1702 return ret;
1703 }
1704
1705 return 0;
1706}
1707
1708/**
1709 * Unbinds an object from the GTT aperture.
1710 */
0f973f27 1711int
673a394b
EA		 1712i915_gem_object_unbind(struct drm_gem_object *obj)
1713{
1714 struct drm_device *dev = obj->dev;
1715 struct drm_i915_gem_object *obj_priv = obj->driver_private;
de151cf6 1716 loff_t offset;
673a394b
EA		 1717 int ret = 0;
1718
1719#if WATCH_BUF
1720 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1721 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1722#endif
1723 if (obj_priv->gtt_space == NULL)
1724 return 0;
1725
1726 if (obj_priv->pin_count != 0) {
1727 DRM_ERROR("Attempting to unbind pinned buffer\n");
1728 return -EINVAL;
1729 }
1730
673a394b
EA		 1731 /* Move the object to the CPU domain to ensure that
1732 * any possible CPU writes while it's not in the GTT
1733 * are flushed when we go to remap it. This will
1734 * also ensure that all pending GPU writes are finished
1735 * before we unbind.
1736 */
e47c68e9 1737 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
673a394b 1738 if (ret) {
e47c68e9
EA		 1739 if (ret != -ERESTARTSYS)
1740 DRM_ERROR("set_domain failed: %d\n", ret);
673a394b
EA		 1741 return ret;
1742 }
1743
1744 if (obj_priv->agp_mem != NULL) {
1745 drm_unbind_agp(obj_priv->agp_mem);
1746 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1747 obj_priv->agp_mem = NULL;
1748 }
1749
1750 BUG_ON(obj_priv->active);
1751
de151cf6
JB		 1752 /* blow away mappings if mapped through GTT */
1753 offset = ((loff_t) obj->map_list.hash.key) << PAGE_SHIFT;
79e53945
JB		 1754 if (dev->dev_mapping)
1755 unmap_mapping_range(dev->dev_mapping, offset, obj->size, 1);
de151cf6
JB		 1756
1757 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1758 i915_gem_clear_fence_reg(obj);
1759
856fa198 1760 i915_gem_object_put_pages(obj);
673a394b
EA		 1761
1762 if (obj_priv->gtt_space) {
1763 atomic_dec(&dev->gtt_count);
1764 atomic_sub(obj->size, &dev->gtt_memory);
1765
1766 drm_mm_put_block(obj_priv->gtt_space);
1767 obj_priv->gtt_space = NULL;
1768 }
1769
1770 /* Remove ourselves from the LRU list if present. */
1771 if (!list_empty(&obj_priv->list))
1772 list_del_init(&obj_priv->list);
1773
1774 return 0;
1775}
1776
1777static int
1778i915_gem_evict_something(struct drm_device *dev)
1779{
1780 drm_i915_private_t *dev_priv = dev->dev_private;
1781 struct drm_gem_object *obj;
1782 struct drm_i915_gem_object *obj_priv;
1783 int ret = 0;
1784
1785 for (;;) {
1786 /* If there's an inactive buffer available now, grab it
1787 * and be done.
1788 */
1789 if (!list_empty(&dev_priv->mm.inactive_list)) {
1790 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
1791 struct drm_i915_gem_object,
1792 list);
1793 obj = obj_priv->obj;
1794 BUG_ON(obj_priv->pin_count != 0);
1795#if WATCH_LRU
1796 DRM_INFO("%s: evicting %p\n", __func__, obj);
1797#endif
1798 BUG_ON(obj_priv->active);
1799
1800 /* Wait on the rendering and unbind the buffer. */
1801 ret = i915_gem_object_unbind(obj);
1802 break;
1803 }
1804
1805 /* If we didn't get anything, but the ring is still processing
1806 * things, wait for one of those things to finish and hopefully
1807 * leave us a buffer to evict.
1808 */
1809 if (!list_empty(&dev_priv->mm.request_list)) {
1810 struct drm_i915_gem_request *request;
1811
1812 request = list_first_entry(&dev_priv->mm.request_list,
1813 struct drm_i915_gem_request,
1814 list);
1815
1816 ret = i915_wait_request(dev, request->seqno);
1817 if (ret)
1818 break;
1819
1820 /* if waiting caused an object to become inactive,
1821 * then loop around and wait for it. Otherwise, we
1822 * assume that waiting freed and unbound something,
1823 * so there should now be some space in the GTT
1824 */
1825 if (!list_empty(&dev_priv->mm.inactive_list))
1826 continue;
1827 break;
1828 }
1829
1830 /* If we didn't have anything on the request list but there
1831 * are buffers awaiting a flush, emit one and try again.
1832 * When we wait on it, those buffers waiting for that flush
1833 * will get moved to inactive.
1834 */
1835 if (!list_empty(&dev_priv->mm.flushing_list)) {
1836 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1837 struct drm_i915_gem_object,
1838 list);
1839 obj = obj_priv->obj;
1840
1841 i915_gem_flush(dev,
1842 obj->write_domain,
1843 obj->write_domain);
1844 i915_add_request(dev, obj->write_domain);
1845
1846 obj = NULL;
1847 continue;
1848 }
1849
1850 DRM_ERROR("inactive empty %d request empty %d "
1851 "flushing empty %d\n",
1852 list_empty(&dev_priv->mm.inactive_list),
1853 list_empty(&dev_priv->mm.request_list),
1854 list_empty(&dev_priv->mm.flushing_list));
1855 /* If we didn't do any of the above, there's nothing to be done
1856 * and we just can't fit it in.
1857 */
1858 return -ENOMEM;
1859 }
1860 return ret;
1861}
1862
ac94a962
KP		 1863static int
1864i915_gem_evict_everything(struct drm_device *dev)
1865{
1866 int ret;
1867
1868 for (;;) {
1869 ret = i915_gem_evict_something(dev);
1870 if (ret != 0)
1871 break;
1872 }
15c35334
OA		 1873 if (ret == -ENOMEM)
1874 return 0;
ac94a962
KP		 1875 return ret;
1876}
1877
673a394b 1878static int
856fa198 1879i915_gem_object_get_pages(struct drm_gem_object *obj)
673a394b
EA		 1880{
1881 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1882 int page_count, i;
1883 struct address_space *mapping;
1884 struct inode *inode;
1885 struct page *page;
1886 int ret;
1887
856fa198 1888 if (obj_priv->pages_refcount++ != 0)
673a394b
EA		 1889 return 0;
1890
1891 /* Get the list of pages out of our struct file. They'll be pinned
1892 * at this point until we release them.
1893 */
1894 page_count = obj->size / PAGE_SIZE;
856fa198
EA		 1895 BUG_ON(obj_priv->pages != NULL);
1896 obj_priv->pages = drm_calloc(page_count, sizeof(struct page *),
1897 DRM_MEM_DRIVER);
1898 if (obj_priv->pages == NULL) {
673a394b 1899 DRM_ERROR("Faled to allocate page list\n");
856fa198 1900 obj_priv->pages_refcount--;
673a394b
EA		 1901 return -ENOMEM;
1902 }
1903
1904 inode = obj->filp->f_path.dentry->d_inode;
1905 mapping = inode->i_mapping;
1906 for (i = 0; i < page_count; i++) {
1907 page = read_mapping_page(mapping, i, NULL);
1908 if (IS_ERR(page)) {
1909 ret = PTR_ERR(page);
1910 DRM_ERROR("read_mapping_page failed: %d\n", ret);
856fa198 1911 i915_gem_object_put_pages(obj);
673a394b
EA		 1912 return ret;
1913 }
856fa198 1914 obj_priv->pages[i] = page;
673a394b
EA		 1915 }
1916 return 0;
1917}
1918
de151cf6
JB		 1919static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
1920{
1921 struct drm_gem_object *obj = reg->obj;
1922 struct drm_device *dev = obj->dev;
1923 drm_i915_private_t *dev_priv = dev->dev_private;
1924 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1925 int regnum = obj_priv->fence_reg;
1926 uint64_t val;
1927
1928 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
1929 0xfffff000) << 32;
1930 val |= obj_priv->gtt_offset & 0xfffff000;
1931 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
1932 if (obj_priv->tiling_mode == I915_TILING_Y)
1933 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
1934 val |= I965_FENCE_REG_VALID;
1935
1936 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
1937}
1938
1939static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
1940{
1941 struct drm_gem_object *obj = reg->obj;
1942 struct drm_device *dev = obj->dev;
1943 drm_i915_private_t *dev_priv = dev->dev_private;
1944 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1945 int regnum = obj_priv->fence_reg;
0f973f27 1946 int tile_width;
dc529a4f 1947 uint32_t fence_reg, val;
de151cf6
JB		 1948 uint32_t pitch_val;
1949
1950 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
1951 (obj_priv->gtt_offset & (obj->size - 1))) {
f06da264 1952 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
0f973f27 1953 __func__, obj_priv->gtt_offset, obj->size);
de151cf6
JB		 1954 return;
1955 }
1956
0f973f27
JB		 1957 if (obj_priv->tiling_mode == I915_TILING_Y &&
1958 HAS_128_BYTE_Y_TILING(dev))
1959 tile_width = 128;
de151cf6 1960 else
0f973f27
JB		 1961 tile_width = 512;
1962
1963 /* Note: pitch better be a power of two tile widths */
1964 pitch_val = obj_priv->stride / tile_width;
1965 pitch_val = ffs(pitch_val) - 1;
de151cf6
JB		 1966
1967 val = obj_priv->gtt_offset;
1968 if (obj_priv->tiling_mode == I915_TILING_Y)
1969 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
1970 val |= I915_FENCE_SIZE_BITS(obj->size);
1971 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
1972 val |= I830_FENCE_REG_VALID;
1973
dc529a4f
EA		 1974 if (regnum < 8)
1975 fence_reg = FENCE_REG_830_0 + (regnum * 4);
1976 else
1977 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
1978 I915_WRITE(fence_reg, val);
de151cf6
JB		 1979}
1980
1981static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
1982{
1983 struct drm_gem_object *obj = reg->obj;
1984 struct drm_device *dev = obj->dev;
1985 drm_i915_private_t *dev_priv = dev->dev_private;
1986 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1987 int regnum = obj_priv->fence_reg;
1988 uint32_t val;
1989 uint32_t pitch_val;
1990
1991 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
1992 (obj_priv->gtt_offset & (obj->size - 1))) {
0f973f27
JB		 1993 WARN(1, "%s: object 0x%08x not 1M or size aligned\n",
1994 __func__, obj_priv->gtt_offset);
de151cf6
JB		 1995 return;
1996 }
1997
1998 pitch_val = (obj_priv->stride / 128) - 1;
1999
2000 val = obj_priv->gtt_offset;
2001 if (obj_priv->tiling_mode == I915_TILING_Y)
2002 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2003 val |= I830_FENCE_SIZE_BITS(obj->size);
2004 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2005 val |= I830_FENCE_REG_VALID;
2006
2007 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
2008
2009}
2010
2011/**
2012 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2013 * @obj: object to map through a fence reg
0f973f27 2014 * @write: object is about to be written
de151cf6
JB		 2015 *
2016 * When mapping objects through the GTT, userspace wants to be able to write
2017 * to them without having to worry about swizzling if the object is tiled.
2018 *
2019 * This function walks the fence regs looking for a free one for @obj,
2020 * stealing one if it can't find any.
2021 *
2022 * It then sets up the reg based on the object's properties: address, pitch
2023 * and tiling format.
2024 */
d9ddcb96 2025static int
0f973f27 2026i915_gem_object_get_fence_reg(struct drm_gem_object *obj, bool write)
de151cf6
JB		 2027{
2028 struct drm_device *dev = obj->dev;
79e53945 2029 struct drm_i915_private *dev_priv = dev->dev_private;
de151cf6
JB		 2030 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2031 struct drm_i915_fence_reg *reg = NULL;
fc7170ba
CW		 2032 struct drm_i915_gem_object *old_obj_priv = NULL;
2033 int i, ret, avail;
de151cf6
JB		 2034
2035 switch (obj_priv->tiling_mode) {
2036 case I915_TILING_NONE:
2037 WARN(1, "allocating a fence for non-tiled object?\n");
2038 break;
2039 case I915_TILING_X:
0f973f27
JB		 2040 if (!obj_priv->stride)
2041 return -EINVAL;
2042 WARN((obj_priv->stride & (512 - 1)),
2043 "object 0x%08x is X tiled but has non-512B pitch\n",
2044 obj_priv->gtt_offset);
de151cf6
JB		 2045 break;
2046 case I915_TILING_Y:
0f973f27
JB		 2047 if (!obj_priv->stride)
2048 return -EINVAL;
2049 WARN((obj_priv->stride & (128 - 1)),
2050 "object 0x%08x is Y tiled but has non-128B pitch\n",
2051 obj_priv->gtt_offset);
de151cf6
JB		 2052 break;
2053 }
2054
2055 /* First try to find a free reg */
9b2412f9 2056try_again:
fc7170ba 2057 avail = 0;
de151cf6
JB		 2058 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2059 reg = &dev_priv->fence_regs[i];
2060 if (!reg->obj)
2061 break;
fc7170ba
CW		 2062
2063 old_obj_priv = reg->obj->driver_private;
2064 if (!old_obj_priv->pin_count)
2065 avail++;
de151cf6
JB		 2066 }
2067
2068 /* None available, try to steal one or wait for a user to finish */
2069 if (i == dev_priv->num_fence_regs) {
d7619c4b 2070 uint32_t seqno = dev_priv->mm.next_gem_seqno;
de151cf6
JB		 2071 loff_t offset;
2072
fc7170ba
CW		 2073 if (avail == 0)
2074 return -ENOMEM;
2075
de151cf6
JB		 2076 for (i = dev_priv->fence_reg_start;
2077 i < dev_priv->num_fence_regs; i++) {
d7619c4b
CW		 2078 uint32_t this_seqno;
2079
de151cf6
JB		 2080 reg = &dev_priv->fence_regs[i];
2081 old_obj_priv = reg->obj->driver_private;
d7619c4b
CW		 2082
2083 if (old_obj_priv->pin_count)
2084 continue;
2085
2086 /* i915 uses fences for GPU access to tiled buffers */
2087 if (IS_I965G(dev) || !old_obj_priv->active)
de151cf6 2088 break;
d7619c4b
CW		 2089
2090 /* find the seqno of the first available fence */
2091 this_seqno = old_obj_priv->last_rendering_seqno;
2092 if (this_seqno != 0 &&
2093 reg->obj->write_domain == 0 &&
2094 i915_seqno_passed(seqno, this_seqno))
2095 seqno = this_seqno;
de151cf6
JB		 2096 }
2097
2098 /*
2099 * Now things get ugly... we have to wait for one of the
2100 * objects to finish before trying again.
2101 */
2102 if (i == dev_priv->num_fence_regs) {
d7619c4b
CW		 2103 if (seqno == dev_priv->mm.next_gem_seqno) {
2104 i915_gem_flush(dev,
2105 I915_GEM_GPU_DOMAINS,
2106 I915_GEM_GPU_DOMAINS);
2107 seqno = i915_add_request(dev,
2108 I915_GEM_GPU_DOMAINS);
2109 if (seqno == 0)
2110 return -ENOMEM;
de151cf6 2111 }
d7619c4b
CW		 2112
2113 ret = i915_wait_request(dev, seqno);
2114 if (ret)
2115 return ret;
de151cf6
JB		 2116 goto try_again;
2117 }
2118
d7619c4b
CW		 2119 BUG_ON(old_obj_priv->active ||
2120 (reg->obj->write_domain & I915_GEM_GPU_DOMAINS));
2121
de151cf6
JB		 2122 /*
2123 * Zap this virtual mapping so we can set up a fence again
2124 * for this object next time we need it.
2125 */
2126 offset = ((loff_t) reg->obj->map_list.hash.key) << PAGE_SHIFT;
79e53945
JB		 2127 if (dev->dev_mapping)
2128 unmap_mapping_range(dev->dev_mapping, offset,
2129 reg->obj->size, 1);
de151cf6
JB		 2130 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
2131 }
2132
2133 obj_priv->fence_reg = i;
2134 reg->obj = obj;
2135
2136 if (IS_I965G(dev))
2137 i965_write_fence_reg(reg);
2138 else if (IS_I9XX(dev))
2139 i915_write_fence_reg(reg);
2140 else
2141 i830_write_fence_reg(reg);
d9ddcb96
EA		 2142
2143 return 0;
de151cf6
JB		 2144}
2145
2146/**
2147 * i915_gem_clear_fence_reg - clear out fence register info
2148 * @obj: object to clear
2149 *
2150 * Zeroes out the fence register itself and clears out the associated
2151 * data structures in dev_priv and obj_priv.
2152 */
2153static void
2154i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2155{
2156 struct drm_device *dev = obj->dev;
79e53945 2157 drm_i915_private_t *dev_priv = dev->dev_private;
de151cf6
JB		 2158 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2159
2160 if (IS_I965G(dev))
2161 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
dc529a4f
EA		 2162 else {
2163 uint32_t fence_reg;
2164
2165 if (obj_priv->fence_reg < 8)
2166 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2167 else
2168 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2169 8) * 4;
2170
2171 I915_WRITE(fence_reg, 0);
2172 }
de151cf6
JB		 2173
2174 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2175 obj_priv->fence_reg = I915_FENCE_REG_NONE;
2176}
2177
673a394b
EA		 2178/**
2179 * Finds free space in the GTT aperture and binds the object there.
2180 */
2181static int
2182i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2183{
2184 struct drm_device *dev = obj->dev;
2185 drm_i915_private_t *dev_priv = dev->dev_private;
2186 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2187 struct drm_mm_node *free_space;
2188 int page_count, ret;
2189
9bb2d6f9
EA		 2190 if (dev_priv->mm.suspended)
2191 return -EBUSY;
673a394b 2192 if (alignment == 0)
0f973f27 2193 alignment = i915_gem_get_gtt_alignment(obj);
673a394b
EA		 2194 if (alignment & (PAGE_SIZE - 1)) {
2195 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2196 return -EINVAL;
2197 }
2198
2199 search_free:
2200 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2201 obj->size, alignment, 0);
2202 if (free_space != NULL) {
2203 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2204 alignment);
2205 if (obj_priv->gtt_space != NULL) {
2206 obj_priv->gtt_space->private = obj;
2207 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2208 }
2209 }
2210 if (obj_priv->gtt_space == NULL) {
2211 /* If the gtt is empty and we're still having trouble
2212 * fitting our object in, we're out of memory.
2213 */
2214#if WATCH_LRU
2215 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2216#endif
2217 if (list_empty(&dev_priv->mm.inactive_list) &&
2218 list_empty(&dev_priv->mm.flushing_list) &&
2219 list_empty(&dev_priv->mm.active_list)) {
2220 DRM_ERROR("GTT full, but LRU list empty\n");
2221 return -ENOMEM;
2222 }
2223
2224 ret = i915_gem_evict_something(dev);
2225 if (ret != 0) {
ac94a962
KP		 2226 if (ret != -ERESTARTSYS)
2227 DRM_ERROR("Failed to evict a buffer %d\n", ret);
673a394b
EA		 2228 return ret;
2229 }
2230 goto search_free;
2231 }
2232
2233#if WATCH_BUF
2234 DRM_INFO("Binding object of size %d at 0x%08x\n",
2235 obj->size, obj_priv->gtt_offset);
2236#endif
856fa198 2237 ret = i915_gem_object_get_pages(obj);
673a394b
EA		 2238 if (ret) {
2239 drm_mm_put_block(obj_priv->gtt_space);
2240 obj_priv->gtt_space = NULL;
2241 return ret;
2242 }
2243
2244 page_count = obj->size / PAGE_SIZE;
2245 /* Create an AGP memory structure pointing at our pages, and bind it
2246 * into the GTT.
2247 */
2248 obj_priv->agp_mem = drm_agp_bind_pages(dev,
856fa198 2249 obj_priv->pages,
673a394b 2250 page_count,
ba1eb1d8
KP		 2251 obj_priv->gtt_offset,
2252 obj_priv->agp_type);
673a394b 2253 if (obj_priv->agp_mem == NULL) {
856fa198 2254 i915_gem_object_put_pages(obj);
673a394b
EA		 2255 drm_mm_put_block(obj_priv->gtt_space);
2256 obj_priv->gtt_space = NULL;
2257 return -ENOMEM;
2258 }
2259 atomic_inc(&dev->gtt_count);
2260 atomic_add(obj->size, &dev->gtt_memory);
2261
2262 /* Assert that the object is not currently in any GPU domain. As it
2263 * wasn't in the GTT, there shouldn't be any way it could have been in
2264 * a GPU cache
2265 */
2266 BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2267 BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2268
2269 return 0;
2270}
2271
2272void
2273i915_gem_clflush_object(struct drm_gem_object *obj)
2274{
2275 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2276
2277 /* If we don't have a page list set up, then we're not pinned
2278 * to GPU, and we can ignore the cache flush because it'll happen
2279 * again at bind time.
2280 */
856fa198 2281 if (obj_priv->pages == NULL)
673a394b
EA		 2282 return;
2283
856fa198 2284 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
673a394b
EA		 2285}
2286
e47c68e9
EA		 2287/** Flushes any GPU write domain for the object if it's dirty. */
2288static void
2289i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2290{
2291 struct drm_device *dev = obj->dev;
2292 uint32_t seqno;
2293
2294 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2295 return;
2296
2297 /* Queue the GPU write cache flushing we need. */
2298 i915_gem_flush(dev, 0, obj->write_domain);
2299 seqno = i915_add_request(dev, obj->write_domain);
2300 obj->write_domain = 0;
2301 i915_gem_object_move_to_active(obj, seqno);
2302}
2303
2304/** Flushes the GTT write domain for the object if it's dirty. */
2305static void
2306i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2307{
2308 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2309 return;
2310
2311 /* No actual flushing is required for the GTT write domain. Writes
2312 * to it immediately go to main memory as far as we know, so there's
2313 * no chipset flush. It also doesn't land in render cache.
2314 */
2315 obj->write_domain = 0;
2316}
2317
2318/** Flushes the CPU write domain for the object if it's dirty. */
2319static void
2320i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2321{
2322 struct drm_device *dev = obj->dev;
2323
2324 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2325 return;
2326
2327 i915_gem_clflush_object(obj);
2328 drm_agp_chipset_flush(dev);
2329 obj->write_domain = 0;
2330}
2331
2ef7eeaa
EA		 2332/**
2333 * Moves a single object to the GTT read, and possibly write domain.
2334 *
2335 * This function returns when the move is complete, including waiting on
2336 * flushes to occur.
2337 */
79e53945 2338int
2ef7eeaa
EA		 2339i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2340{
2ef7eeaa 2341 struct drm_i915_gem_object *obj_priv = obj->driver_private;
e47c68e9 2342 int ret;
2ef7eeaa 2343
02354392
EA		 2344 /* Not valid to be called on unbound objects. */
2345 if (obj_priv->gtt_space == NULL)
2346 return -EINVAL;
2347
e47c68e9
EA		 2348 i915_gem_object_flush_gpu_write_domain(obj);
2349 /* Wait on any GPU rendering and flushing to occur. */
2350 ret = i915_gem_object_wait_rendering(obj);
2351 if (ret != 0)
2352 return ret;
2353
2354 /* If we're writing through the GTT domain, then CPU and GPU caches
2355 * will need to be invalidated at next use.
2ef7eeaa 2356 */
e47c68e9
EA		 2357 if (write)
2358 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2ef7eeaa 2359
e47c68e9 2360 i915_gem_object_flush_cpu_write_domain(obj);
2ef7eeaa 2361
e47c68e9
EA		 2362 /* It should now be out of any other write domains, and we can update
2363 * the domain values for our changes.
2364 */
2365 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2366 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2367 if (write) {
2368 obj->write_domain = I915_GEM_DOMAIN_GTT;
2369 obj_priv->dirty = 1;
2ef7eeaa
EA		 2370 }
2371
e47c68e9
EA		 2372 return 0;
2373}
2374
2375/**
2376 * Moves a single object to the CPU read, and possibly write domain.
2377 *
2378 * This function returns when the move is complete, including waiting on
2379 * flushes to occur.
2380 */
2381static int
2382i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2383{
e47c68e9
EA		 2384 int ret;
2385
2386 i915_gem_object_flush_gpu_write_domain(obj);
2ef7eeaa 2387 /* Wait on any GPU rendering and flushing to occur. */
e47c68e9
EA		 2388 ret = i915_gem_object_wait_rendering(obj);
2389 if (ret != 0)
2390 return ret;
2ef7eeaa 2391
e47c68e9 2392 i915_gem_object_flush_gtt_write_domain(obj);
2ef7eeaa 2393
e47c68e9
EA		 2394 /* If we have a partially-valid cache of the object in the CPU,
2395 * finish invalidating it and free the per-page flags.
2ef7eeaa 2396 */
e47c68e9 2397 i915_gem_object_set_to_full_cpu_read_domain(obj);
2ef7eeaa 2398
e47c68e9
EA		 2399 /* Flush the CPU cache if it's still invalid. */
2400 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2ef7eeaa 2401 i915_gem_clflush_object(obj);
2ef7eeaa 2402
e47c68e9 2403 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2ef7eeaa
EA		 2404 }
2405
2406 /* It should now be out of any other write domains, and we can update
2407 * the domain values for our changes.
2408 */
e47c68e9
EA		 2409 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2410
2411 /* If we're writing through the CPU, then the GPU read domains will
2412 * need to be invalidated at next use.
2413 */
2414 if (write) {
2415 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2416 obj->write_domain = I915_GEM_DOMAIN_CPU;
2417 }
2ef7eeaa
EA		 2418
2419 return 0;
2420}
2421
673a394b
EA		 2422/*
2423 * Set the next domain for the specified object. This
2424 * may not actually perform the necessary flushing/invaliding though,
2425 * as that may want to be batched with other set_domain operations
2426 *
2427 * This is (we hope) the only really tricky part of gem. The goal
2428 * is fairly simple -- track which caches hold bits of the object
2429 * and make sure they remain coherent. A few concrete examples may
2430 * help to explain how it works. For shorthand, we use the notation
2431 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2432 * a pair of read and write domain masks.
2433 *
2434 * Case 1: the batch buffer
2435 *
2436 * 1. Allocated
2437 * 2. Written by CPU
2438 * 3. Mapped to GTT
2439 * 4. Read by GPU
2440 * 5. Unmapped from GTT
2441 * 6. Freed
2442 *
2443 * Let's take these a step at a time
2444 *
2445 * 1. Allocated
2446 * Pages allocated from the kernel may still have
2447 * cache contents, so we set them to (CPU, CPU) always.
2448 * 2. Written by CPU (using pwrite)
2449 * The pwrite function calls set_domain (CPU, CPU) and
2450 * this function does nothing (as nothing changes)
2451 * 3. Mapped by GTT
2452 * This function asserts that the object is not
2453 * currently in any GPU-based read or write domains
2454 * 4. Read by GPU
2455 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2456 * As write_domain is zero, this function adds in the
2457 * current read domains (CPU+COMMAND, 0).
2458 * flush_domains is set to CPU.
2459 * invalidate_domains is set to COMMAND
2460 * clflush is run to get data out of the CPU caches
2461 * then i915_dev_set_domain calls i915_gem_flush to
2462 * emit an MI_FLUSH and drm_agp_chipset_flush
2463 * 5. Unmapped from GTT
2464 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2465 * flush_domains and invalidate_domains end up both zero
2466 * so no flushing/invalidating happens
2467 * 6. Freed
2468 * yay, done
2469 *
2470 * Case 2: The shared render buffer
2471 *
2472 * 1. Allocated
2473 * 2. Mapped to GTT
2474 * 3. Read/written by GPU
2475 * 4. set_domain to (CPU,CPU)
2476 * 5. Read/written by CPU
2477 * 6. Read/written by GPU
2478 *
2479 * 1. Allocated
2480 * Same as last example, (CPU, CPU)
2481 * 2. Mapped to GTT
2482 * Nothing changes (assertions find that it is not in the GPU)
2483 * 3. Read/written by GPU
2484 * execbuffer calls set_domain (RENDER, RENDER)
2485 * flush_domains gets CPU
2486 * invalidate_domains gets GPU
2487 * clflush (obj)
2488 * MI_FLUSH and drm_agp_chipset_flush
2489 * 4. set_domain (CPU, CPU)
2490 * flush_domains gets GPU
2491 * invalidate_domains gets CPU
2492 * wait_rendering (obj) to make sure all drawing is complete.
2493 * This will include an MI_FLUSH to get the data from GPU
2494 * to memory
2495 * clflush (obj) to invalidate the CPU cache
2496 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2497 * 5. Read/written by CPU
2498 * cache lines are loaded and dirtied
2499 * 6. Read written by GPU
2500 * Same as last GPU access
2501 *
2502 * Case 3: The constant buffer
2503 *
2504 * 1. Allocated
2505 * 2. Written by CPU
2506 * 3. Read by GPU
2507 * 4. Updated (written) by CPU again
2508 * 5. Read by GPU
2509 *
2510 * 1. Allocated
2511 * (CPU, CPU)
2512 * 2. Written by CPU
2513 * (CPU, CPU)
2514 * 3. Read by GPU
2515 * (CPU+RENDER, 0)
2516 * flush_domains = CPU
2517 * invalidate_domains = RENDER
2518 * clflush (obj)
2519 * MI_FLUSH
2520 * drm_agp_chipset_flush
2521 * 4. Updated (written) by CPU again
2522 * (CPU, CPU)
2523 * flush_domains = 0 (no previous write domain)
2524 * invalidate_domains = 0 (no new read domains)
2525 * 5. Read by GPU
2526 * (CPU+RENDER, 0)
2527 * flush_domains = CPU
2528 * invalidate_domains = RENDER
2529 * clflush (obj)
2530 * MI_FLUSH
2531 * drm_agp_chipset_flush
2532 */
c0d90829 2533static void
8b0e378a 2534i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
673a394b
EA		 2535{
2536 struct drm_device *dev = obj->dev;
2537 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2538 uint32_t invalidate_domains = 0;
2539 uint32_t flush_domains = 0;
e47c68e9 2540
8b0e378a
EA		 2541 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2542 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
673a394b
EA		 2543
2544#if WATCH_BUF
2545 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2546 __func__, obj,
8b0e378a
EA		 2547 obj->read_domains, obj->pending_read_domains,
2548 obj->write_domain, obj->pending_write_domain);
673a394b
EA		 2549#endif
2550 /*
2551 * If the object isn't moving to a new write domain,
2552 * let the object stay in multiple read domains
2553 */
8b0e378a
EA		 2554 if (obj->pending_write_domain == 0)
2555 obj->pending_read_domains |= obj->read_domains;
673a394b
EA		 2556 else
2557 obj_priv->dirty = 1;
2558
2559 /*
2560 * Flush the current write domain if
2561 * the new read domains don't match. Invalidate
2562 * any read domains which differ from the old
2563 * write domain
2564 */
8b0e378a
EA		 2565 if (obj->write_domain &&
2566 obj->write_domain != obj->pending_read_domains) {
673a394b 2567 flush_domains |= obj->write_domain;
8b0e378a
EA		 2568 invalidate_domains |=
2569 obj->pending_read_domains & ~obj->write_domain;
673a394b
EA		 2570 }
2571 /*
2572 * Invalidate any read caches which may have
2573 * stale data. That is, any new read domains.
2574 */
8b0e378a 2575 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
673a394b
EA		 2576 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
2577#if WATCH_BUF
2578 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
2579 __func__, flush_domains, invalidate_domains);
2580#endif
673a394b
EA		 2581 i915_gem_clflush_object(obj);
2582 }
2583
efbeed96
EA		 2584 /* The actual obj->write_domain will be updated with
2585 * pending_write_domain after we emit the accumulated flush for all
2586 * of our domain changes in execbuffers (which clears objects'
2587 * write_domains). So if we have a current write domain that we
2588 * aren't changing, set pending_write_domain to that.
2589 */
2590 if (flush_domains == 0 && obj->pending_write_domain == 0)
2591 obj->pending_write_domain = obj->write_domain;
8b0e378a 2592 obj->read_domains = obj->pending_read_domains;
673a394b
EA		 2593
2594 dev->invalidate_domains |= invalidate_domains;
2595 dev->flush_domains |= flush_domains;
2596#if WATCH_BUF
2597 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
2598 __func__,
2599 obj->read_domains, obj->write_domain,
2600 dev->invalidate_domains, dev->flush_domains);
2601#endif
673a394b
EA		 2602}
2603
2604/**
e47c68e9 2605 * Moves the object from a partially CPU read to a full one.
673a394b 2606 *
e47c68e9
EA		 2607 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
2608 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
673a394b 2609 */
e47c68e9
EA		 2610static void
2611i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
673a394b
EA		 2612{
2613 struct drm_i915_gem_object *obj_priv = obj->driver_private;
673a394b 2614
e47c68e9
EA		 2615 if (!obj_priv->page_cpu_valid)
2616 return;
2617
2618 /* If we're partially in the CPU read domain, finish moving it in.
2619 */
2620 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
2621 int i;
2622
2623 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
2624 if (obj_priv->page_cpu_valid[i])
2625 continue;
856fa198 2626 drm_clflush_pages(obj_priv->pages + i, 1);
e47c68e9 2627 }
e47c68e9
EA		 2628 }
2629
2630 /* Free the page_cpu_valid mappings which are now stale, whether
2631 * or not we've got I915_GEM_DOMAIN_CPU.
2632 */
2633 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
2634 DRM_MEM_DRIVER);
2635 obj_priv->page_cpu_valid = NULL;
2636}
2637
2638/**
2639 * Set the CPU read domain on a range of the object.
2640 *
2641 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
2642 * not entirely valid. The page_cpu_valid member of the object flags which
2643 * pages have been flushed, and will be respected by
2644 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
2645 * of the whole object.
2646 *
2647 * This function returns when the move is complete, including waiting on
2648 * flushes to occur.
2649 */
2650static int
2651i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
2652 uint64_t offset, uint64_t size)
2653{
2654 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2655 int i, ret;
673a394b 2656
e47c68e9
EA		 2657 if (offset == 0 && size == obj->size)
2658 return i915_gem_object_set_to_cpu_domain(obj, 0);
673a394b 2659
e47c68e9
EA		 2660 i915_gem_object_flush_gpu_write_domain(obj);
2661 /* Wait on any GPU rendering and flushing to occur. */
6a47baa6 2662 ret = i915_gem_object_wait_rendering(obj);
e47c68e9 2663 if (ret != 0)
6a47baa6 2664 return ret;
e47c68e9
EA		 2665 i915_gem_object_flush_gtt_write_domain(obj);
2666
2667 /* If we're already fully in the CPU read domain, we're done. */
2668 if (obj_priv->page_cpu_valid == NULL &&
2669 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
2670 return 0;
673a394b 2671
e47c68e9
EA		 2672 /* Otherwise, create/clear the per-page CPU read domain flag if we're
2673 * newly adding I915_GEM_DOMAIN_CPU
2674 */
673a394b
EA		 2675 if (obj_priv->page_cpu_valid == NULL) {
2676 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
2677 DRM_MEM_DRIVER);
e47c68e9
EA		 2678 if (obj_priv->page_cpu_valid == NULL)
2679 return -ENOMEM;
2680 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
2681 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
673a394b
EA		 2682
2683 /* Flush the cache on any pages that are still invalid from the CPU's
2684 * perspective.
2685 */
e47c68e9
EA		 2686 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
2687 i++) {
673a394b
EA		 2688 if (obj_priv->page_cpu_valid[i])
2689 continue;
2690
856fa198 2691 drm_clflush_pages(obj_priv->pages + i, 1);
673a394b
EA		 2692
2693 obj_priv->page_cpu_valid[i] = 1;
2694 }
2695
e47c68e9
EA		 2696 /* It should now be out of any other write domains, and we can update
2697 * the domain values for our changes.
2698 */
2699 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2700
2701 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2702
673a394b
EA		 2703 return 0;
2704}
2705
673a394b
EA		 2706/**
2707 * Pin an object to the GTT and evaluate the relocations landing in it.
2708 */
2709static int
2710i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
2711 struct drm_file *file_priv,
40a5f0de
EA		 2712 struct drm_i915_gem_exec_object *entry,
2713 struct drm_i915_gem_relocation_entry *relocs)
673a394b
EA		 2714{
2715 struct drm_device *dev = obj->dev;
0839ccb8 2716 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 2717 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2718 int i, ret;
0839ccb8 2719 void __iomem *reloc_page;
673a394b
EA		 2720
2721 /* Choose the GTT offset for our buffer and put it there. */
2722 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
2723 if (ret)
2724 return ret;
2725
2726 entry->offset = obj_priv->gtt_offset;
2727
673a394b
EA		 2728 /* Apply the relocations, using the GTT aperture to avoid cache
2729 * flushing requirements.
2730 */
2731 for (i = 0; i < entry->relocation_count; i++) {
40a5f0de 2732 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
673a394b
EA		 2733 struct drm_gem_object *target_obj;
2734 struct drm_i915_gem_object *target_obj_priv;
3043c60c
EA		 2735 uint32_t reloc_val, reloc_offset;
2736 uint32_t __iomem *reloc_entry;
673a394b 2737
673a394b 2738 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
40a5f0de 2739 reloc->target_handle);
673a394b
EA		 2740 if (target_obj == NULL) {
2741 i915_gem_object_unpin(obj);
2742 return -EBADF;
2743 }
2744 target_obj_priv = target_obj->driver_private;
2745
2746 /* The target buffer should have appeared before us in the
2747 * exec_object list, so it should have a GTT space bound by now.
2748 */
2749 if (target_obj_priv->gtt_space == NULL) {
2750 DRM_ERROR("No GTT space found for object %d\n",
40a5f0de 2751 reloc->target_handle);
673a394b
EA		 2752 drm_gem_object_unreference(target_obj);
2753 i915_gem_object_unpin(obj);
2754 return -EINVAL;
2755 }
2756
40a5f0de 2757 if (reloc->offset > obj->size - 4) {
673a394b
EA		 2758 DRM_ERROR("Relocation beyond object bounds: "
2759 "obj %p target %d offset %d size %d.\n",
40a5f0de
EA		 2760 obj, reloc->target_handle,
2761 (int) reloc->offset, (int) obj->size);
673a394b
EA		 2762 drm_gem_object_unreference(target_obj);
2763 i915_gem_object_unpin(obj);
2764 return -EINVAL;
2765 }
40a5f0de 2766 if (reloc->offset & 3) {
673a394b
EA		 2767 DRM_ERROR("Relocation not 4-byte aligned: "
2768 "obj %p target %d offset %d.\n",
40a5f0de
EA		 2769 obj, reloc->target_handle,
2770 (int) reloc->offset);
673a394b
EA		 2771 drm_gem_object_unreference(target_obj);
2772 i915_gem_object_unpin(obj);
2773 return -EINVAL;
2774 }
2775
40a5f0de
EA		 2776 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
2777 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
e47c68e9
EA		 2778 DRM_ERROR("reloc with read/write CPU domains: "
2779 "obj %p target %d offset %d "
2780 "read %08x write %08x",
40a5f0de
EA		 2781 obj, reloc->target_handle,
2782 (int) reloc->offset,
2783 reloc->read_domains,
2784 reloc->write_domain);
491152b8
CW		 2785 drm_gem_object_unreference(target_obj);
2786 i915_gem_object_unpin(obj);
e47c68e9
EA		 2787 return -EINVAL;
2788 }
2789
40a5f0de
EA		 2790 if (reloc->write_domain && target_obj->pending_write_domain &&
2791 reloc->write_domain != target_obj->pending_write_domain) {
673a394b
EA		 2792 DRM_ERROR("Write domain conflict: "
2793 "obj %p target %d offset %d "
2794 "new %08x old %08x\n",
40a5f0de
EA		 2795 obj, reloc->target_handle,
2796 (int) reloc->offset,
2797 reloc->write_domain,
673a394b
EA		 2798 target_obj->pending_write_domain);
2799 drm_gem_object_unreference(target_obj);
2800 i915_gem_object_unpin(obj);
2801 return -EINVAL;
2802 }
2803
2804#if WATCH_RELOC
2805 DRM_INFO("%s: obj %p offset %08x target %d "
2806 "read %08x write %08x gtt %08x "
2807 "presumed %08x delta %08x\n",
2808 __func__,
2809 obj,
40a5f0de
EA		 2810 (int) reloc->offset,
2811 (int) reloc->target_handle,
2812 (int) reloc->read_domains,
2813 (int) reloc->write_domain,
673a394b 2814 (int) target_obj_priv->gtt_offset,
40a5f0de
EA		 2815 (int) reloc->presumed_offset,
2816 reloc->delta);
673a394b
EA		 2817#endif
2818
40a5f0de
EA		 2819 target_obj->pending_read_domains |= reloc->read_domains;
2820 target_obj->pending_write_domain |= reloc->write_domain;
673a394b
EA		 2821
2822 /* If the relocation already has the right value in it, no
2823 * more work needs to be done.
2824 */
40a5f0de 2825 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
673a394b
EA		 2826 drm_gem_object_unreference(target_obj);
2827 continue;
2828 }
2829
2ef7eeaa
EA		 2830 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
2831 if (ret != 0) {
2832 drm_gem_object_unreference(target_obj);
2833 i915_gem_object_unpin(obj);
2834 return -EINVAL;
673a394b
EA		 2835 }
2836
2837 /* Map the page containing the relocation we're going to
2838 * perform.
2839 */
40a5f0de 2840 reloc_offset = obj_priv->gtt_offset + reloc->offset;
0839ccb8
KP		 2841 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
2842 (reloc_offset &
2843 ~(PAGE_SIZE - 1)));
3043c60c 2844 reloc_entry = (uint32_t __iomem *)(reloc_page +
0839ccb8 2845 (reloc_offset & (PAGE_SIZE - 1)));
40a5f0de 2846 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
673a394b
EA		 2847
2848#if WATCH_BUF
2849 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
40a5f0de 2850 obj, (unsigned int) reloc->offset,
673a394b
EA		 2851 readl(reloc_entry), reloc_val);
2852#endif
2853 writel(reloc_val, reloc_entry);
0839ccb8 2854 io_mapping_unmap_atomic(reloc_page);
673a394b 2855
40a5f0de
EA		 2856 /* The updated presumed offset for this entry will be
2857 * copied back out to the user.
673a394b 2858 */
40a5f0de 2859 reloc->presumed_offset = target_obj_priv->gtt_offset;
673a394b
EA		 2860
2861 drm_gem_object_unreference(target_obj);
2862 }
2863
673a394b
EA		 2864#if WATCH_BUF
2865 if (0)
2866 i915_gem_dump_object(obj, 128, __func__, ~0);
2867#endif
2868 return 0;
2869}
2870
2871/** Dispatch a batchbuffer to the ring
2872 */
2873static int
2874i915_dispatch_gem_execbuffer(struct drm_device *dev,
2875 struct drm_i915_gem_execbuffer *exec,
201361a5 2876 struct drm_clip_rect *cliprects,
673a394b
EA		 2877 uint64_t exec_offset)
2878{
2879 drm_i915_private_t *dev_priv = dev->dev_private;
673a394b
EA		 2880 int nbox = exec->num_cliprects;
2881 int i = 0, count;
2882 uint32_t exec_start, exec_len;
2883 RING_LOCALS;
2884
2885 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
2886 exec_len = (uint32_t) exec->batch_len;
2887
2888 if ((exec_start | exec_len) & 0x7) {
2889 DRM_ERROR("alignment\n");
2890 return -EINVAL;
2891 }
2892
2893 if (!exec_start)
2894 return -EINVAL;
2895
2896 count = nbox ? nbox : 1;
2897
2898 for (i = 0; i < count; i++) {
2899 if (i < nbox) {
201361a5 2900 int ret = i915_emit_box(dev, cliprects, i,
673a394b
EA		 2901 exec->DR1, exec->DR4);
2902 if (ret)
2903 return ret;
2904 }
2905
2906 if (IS_I830(dev) || IS_845G(dev)) {
2907 BEGIN_LP_RING(4);
2908 OUT_RING(MI_BATCH_BUFFER);
2909 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
2910 OUT_RING(exec_start + exec_len - 4);
2911 OUT_RING(0);
2912 ADVANCE_LP_RING();
2913 } else {
2914 BEGIN_LP_RING(2);
2915 if (IS_I965G(dev)) {
2916 OUT_RING(MI_BATCH_BUFFER_START |
2917 (2 << 6) |
2918 MI_BATCH_NON_SECURE_I965);
2919 OUT_RING(exec_start);
2920 } else {
2921 OUT_RING(MI_BATCH_BUFFER_START |
2922 (2 << 6));
2923 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
2924 }
2925 ADVANCE_LP_RING();
2926 }
2927 }
2928
2929 /* XXX breadcrumb */
2930 return 0;
2931}
2932
2933/* Throttle our rendering by waiting until the ring has completed our requests
2934 * emitted over 20 msec ago.
2935 *
2936 * This should get us reasonable parallelism between CPU and GPU but also
2937 * relatively low latency when blocking on a particular request to finish.
2938 */
2939static int
2940i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
2941{
2942 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
2943 int ret = 0;
2944 uint32_t seqno;
2945
2946 mutex_lock(&dev->struct_mutex);
2947 seqno = i915_file_priv->mm.last_gem_throttle_seqno;
2948 i915_file_priv->mm.last_gem_throttle_seqno =
2949 i915_file_priv->mm.last_gem_seqno;
2950 if (seqno)
2951 ret = i915_wait_request(dev, seqno);
2952 mutex_unlock(&dev->struct_mutex);
2953 return ret;
2954}
2955
40a5f0de
EA		 2956static int
2957i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object *exec_list,
2958 uint32_t buffer_count,
2959 struct drm_i915_gem_relocation_entry **relocs)
2960{
2961 uint32_t reloc_count = 0, reloc_index = 0, i;
2962 int ret;
2963
2964 *relocs = NULL;
2965 for (i = 0; i < buffer_count; i++) {
2966 if (reloc_count + exec_list[i].relocation_count < reloc_count)
2967 return -EINVAL;
2968 reloc_count += exec_list[i].relocation_count;
2969 }
2970
2971 *relocs = drm_calloc(reloc_count, sizeof(**relocs), DRM_MEM_DRIVER);
2972 if (*relocs == NULL)
2973 return -ENOMEM;
2974
2975 for (i = 0; i < buffer_count; i++) {
2976 struct drm_i915_gem_relocation_entry __user *user_relocs;
2977
2978 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
2979
2980 ret = copy_from_user(&(*relocs)[reloc_index],
2981 user_relocs,
2982 exec_list[i].relocation_count *
2983 sizeof(**relocs));
2984 if (ret != 0) {
2985 drm_free(*relocs, reloc_count * sizeof(**relocs),
2986 DRM_MEM_DRIVER);
2987 *relocs = NULL;
2988 return ret;
2989 }
2990
2991 reloc_index += exec_list[i].relocation_count;
2992 }
2993
2994 return ret;
2995}
2996
2997static int
2998i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object *exec_list,
2999 uint32_t buffer_count,
3000 struct drm_i915_gem_relocation_entry *relocs)
3001{
3002 uint32_t reloc_count = 0, i;
3003 int ret;
3004
3005 for (i = 0; i < buffer_count; i++) {
3006 struct drm_i915_gem_relocation_entry __user *user_relocs;
3007
3008 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3009
3010 if (ret == 0) {
3011 ret = copy_to_user(user_relocs,
3012 &relocs[reloc_count],
3013 exec_list[i].relocation_count *
3014 sizeof(*relocs));
3015 }
3016
3017 reloc_count += exec_list[i].relocation_count;
3018 }
3019
3020 drm_free(relocs, reloc_count * sizeof(*relocs), DRM_MEM_DRIVER);
3021
3022 return ret;
3023}
3024
673a394b
EA		 3025int
3026i915_gem_execbuffer(struct drm_device *dev, void *data,
3027 struct drm_file *file_priv)
3028{
3029 drm_i915_private_t *dev_priv = dev->dev_private;
3030 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3031 struct drm_i915_gem_execbuffer *args = data;
3032 struct drm_i915_gem_exec_object *exec_list = NULL;
3033 struct drm_gem_object **object_list = NULL;
3034 struct drm_gem_object *batch_obj;
b70d11da 3035 struct drm_i915_gem_object *obj_priv;
201361a5 3036 struct drm_clip_rect *cliprects = NULL;
40a5f0de
EA		 3037 struct drm_i915_gem_relocation_entry *relocs;
3038 int ret, ret2, i, pinned = 0;
673a394b 3039 uint64_t exec_offset;
40a5f0de 3040 uint32_t seqno, flush_domains, reloc_index;
ac94a962 3041 int pin_tries;
673a394b
EA		 3042
3043#if WATCH_EXEC
3044 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3045 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3046#endif
3047
4f481ed2
EA		 3048 if (args->buffer_count < 1) {
3049 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3050 return -EINVAL;
3051 }
673a394b
EA		 3052 /* Copy in the exec list from userland */
3053 exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
3054 DRM_MEM_DRIVER);
3055 object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
3056 DRM_MEM_DRIVER);
3057 if (exec_list == NULL || object_list == NULL) {
3058 DRM_ERROR("Failed to allocate exec or object list "
3059 "for %d buffers\n",
3060 args->buffer_count);
3061 ret = -ENOMEM;
3062 goto pre_mutex_err;
3063 }
3064 ret = copy_from_user(exec_list,
3065 (struct drm_i915_relocation_entry __user *)
3066 (uintptr_t) args->buffers_ptr,
3067 sizeof(*exec_list) * args->buffer_count);
3068 if (ret != 0) {
3069 DRM_ERROR("copy %d exec entries failed %d\n",
3070 args->buffer_count, ret);
3071 goto pre_mutex_err;
3072 }
3073
201361a5
EA		 3074 if (args->num_cliprects != 0) {
3075 cliprects = drm_calloc(args->num_cliprects, sizeof(*cliprects),
3076 DRM_MEM_DRIVER);
3077 if (cliprects == NULL)
3078 goto pre_mutex_err;
3079
3080 ret = copy_from_user(cliprects,
3081 (struct drm_clip_rect __user *)
3082 (uintptr_t) args->cliprects_ptr,
3083 sizeof(*cliprects) * args->num_cliprects);
3084 if (ret != 0) {
3085 DRM_ERROR("copy %d cliprects failed: %d\n",
3086 args->num_cliprects, ret);
3087 goto pre_mutex_err;
3088 }
3089 }
3090
40a5f0de
EA		 3091 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3092 &relocs);
3093 if (ret != 0)
3094 goto pre_mutex_err;
3095
673a394b
EA		 3096 mutex_lock(&dev->struct_mutex);
3097
3098 i915_verify_inactive(dev, __FILE__, __LINE__);
3099
3100 if (dev_priv->mm.wedged) {
3101 DRM_ERROR("Execbuf while wedged\n");
3102 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3103 ret = -EIO;
3104 goto pre_mutex_err;
673a394b
EA		 3105 }
3106
3107 if (dev_priv->mm.suspended) {
3108 DRM_ERROR("Execbuf while VT-switched.\n");
3109 mutex_unlock(&dev->struct_mutex);
a198bc80
CW		 3110 ret = -EBUSY;
3111 goto pre_mutex_err;
673a394b
EA		 3112 }
3113
ac94a962 3114 /* Look up object handles */
673a394b
EA		 3115 for (i = 0; i < args->buffer_count; i++) {
3116 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3117 exec_list[i].handle);
3118 if (object_list[i] == NULL) {
3119 DRM_ERROR("Invalid object handle %d at index %d\n",
3120 exec_list[i].handle, i);
3121 ret = -EBADF;
3122 goto err;
3123 }
b70d11da
KH		 3124
3125 obj_priv = object_list[i]->driver_private;
3126 if (obj_priv->in_execbuffer) {
3127 DRM_ERROR("Object %p appears more than once in object list\n",
3128 object_list[i]);
3129 ret = -EBADF;
3130 goto err;
3131 }
3132 obj_priv->in_execbuffer = true;
ac94a962 3133 }
673a394b 3134
ac94a962
KP		 3135 /* Pin and relocate */
3136 for (pin_tries = 0; ; pin_tries++) {
3137 ret = 0;
40a5f0de
EA		 3138 reloc_index = 0;
3139
ac94a962
KP		 3140 for (i = 0; i < args->buffer_count; i++) {
3141 object_list[i]->pending_read_domains = 0;
3142 object_list[i]->pending_write_domain = 0;
3143 ret = i915_gem_object_pin_and_relocate(object_list[i],
3144 file_priv,
40a5f0de
EA		 3145 &exec_list[i],
3146 &relocs[reloc_index]);
ac94a962
KP		 3147 if (ret)
3148 break;
3149 pinned = i + 1;
40a5f0de 3150 reloc_index += exec_list[i].relocation_count;
ac94a962
KP		 3151 }
3152 /* success */
3153 if (ret == 0)
3154 break;
3155
3156 /* error other than GTT full, or we've already tried again */
3157 if (ret != -ENOMEM || pin_tries >= 1) {
f1acec93
EA		 3158 if (ret != -ERESTARTSYS)
3159 DRM_ERROR("Failed to pin buffers %d\n", ret);
673a394b
EA		 3160 goto err;
3161 }
ac94a962
KP		 3162
3163 /* unpin all of our buffers */
3164 for (i = 0; i < pinned; i++)
3165 i915_gem_object_unpin(object_list[i]);
b1177636 3166 pinned = 0;
ac94a962
KP		 3167
3168 /* evict everyone we can from the aperture */
3169 ret = i915_gem_evict_everything(dev);
3170 if (ret)
3171 goto err;
673a394b
EA		 3172 }
3173
3174 /* Set the pending read domains for the batch buffer to COMMAND */
3175 batch_obj = object_list[args->buffer_count-1];
3176 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
3177 batch_obj->pending_write_domain = 0;
3178
3179 i915_verify_inactive(dev, __FILE__, __LINE__);
3180
646f0f6e
KP		 3181 /* Zero the global flush/invalidate flags. These
3182 * will be modified as new domains are computed
3183 * for each object
3184 */
3185 dev->invalidate_domains = 0;
3186 dev->flush_domains = 0;
3187
673a394b
EA		 3188 for (i = 0; i < args->buffer_count; i++) {
3189 struct drm_gem_object *obj = object_list[i];
673a394b 3190
646f0f6e 3191 /* Compute new gpu domains and update invalidate/flush */
8b0e378a 3192 i915_gem_object_set_to_gpu_domain(obj);
673a394b
EA		 3193 }
3194
3195 i915_verify_inactive(dev, __FILE__, __LINE__);
3196
646f0f6e
KP		 3197 if (dev->invalidate_domains | dev->flush_domains) {
3198#if WATCH_EXEC
3199 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3200 __func__,
3201 dev->invalidate_domains,
3202 dev->flush_domains);
3203#endif
3204 i915_gem_flush(dev,
3205 dev->invalidate_domains,
3206 dev->flush_domains);
3207 if (dev->flush_domains)
3208 (void)i915_add_request(dev, dev->flush_domains);
3209 }
673a394b 3210
efbeed96
EA		 3211 for (i = 0; i < args->buffer_count; i++) {
3212 struct drm_gem_object *obj = object_list[i];
3213
3214 obj->write_domain = obj->pending_write_domain;
3215 }
3216
673a394b
EA		 3217 i915_verify_inactive(dev, __FILE__, __LINE__);
3218
3219#if WATCH_COHERENCY
3220 for (i = 0; i < args->buffer_count; i++) {
3221 i915_gem_object_check_coherency(object_list[i],
3222 exec_list[i].handle);
3223 }
3224#endif
3225
3226 exec_offset = exec_list[args->buffer_count - 1].offset;
3227
3228#if WATCH_EXEC
3229 i915_gem_dump_object(object_list[args->buffer_count - 1],
3230 args->batch_len,
3231 __func__,
3232 ~0);
3233#endif
3234
673a394b 3235 /* Exec the batchbuffer */
201361a5 3236 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
673a394b
EA		 3237 if (ret) {
3238 DRM_ERROR("dispatch failed %d\n", ret);
3239 goto err;
3240 }
3241
3242 /*
3243 * Ensure that the commands in the batch buffer are
3244 * finished before the interrupt fires
3245 */
3246 flush_domains = i915_retire_commands(dev);
3247
3248 i915_verify_inactive(dev, __FILE__, __LINE__);
3249
3250 /*
3251 * Get a seqno representing the execution of the current buffer,
3252 * which we can wait on. We would like to mitigate these interrupts,
3253 * likely by only creating seqnos occasionally (so that we have
3254 * *some* interrupts representing completion of buffers that we can
3255 * wait on when trying to clear up gtt space).
3256 */
3257 seqno = i915_add_request(dev, flush_domains);
3258 BUG_ON(seqno == 0);
3259 i915_file_priv->mm.last_gem_seqno = seqno;
3260 for (i = 0; i < args->buffer_count; i++) {
3261 struct drm_gem_object *obj = object_list[i];
673a394b 3262
ce44b0ea 3263 i915_gem_object_move_to_active(obj, seqno);
673a394b
EA		 3264#if WATCH_LRU
3265 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3266#endif
3267 }
3268#if WATCH_LRU
3269 i915_dump_lru(dev, __func__);
3270#endif
3271
3272 i915_verify_inactive(dev, __FILE__, __LINE__);
3273
673a394b 3274err:
aad87dff
JL		 3275 for (i = 0; i < pinned; i++)
3276 i915_gem_object_unpin(object_list[i]);
3277
b70d11da
KH		 3278 for (i = 0; i < args->buffer_count; i++) {
3279 if (object_list[i]) {
3280 obj_priv = object_list[i]->driver_private;
3281 obj_priv->in_execbuffer = false;
3282 }
aad87dff 3283 drm_gem_object_unreference(object_list[i]);
b70d11da 3284 }
673a394b 3285
673a394b
EA		 3286 mutex_unlock(&dev->struct_mutex);
3287
a35f2e2b
RD		 3288 if (!ret) {
3289 /* Copy the new buffer offsets back to the user's exec list. */
3290 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3291 (uintptr_t) args->buffers_ptr,
3292 exec_list,
3293 sizeof(*exec_list) * args->buffer_count);
3294 if (ret)
3295 DRM_ERROR("failed to copy %d exec entries "
3296 "back to user (%d)\n",
3297 args->buffer_count, ret);
3298 }
3299
40a5f0de
EA		 3300 /* Copy the updated relocations out regardless of current error
3301 * state. Failure to update the relocs would mean that the next
3302 * time userland calls execbuf, it would do so with presumed offset
3303 * state that didn't match the actual object state.
3304 */
3305 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3306 relocs);
3307 if (ret2 != 0) {
3308 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3309
3310 if (ret == 0)
3311 ret = ret2;
3312 }
3313
673a394b
EA		 3314pre_mutex_err:
3315 drm_free(object_list, sizeof(*object_list) * args->buffer_count,
3316 DRM_MEM_DRIVER);
3317 drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
3318 DRM_MEM_DRIVER);
201361a5
EA		 3319 drm_free(cliprects, sizeof(*cliprects) * args->num_cliprects,
3320 DRM_MEM_DRIVER);
673a394b
EA		 3321
3322 return ret;
3323}
3324
3325int
3326i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
3327{
3328 struct drm_device *dev = obj->dev;
3329 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3330 int ret;
3331
3332 i915_verify_inactive(dev, __FILE__, __LINE__);
3333 if (obj_priv->gtt_space == NULL) {
3334 ret = i915_gem_object_bind_to_gtt(obj, alignment);
3335 if (ret != 0) {
9bb2d6f9 3336 if (ret != -EBUSY && ret != -ERESTARTSYS)
0fce81e3 3337 DRM_ERROR("Failure to bind: %d\n", ret);
673a394b
EA		 3338 return ret;
3339 }
22c344e9
CW		 3340 }
3341 /*
3342 * Pre-965 chips need a fence register set up in order to
3343 * properly handle tiled surfaces.
3344 */
3345 if (!IS_I965G(dev) &&
3346 obj_priv->fence_reg == I915_FENCE_REG_NONE &&
3347 obj_priv->tiling_mode != I915_TILING_NONE) {
3348 ret = i915_gem_object_get_fence_reg(obj, true);
3349 if (ret != 0) {
3350 if (ret != -EBUSY && ret != -ERESTARTSYS)
3351 DRM_ERROR("Failure to install fence: %d\n",
3352 ret);
3353 return ret;
3354 }
673a394b
EA		 3355 }
3356 obj_priv->pin_count++;
3357
3358 /* If the object is not active and not pending a flush,
3359 * remove it from the inactive list
3360 */
3361 if (obj_priv->pin_count == 1) {
3362 atomic_inc(&dev->pin_count);
3363 atomic_add(obj->size, &dev->pin_memory);
3364 if (!obj_priv->active &&
3365 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
3366 I915_GEM_DOMAIN_GTT)) == 0 &&
3367 !list_empty(&obj_priv->list))
3368 list_del_init(&obj_priv->list);
3369 }
3370 i915_verify_inactive(dev, __FILE__, __LINE__);
3371
3372 return 0;
3373}
3374
3375void
3376i915_gem_object_unpin(struct drm_gem_object *obj)
3377{
3378 struct drm_device *dev = obj->dev;
3379 drm_i915_private_t *dev_priv = dev->dev_private;
3380 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3381
3382 i915_verify_inactive(dev, __FILE__, __LINE__);
3383 obj_priv->pin_count--;
3384 BUG_ON(obj_priv->pin_count < 0);
3385 BUG_ON(obj_priv->gtt_space == NULL);
3386
3387 /* If the object is no longer pinned, and is
3388 * neither active nor being flushed, then stick it on
3389 * the inactive list
3390 */
3391 if (obj_priv->pin_count == 0) {
3392 if (!obj_priv->active &&
3393 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
3394 I915_GEM_DOMAIN_GTT)) == 0)
3395 list_move_tail(&obj_priv->list,
3396 &dev_priv->mm.inactive_list);
3397 atomic_dec(&dev->pin_count);
3398 atomic_sub(obj->size, &dev->pin_memory);
3399 }
3400 i915_verify_inactive(dev, __FILE__, __LINE__);
3401}
3402
3403int
3404i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3405 struct drm_file *file_priv)
3406{
3407 struct drm_i915_gem_pin *args = data;
3408 struct drm_gem_object *obj;
3409 struct drm_i915_gem_object *obj_priv;
3410 int ret;
3411
3412 mutex_lock(&dev->struct_mutex);
3413
3414 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3415 if (obj == NULL) {
3416 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
3417 args->handle);
3418 mutex_unlock(&dev->struct_mutex);
3419 return -EBADF;
3420 }
3421 obj_priv = obj->driver_private;
3422
79e53945
JB		 3423 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
3424 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3425 args->handle);
96dec61d 3426 drm_gem_object_unreference(obj);
673a394b 3427 mutex_unlock(&dev->struct_mutex);
79e53945
JB		 3428 return -EINVAL;
3429 }
3430
3431 obj_priv->user_pin_count++;
3432 obj_priv->pin_filp = file_priv;
3433 if (obj_priv->user_pin_count == 1) {
3434 ret = i915_gem_object_pin(obj, args->alignment);
3435 if (ret != 0) {
3436 drm_gem_object_unreference(obj);
3437 mutex_unlock(&dev->struct_mutex);
3438 return ret;
3439 }
673a394b
EA		 3440 }
3441
3442 /* XXX - flush the CPU caches for pinned objects
3443 * as the X server doesn't manage domains yet
3444 */
e47c68e9 3445 i915_gem_object_flush_cpu_write_domain(obj);
673a394b
EA		 3446 args->offset = obj_priv->gtt_offset;
3447 drm_gem_object_unreference(obj);
3448 mutex_unlock(&dev->struct_mutex);
3449
3450 return 0;
3451}
3452
3453int
3454i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3455 struct drm_file *file_priv)
3456{
3457 struct drm_i915_gem_pin *args = data;
3458 struct drm_gem_object *obj;
79e53945 3459 struct drm_i915_gem_object *obj_priv;
673a394b
EA		 3460
3461 mutex_lock(&dev->struct_mutex);
3462
3463 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3464 if (obj == NULL) {
3465 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
3466 args->handle);
3467 mutex_unlock(&dev->struct_mutex);
3468 return -EBADF;
3469 }
3470
79e53945
JB		 3471 obj_priv = obj->driver_private;
3472 if (obj_priv->pin_filp != file_priv) {
3473 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3474 args->handle);
3475 drm_gem_object_unreference(obj);
3476 mutex_unlock(&dev->struct_mutex);
3477 return -EINVAL;
3478 }
3479 obj_priv->user_pin_count--;
3480 if (obj_priv->user_pin_count == 0) {
3481 obj_priv->pin_filp = NULL;
3482 i915_gem_object_unpin(obj);
3483 }
673a394b
EA		 3484
3485 drm_gem_object_unreference(obj);
3486 mutex_unlock(&dev->struct_mutex);
3487 return 0;
3488}
3489
3490int
3491i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3492 struct drm_file *file_priv)
3493{
3494 struct drm_i915_gem_busy *args = data;
3495 struct drm_gem_object *obj;
3496 struct drm_i915_gem_object *obj_priv;
3497
3498 mutex_lock(&dev->struct_mutex);
3499 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
3500 if (obj == NULL) {
3501 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
3502 args->handle);
3503 mutex_unlock(&dev->struct_mutex);
3504 return -EBADF;
3505 }
3506
f21289b3
EA		 3507 /* Update the active list for the hardware's current position.
3508 * Otherwise this only updates on a delayed timer or when irqs are
3509 * actually unmasked, and our working set ends up being larger than
3510 * required.
3511 */
3512 i915_gem_retire_requests(dev);
3513
673a394b 3514 obj_priv = obj->driver_private;
c4de0a5d
EA		 3515 /* Don't count being on the flushing list against the object being
3516 * done. Otherwise, a buffer left on the flushing list but not getting
3517 * flushed (because nobody's flushing that domain) won't ever return
3518 * unbusy and get reused by libdrm's bo cache. The other expected
3519 * consumer of this interface, OpenGL's occlusion queries, also specs
3520 * that the objects get unbusy "eventually" without any interference.
3521 */
3522 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
673a394b
EA		 3523
3524 drm_gem_object_unreference(obj);
3525 mutex_unlock(&dev->struct_mutex);
3526 return 0;
3527}
3528
3529int
3530i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3531 struct drm_file *file_priv)
3532{
3533 return i915_gem_ring_throttle(dev, file_priv);
3534}
3535
3536int i915_gem_init_object(struct drm_gem_object *obj)
3537{
3538 struct drm_i915_gem_object *obj_priv;
3539
3540 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
3541 if (obj_priv == NULL)
3542 return -ENOMEM;
3543
3544 /*
3545 * We've just allocated pages from the kernel,
3546 * so they've just been written by the CPU with
3547 * zeros. They'll need to be clflushed before we
3548 * use them with the GPU.
3549 */
3550 obj->write_domain = I915_GEM_DOMAIN_CPU;
3551 obj->read_domains = I915_GEM_DOMAIN_CPU;
3552
ba1eb1d8
KP		 3553 obj_priv->agp_type = AGP_USER_MEMORY;
3554
673a394b
EA		 3555 obj->driver_private = obj_priv;
3556 obj_priv->obj = obj;
de151cf6 3557 obj_priv->fence_reg = I915_FENCE_REG_NONE;
673a394b 3558 INIT_LIST_HEAD(&obj_priv->list);
de151cf6 3559
673a394b
EA		 3560 return 0;
3561}
3562
3563void i915_gem_free_object(struct drm_gem_object *obj)
3564{
de151cf6 3565 struct drm_device *dev = obj->dev;
673a394b
EA		 3566 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3567
3568 while (obj_priv->pin_count > 0)
3569 i915_gem_object_unpin(obj);
3570
71acb5eb
DA		 3571 if (obj_priv->phys_obj)
3572 i915_gem_detach_phys_object(dev, obj);
3573
673a394b
EA		 3574 i915_gem_object_unbind(obj);
3575
ab00b3e5 3576 i915_gem_free_mmap_offset(obj);
de151cf6 3577
673a394b
EA		 3578 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
3579 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
3580}
3581
673a394b
EA		 3582/** Unbinds all objects that are on the given buffer list. */
3583static int
3584i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
3585{
3586 struct drm_gem_object *obj;
3587 struct drm_i915_gem_object *obj_priv;
3588 int ret;
3589
3590 while (!list_empty(head)) {
3591 obj_priv = list_first_entry(head,
3592 struct drm_i915_gem_object,
3593 list);
3594 obj = obj_priv->obj;
3595
3596 if (obj_priv->pin_count != 0) {
3597 DRM_ERROR("Pinned object in unbind list\n");
3598 mutex_unlock(&dev->struct_mutex);
3599 return -EINVAL;
3600 }
3601
3602 ret = i915_gem_object_unbind(obj);
3603 if (ret != 0) {
3604 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
3605 ret);
3606 mutex_unlock(&dev->struct_mutex);
3607 return ret;
3608 }
3609 }
3610
3611
3612 return 0;
3613}
3614
5669fcac 3615int
673a394b
EA		 3616i915_gem_idle(struct drm_device *dev)
3617{
3618 drm_i915_private_t *dev_priv = dev->dev_private;
3619 uint32_t seqno, cur_seqno, last_seqno;
3620 int stuck, ret;
3621
6dbe2772
KP		 3622 mutex_lock(&dev->struct_mutex);
3623
3624 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
3625 mutex_unlock(&dev->struct_mutex);
673a394b 3626 return 0;
6dbe2772 3627 }
673a394b
EA		 3628
3629 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3630 * We need to replace this with a semaphore, or something.
3631 */
3632 dev_priv->mm.suspended = 1;
3633
6dbe2772
KP		 3634 /* Cancel the retire work handler, wait for it to finish if running
3635 */
3636 mutex_unlock(&dev->struct_mutex);
3637 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3638 mutex_lock(&dev->struct_mutex);
3639
673a394b
EA		 3640 i915_kernel_lost_context(dev);
3641
3642 /* Flush the GPU along with all non-CPU write domains
3643 */
3644 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
3645 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
de151cf6 3646 seqno = i915_add_request(dev, ~I915_GEM_DOMAIN_CPU);
673a394b
EA		 3647
3648 if (seqno == 0) {
3649 mutex_unlock(&dev->struct_mutex);
3650 return -ENOMEM;
3651 }
3652
3653 dev_priv->mm.waiting_gem_seqno = seqno;
3654 last_seqno = 0;
3655 stuck = 0;
3656 for (;;) {
3657 cur_seqno = i915_get_gem_seqno(dev);
3658 if (i915_seqno_passed(cur_seqno, seqno))
3659 break;
3660 if (last_seqno == cur_seqno) {
3661 if (stuck++ > 100) {
3662 DRM_ERROR("hardware wedged\n");
3663 dev_priv->mm.wedged = 1;
3664 DRM_WAKEUP(&dev_priv->irq_queue);
3665 break;
3666 }
3667 }
3668 msleep(10);
3669 last_seqno = cur_seqno;
3670 }
3671 dev_priv->mm.waiting_gem_seqno = 0;
3672
3673 i915_gem_retire_requests(dev);
3674
28dfe52a
EA		 3675 if (!dev_priv->mm.wedged) {
3676 /* Active and flushing should now be empty as we've
3677 * waited for a sequence higher than any pending execbuffer
3678 */
3679 WARN_ON(!list_empty(&dev_priv->mm.active_list));
3680 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
3681 /* Request should now be empty as we've also waited
3682 * for the last request in the list
3683 */
3684 WARN_ON(!list_empty(&dev_priv->mm.request_list));
3685 }
673a394b 3686
28dfe52a
EA		 3687 /* Empty the active and flushing lists to inactive. If there's
3688 * anything left at this point, it means that we're wedged and
3689 * nothing good's going to happen by leaving them there. So strip
3690 * the GPU domains and just stuff them onto inactive.
673a394b 3691 */
28dfe52a
EA		 3692 while (!list_empty(&dev_priv->mm.active_list)) {
3693 struct drm_i915_gem_object *obj_priv;
673a394b 3694
28dfe52a
EA		 3695 obj_priv = list_first_entry(&dev_priv->mm.active_list,
3696 struct drm_i915_gem_object,
3697 list);
3698 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3699 i915_gem_object_move_to_inactive(obj_priv->obj);
3700 }
3701
3702 while (!list_empty(&dev_priv->mm.flushing_list)) {
3703 struct drm_i915_gem_object *obj_priv;
3704
151903d5 3705 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
28dfe52a
EA		 3706 struct drm_i915_gem_object,
3707 list);
3708 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
3709 i915_gem_object_move_to_inactive(obj_priv->obj);
3710 }
3711
3712
3713 /* Move all inactive buffers out of the GTT. */
673a394b 3714 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
28dfe52a 3715 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
6dbe2772
KP		 3716 if (ret) {
3717 mutex_unlock(&dev->struct_mutex);
673a394b 3718 return ret;
6dbe2772 3719 }
673a394b 3720
6dbe2772
KP		 3721 i915_gem_cleanup_ringbuffer(dev);
3722 mutex_unlock(&dev->struct_mutex);
3723
673a394b
EA		 3724 return 0;
3725}
3726
3727static int
3728i915_gem_init_hws(struct drm_device *dev)
3729{
3730 drm_i915_private_t *dev_priv = dev->dev_private;
3731 struct drm_gem_object *obj;
3732 struct drm_i915_gem_object *obj_priv;
3733 int ret;
3734
3735 /* If we need a physical address for the status page, it's already
3736 * initialized at driver load time.
3737 */
3738 if (!I915_NEED_GFX_HWS(dev))
3739 return 0;
3740
3741 obj = drm_gem_object_alloc(dev, 4096);
3742 if (obj == NULL) {
3743 DRM_ERROR("Failed to allocate status page\n");
3744 return -ENOMEM;
3745 }
3746 obj_priv = obj->driver_private;
ba1eb1d8 3747 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
673a394b
EA		 3748
3749 ret = i915_gem_object_pin(obj, 4096);
3750 if (ret != 0) {
3751 drm_gem_object_unreference(obj);
3752 return ret;
3753 }
3754
3755 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
673a394b 3756
856fa198 3757 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
ba1eb1d8 3758 if (dev_priv->hw_status_page == NULL) {
673a394b
EA		 3759 DRM_ERROR("Failed to map status page.\n");
3760 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
3eb2ee77 3761 i915_gem_object_unpin(obj);
673a394b
EA		 3762 drm_gem_object_unreference(obj);
3763 return -EINVAL;
3764 }
3765 dev_priv->hws_obj = obj;
673a394b
EA		 3766 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
3767 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
ba1eb1d8 3768 I915_READ(HWS_PGA); /* posting read */
673a394b
EA		 3769 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
3770
3771 return 0;
3772}
3773
85a7bb98
CW		 3774static void
3775i915_gem_cleanup_hws(struct drm_device *dev)
3776{
3777 drm_i915_private_t *dev_priv = dev->dev_private;
bab2d1f6
CW		 3778 struct drm_gem_object *obj;
3779 struct drm_i915_gem_object *obj_priv;
85a7bb98
CW		 3780
3781 if (dev_priv->hws_obj == NULL)
3782 return;
3783
bab2d1f6
CW		 3784 obj = dev_priv->hws_obj;
3785 obj_priv = obj->driver_private;
3786
856fa198 3787 kunmap(obj_priv->pages[0]);
85a7bb98
CW		 3788 i915_gem_object_unpin(obj);
3789 drm_gem_object_unreference(obj);
3790 dev_priv->hws_obj = NULL;
bab2d1f6 3791
85a7bb98
CW		 3792 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
3793 dev_priv->hw_status_page = NULL;
3794
3795 /* Write high address into HWS_PGA when disabling. */
3796 I915_WRITE(HWS_PGA, 0x1ffff000);
3797}
3798
79e53945 3799int
673a394b
EA		 3800i915_gem_init_ringbuffer(struct drm_device *dev)
3801{
3802 drm_i915_private_t *dev_priv = dev->dev_private;
3803 struct drm_gem_object *obj;
3804 struct drm_i915_gem_object *obj_priv;
79e53945 3805 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
673a394b 3806 int ret;
50aa253d 3807 u32 head;
673a394b
EA		 3808
3809 ret = i915_gem_init_hws(dev);
3810 if (ret != 0)
3811 return ret;
3812
3813 obj = drm_gem_object_alloc(dev, 128 * 1024);
3814 if (obj == NULL) {
3815 DRM_ERROR("Failed to allocate ringbuffer\n");
85a7bb98 3816 i915_gem_cleanup_hws(dev);
673a394b
EA		 3817 return -ENOMEM;
3818 }
3819 obj_priv = obj->driver_private;
3820
3821 ret = i915_gem_object_pin(obj, 4096);
3822 if (ret != 0) {
3823 drm_gem_object_unreference(obj);
85a7bb98 3824 i915_gem_cleanup_hws(dev);
673a394b
EA		 3825 return ret;
3826 }
3827
3828 /* Set up the kernel mapping for the ring. */
79e53945
JB		 3829 ring->Size = obj->size;
3830 ring->tail_mask = obj->size - 1;
673a394b 3831
79e53945
JB		 3832 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
3833 ring->map.size = obj->size;
3834 ring->map.type = 0;
3835 ring->map.flags = 0;
3836 ring->map.mtrr = 0;
673a394b 3837
79e53945
JB		 3838 drm_core_ioremap_wc(&ring->map, dev);
3839 if (ring->map.handle == NULL) {
673a394b
EA		 3840 DRM_ERROR("Failed to map ringbuffer.\n");
3841 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
47ed185a 3842 i915_gem_object_unpin(obj);
673a394b 3843 drm_gem_object_unreference(obj);
85a7bb98 3844 i915_gem_cleanup_hws(dev);
673a394b
EA		 3845 return -EINVAL;
3846 }
79e53945
JB		 3847 ring->ring_obj = obj;
3848 ring->virtual_start = ring->map.handle;
673a394b
EA		 3849
3850 /* Stop the ring if it's running. */
3851 I915_WRITE(PRB0_CTL, 0);
673a394b 3852 I915_WRITE(PRB0_TAIL, 0);
50aa253d 3853 I915_WRITE(PRB0_HEAD, 0);
673a394b
EA		 3854
3855 /* Initialize the ring. */
3856 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
50aa253d
KP		 3857 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
3858
3859 /* G45 ring initialization fails to reset head to zero */
3860 if (head != 0) {
3861 DRM_ERROR("Ring head not reset to zero "
3862 "ctl %08x head %08x tail %08x start %08x\n",
3863 I915_READ(PRB0_CTL),
3864 I915_READ(PRB0_HEAD),
3865 I915_READ(PRB0_TAIL),
3866 I915_READ(PRB0_START));
3867 I915_WRITE(PRB0_HEAD, 0);
3868
3869 DRM_ERROR("Ring head forced to zero "
3870 "ctl %08x head %08x tail %08x start %08x\n",
3871 I915_READ(PRB0_CTL),
3872 I915_READ(PRB0_HEAD),
3873 I915_READ(PRB0_TAIL),
3874 I915_READ(PRB0_START));
3875 }
3876
673a394b
EA		 3877 I915_WRITE(PRB0_CTL,
3878 ((obj->size - 4096) & RING_NR_PAGES) |
3879 RING_NO_REPORT |
3880 RING_VALID);
3881
50aa253d
KP		 3882 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
3883
3884 /* If the head is still not zero, the ring is dead */
3885 if (head != 0) {
3886 DRM_ERROR("Ring initialization failed "
3887 "ctl %08x head %08x tail %08x start %08x\n",
3888 I915_READ(PRB0_CTL),
3889 I915_READ(PRB0_HEAD),
3890 I915_READ(PRB0_TAIL),
3891 I915_READ(PRB0_START));
3892 return -EIO;
3893 }
3894
673a394b 3895 /* Update our cache of the ring state */
79e53945
JB		 3896 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3897 i915_kernel_lost_context(dev);
3898 else {
3899 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
3900 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
3901 ring->space = ring->head - (ring->tail + 8);
3902 if (ring->space < 0)
3903 ring->space += ring->Size;
3904 }
673a394b
EA		 3905
3906 return 0;
3907}
3908
79e53945 3909void
673a394b
EA		 3910i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3911{
3912 drm_i915_private_t *dev_priv = dev->dev_private;
3913
3914 if (dev_priv->ring.ring_obj == NULL)
3915 return;
3916
3917 drm_core_ioremapfree(&dev_priv->ring.map, dev);
3918
3919 i915_gem_object_unpin(dev_priv->ring.ring_obj);
3920 drm_gem_object_unreference(dev_priv->ring.ring_obj);
3921 dev_priv->ring.ring_obj = NULL;
3922 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
3923
85a7bb98 3924 i915_gem_cleanup_hws(dev);
673a394b
EA		 3925}
3926
3927int
3928i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3929 struct drm_file *file_priv)
3930{
3931 drm_i915_private_t *dev_priv = dev->dev_private;
3932 int ret;
3933
79e53945
JB		 3934 if (drm_core_check_feature(dev, DRIVER_MODESET))
3935 return 0;
3936
673a394b
EA		 3937 if (dev_priv->mm.wedged) {
3938 DRM_ERROR("Reenabling wedged hardware, good luck\n");
3939 dev_priv->mm.wedged = 0;
3940 }
3941
673a394b 3942 mutex_lock(&dev->struct_mutex);
9bb2d6f9
EA		 3943 dev_priv->mm.suspended = 0;
3944
3945 ret = i915_gem_init_ringbuffer(dev);
3946 if (ret != 0)
3947 return ret;
3948
673a394b
EA		 3949 BUG_ON(!list_empty(&dev_priv->mm.active_list));
3950 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3951 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3952 BUG_ON(!list_empty(&dev_priv->mm.request_list));
673a394b 3953 mutex_unlock(&dev->struct_mutex);
dbb19d30
KH		 3954
3955 drm_irq_install(dev);
3956
673a394b
EA		 3957 return 0;
3958}
3959
3960int
3961i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3962 struct drm_file *file_priv)
3963{
3964 int ret;
3965
79e53945
JB		 3966 if (drm_core_check_feature(dev, DRIVER_MODESET))
3967 return 0;
3968
673a394b 3969 ret = i915_gem_idle(dev);
dbb19d30
KH		 3970 drm_irq_uninstall(dev);
3971
6dbe2772 3972 return ret;
673a394b
EA		 3973}
3974
3975void
3976i915_gem_lastclose(struct drm_device *dev)
3977{
3978 int ret;
673a394b 3979
e806b495
EA		 3980 if (drm_core_check_feature(dev, DRIVER_MODESET))
3981 return;
3982
6dbe2772
KP		 3983 ret = i915_gem_idle(dev);
3984 if (ret)
3985 DRM_ERROR("failed to idle hardware: %d\n", ret);
673a394b
EA		 3986}
3987
3988void
3989i915_gem_load(struct drm_device *dev)
3990{
3991 drm_i915_private_t *dev_priv = dev->dev_private;
3992
3993 INIT_LIST_HEAD(&dev_priv->mm.active_list);
3994 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3995 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3996 INIT_LIST_HEAD(&dev_priv->mm.request_list);
3997 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3998 i915_gem_retire_work_handler);
3999 dev_priv->mm.next_gem_seqno = 1;
4000
de151cf6
JB		 4001 /* Old X drivers will take 0-2 for front, back, depth buffers */
4002 dev_priv->fence_reg_start = 3;
4003
0f973f27 4004 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
de151cf6
JB		 4005 dev_priv->num_fence_regs = 16;
4006 else
4007 dev_priv->num_fence_regs = 8;
4008
673a394b
EA		 4009 i915_gem_detect_bit_6_swizzle(dev);
4010}
71acb5eb
DA		 4011
4012/*
4013 * Create a physically contiguous memory object for this object
4014 * e.g. for cursor + overlay regs
4015 */
4016int i915_gem_init_phys_object(struct drm_device *dev,
4017 int id, int size)
4018{
4019 drm_i915_private_t *dev_priv = dev->dev_private;
4020 struct drm_i915_gem_phys_object *phys_obj;
4021 int ret;
4022
4023 if (dev_priv->mm.phys_objs[id - 1] || !size)
4024 return 0;
4025
4026 phys_obj = drm_calloc(1, sizeof(struct drm_i915_gem_phys_object), DRM_MEM_DRIVER);
4027 if (!phys_obj)
4028 return -ENOMEM;
4029
4030 phys_obj->id = id;
4031
4032 phys_obj->handle = drm_pci_alloc(dev, size, 0, 0xffffffff);
4033 if (!phys_obj->handle) {
4034 ret = -ENOMEM;
4035 goto kfree_obj;
4036 }
4037#ifdef CONFIG_X86
4038 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4039#endif
4040
4041 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4042
4043 return 0;
4044kfree_obj:
4045 drm_free(phys_obj, sizeof(struct drm_i915_gem_phys_object), DRM_MEM_DRIVER);
4046 return ret;
4047}
4048
4049void i915_gem_free_phys_object(struct drm_device *dev, int id)
4050{
4051 drm_i915_private_t *dev_priv = dev->dev_private;
4052 struct drm_i915_gem_phys_object *phys_obj;
4053
4054 if (!dev_priv->mm.phys_objs[id - 1])
4055 return;
4056
4057 phys_obj = dev_priv->mm.phys_objs[id - 1];
4058 if (phys_obj->cur_obj) {
4059 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4060 }
4061
4062#ifdef CONFIG_X86
4063 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4064#endif
4065 drm_pci_free(dev, phys_obj->handle);
4066 kfree(phys_obj);
4067 dev_priv->mm.phys_objs[id - 1] = NULL;
4068}
4069
4070void i915_gem_free_all_phys_object(struct drm_device *dev)
4071{
4072 int i;
4073
260883c8 4074 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
71acb5eb
DA		 4075 i915_gem_free_phys_object(dev, i);
4076}
4077
4078void i915_gem_detach_phys_object(struct drm_device *dev,
4079 struct drm_gem_object *obj)
4080{
4081 struct drm_i915_gem_object *obj_priv;
4082 int i;
4083 int ret;
4084 int page_count;
4085
4086 obj_priv = obj->driver_private;
4087 if (!obj_priv->phys_obj)
4088 return;
4089
856fa198 4090 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4091 if (ret)
4092 goto out;
4093
4094 page_count = obj->size / PAGE_SIZE;
4095
4096 for (i = 0; i < page_count; i++) {
856fa198 4097 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4098 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4099
4100 memcpy(dst, src, PAGE_SIZE);
4101 kunmap_atomic(dst, KM_USER0);
4102 }
856fa198 4103 drm_clflush_pages(obj_priv->pages, page_count);
71acb5eb
DA		 4104 drm_agp_chipset_flush(dev);
4105out:
4106 obj_priv->phys_obj->cur_obj = NULL;
4107 obj_priv->phys_obj = NULL;
4108}
4109
4110int
4111i915_gem_attach_phys_object(struct drm_device *dev,
4112 struct drm_gem_object *obj, int id)
4113{
4114 drm_i915_private_t *dev_priv = dev->dev_private;
4115 struct drm_i915_gem_object *obj_priv;
4116 int ret = 0;
4117 int page_count;
4118 int i;
4119
4120 if (id > I915_MAX_PHYS_OBJECT)
4121 return -EINVAL;
4122
4123 obj_priv = obj->driver_private;
4124
4125 if (obj_priv->phys_obj) {
4126 if (obj_priv->phys_obj->id == id)
4127 return 0;
4128 i915_gem_detach_phys_object(dev, obj);
4129 }
4130
4131
4132 /* create a new object */
4133 if (!dev_priv->mm.phys_objs[id - 1]) {
4134 ret = i915_gem_init_phys_object(dev, id,
4135 obj->size);
4136 if (ret) {
aeb565df 4137 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
71acb5eb
DA		 4138 goto out;
4139 }
4140 }
4141
4142 /* bind to the object */
4143 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4144 obj_priv->phys_obj->cur_obj = obj;
4145
856fa198 4146 ret = i915_gem_object_get_pages(obj);
71acb5eb
DA		 4147 if (ret) {
4148 DRM_ERROR("failed to get page list\n");
4149 goto out;
4150 }
4151
4152 page_count = obj->size / PAGE_SIZE;
4153
4154 for (i = 0; i < page_count; i++) {
856fa198 4155 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
71acb5eb
DA		 4156 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4157
4158 memcpy(dst, src, PAGE_SIZE);
4159 kunmap_atomic(src, KM_USER0);
4160 }
4161
4162 return 0;
4163out:
4164 return ret;
4165}
4166
4167static int
4168i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4169 struct drm_i915_gem_pwrite *args,
4170 struct drm_file *file_priv)
4171{
4172 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4173 void *obj_addr;
4174 int ret;
4175 char __user *user_data;
4176
4177 user_data = (char __user *) (uintptr_t) args->data_ptr;
4178 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4179
e08fb4f6 4180 DRM_DEBUG("obj_addr %p, %lld\n", obj_addr, args->size);
71acb5eb
DA		 4181 ret = copy_from_user(obj_addr, user_data, args->size);
4182 if (ret)
4183 return -EFAULT;
4184
4185 drm_agp_chipset_flush(dev);
4186 return 0;
4187}
Clone of davem's net-next-2.6 tree