2 * Copyright © 2008-2010 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
34 #include "i915_trace.h"
37 render_ring_flush(struct drm_device *dev,
38 struct intel_ring_buffer *ring,
39 u32 invalidate_domains,
43 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
44 invalidate_domains, flush_domains);
47 trace_i915_gem_request_flush(dev, ring->next_seqno,
48 invalidate_domains, flush_domains);
50 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
54 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
55 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
56 * also flushed at 2d versus 3d pipeline switches.
60 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
61 * MI_READ_FLUSH is set, and is always flushed on 965.
63 * I915_GEM_DOMAIN_COMMAND may not exist?
65 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
66 * invalidated when MI_EXE_FLUSH is set.
68 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
69 * invalidated with every MI_FLUSH.
73 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
74 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
75 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
76 * are flushed at any MI_FLUSH.
79 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
80 if ((invalidate_domains|flush_domains) &
81 I915_GEM_DOMAIN_RENDER)
82 cmd &= ~MI_NO_WRITE_FLUSH;
85 * On the 965, the sampler cache always gets flushed
86 * and this bit is reserved.
88 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
91 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
95 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
97 intel_ring_begin(dev, ring, 8);
98 intel_ring_emit(dev, ring, cmd);
99 intel_ring_emit(dev, ring, MI_NOOP);
100 intel_ring_advance(dev, ring);
104 static unsigned int render_ring_get_head(struct drm_device *dev,
105 struct intel_ring_buffer *ring)
107 drm_i915_private_t *dev_priv = dev->dev_private;
108 return I915_READ(PRB0_HEAD) & HEAD_ADDR;
111 static unsigned int render_ring_get_tail(struct drm_device *dev,
112 struct intel_ring_buffer *ring)
114 drm_i915_private_t *dev_priv = dev->dev_private;
115 return I915_READ(PRB0_TAIL) & TAIL_ADDR;
118 static unsigned int render_ring_get_active_head(struct drm_device *dev,
119 struct intel_ring_buffer *ring)
121 drm_i915_private_t *dev_priv = dev->dev_private;
122 u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
124 return I915_READ(acthd_reg);
127 static void render_ring_advance_ring(struct drm_device *dev,
128 struct intel_ring_buffer *ring)
130 drm_i915_private_t *dev_priv = dev->dev_private;
131 I915_WRITE(PRB0_TAIL, ring->tail);
134 static int init_ring_common(struct drm_device *dev,
135 struct intel_ring_buffer *ring)
138 drm_i915_private_t *dev_priv = dev->dev_private;
139 struct drm_i915_gem_object *obj_priv;
140 obj_priv = to_intel_bo(ring->gem_object);
142 /* Stop the ring if it's running. */
143 I915_WRITE(ring->regs.ctl, 0);
144 I915_WRITE(ring->regs.head, 0);
145 I915_WRITE(ring->regs.tail, 0);
147 /* Initialize the ring. */
148 I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
149 head = ring->get_head(dev, ring);
151 /* G45 ring initialization fails to reset head to zero */
153 DRM_ERROR("%s head not reset to zero "
154 "ctl %08x head %08x tail %08x start %08x\n",
156 I915_READ(ring->regs.ctl),
157 I915_READ(ring->regs.head),
158 I915_READ(ring->regs.tail),
159 I915_READ(ring->regs.start));
161 I915_WRITE(ring->regs.head, 0);
163 DRM_ERROR("%s head forced to zero "
164 "ctl %08x head %08x tail %08x start %08x\n",
166 I915_READ(ring->regs.ctl),
167 I915_READ(ring->regs.head),
168 I915_READ(ring->regs.tail),
169 I915_READ(ring->regs.start));
172 I915_WRITE(ring->regs.ctl,
173 ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
174 | RING_NO_REPORT | RING_VALID);
176 head = I915_READ(ring->regs.head) & HEAD_ADDR;
177 /* If the head is still not zero, the ring is dead */
179 DRM_ERROR("%s initialization failed "
180 "ctl %08x head %08x tail %08x start %08x\n",
182 I915_READ(ring->regs.ctl),
183 I915_READ(ring->regs.head),
184 I915_READ(ring->regs.tail),
185 I915_READ(ring->regs.start));
189 if (!drm_core_check_feature(dev, DRIVER_MODESET))
190 i915_kernel_lost_context(dev);
192 ring->head = ring->get_head(dev, ring);
193 ring->tail = ring->get_tail(dev, ring);
194 ring->space = ring->head - (ring->tail + 8);
196 ring->space += ring->size;
201 static int init_render_ring(struct drm_device *dev,
202 struct intel_ring_buffer *ring)
204 drm_i915_private_t *dev_priv = dev->dev_private;
205 int ret = init_ring_common(dev, ring);
206 if (IS_I9XX(dev) && !IS_GEN3(dev)) {
208 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
213 #define PIPE_CONTROL_FLUSH(addr) \
215 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
216 PIPE_CONTROL_DEPTH_STALL); \
217 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \
223 * Creates a new sequence number, emitting a write of it to the status page
224 * plus an interrupt, which will trigger i915_user_interrupt_handler.
226 * Must be called with struct_lock held.
228 * Returned sequence numbers are nonzero on success.
231 render_ring_add_request(struct drm_device *dev,
232 struct intel_ring_buffer *ring,
233 struct drm_file *file_priv,
237 drm_i915_private_t *dev_priv = dev->dev_private;
238 seqno = intel_ring_get_seqno(dev, ring);
239 if (HAS_PIPE_CONTROL(dev)) {
240 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
243 * Workaround qword write incoherence by flushing the
244 * PIPE_NOTIFY buffers out to memory before requesting
248 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
249 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
250 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
253 PIPE_CONTROL_FLUSH(scratch_addr);
254 scratch_addr += 128; /* write to separate cachelines */
255 PIPE_CONTROL_FLUSH(scratch_addr);
257 PIPE_CONTROL_FLUSH(scratch_addr);
259 PIPE_CONTROL_FLUSH(scratch_addr);
261 PIPE_CONTROL_FLUSH(scratch_addr);
263 PIPE_CONTROL_FLUSH(scratch_addr);
264 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
265 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
266 PIPE_CONTROL_NOTIFY);
267 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
273 OUT_RING(MI_STORE_DWORD_INDEX);
274 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
277 OUT_RING(MI_USER_INTERRUPT);
284 render_ring_get_gem_seqno(struct drm_device *dev,
285 struct intel_ring_buffer *ring)
287 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
288 if (HAS_PIPE_CONTROL(dev))
289 return ((volatile u32 *)(dev_priv->seqno_page))[0];
291 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
295 render_ring_get_user_irq(struct drm_device *dev,
296 struct intel_ring_buffer *ring)
298 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
299 unsigned long irqflags;
301 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
302 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
303 if (HAS_PCH_SPLIT(dev))
304 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
306 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
308 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
312 render_ring_put_user_irq(struct drm_device *dev,
313 struct intel_ring_buffer *ring)
315 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
316 unsigned long irqflags;
318 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
319 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
320 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
321 if (HAS_PCH_SPLIT(dev))
322 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
324 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
326 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
329 static void render_setup_status_page(struct drm_device *dev,
330 struct intel_ring_buffer *ring)
332 drm_i915_private_t *dev_priv = dev->dev_private;
334 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
335 I915_READ(HWS_PGA_GEN6); /* posting read */
337 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
338 I915_READ(HWS_PGA); /* posting read */
344 render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
345 struct intel_ring_buffer *ring,
346 struct drm_i915_gem_execbuffer2 *exec,
347 struct drm_clip_rect *cliprects,
348 uint64_t exec_offset)
350 drm_i915_private_t *dev_priv = dev->dev_private;
351 int nbox = exec->num_cliprects;
353 uint32_t exec_start, exec_len;
354 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
355 exec_len = (uint32_t) exec->batch_len;
357 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
359 count = nbox ? nbox : 1;
361 for (i = 0; i < count; i++) {
363 int ret = i915_emit_box(dev, cliprects, i,
364 exec->DR1, exec->DR4);
369 if (IS_I830(dev) || IS_845G(dev)) {
370 intel_ring_begin(dev, ring, 4);
371 intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
372 intel_ring_emit(dev, ring,
373 exec_start | MI_BATCH_NON_SECURE);
374 intel_ring_emit(dev, ring, exec_start + exec_len - 4);
375 intel_ring_emit(dev, ring, 0);
377 intel_ring_begin(dev, ring, 4);
379 intel_ring_emit(dev, ring,
380 MI_BATCH_BUFFER_START | (2 << 6)
381 | MI_BATCH_NON_SECURE_I965);
382 intel_ring_emit(dev, ring, exec_start);
384 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
386 intel_ring_emit(dev, ring, exec_start |
387 MI_BATCH_NON_SECURE);
390 intel_ring_advance(dev, ring);
397 static void cleanup_status_page(struct drm_device *dev,
398 struct intel_ring_buffer *ring)
400 drm_i915_private_t *dev_priv = dev->dev_private;
401 struct drm_gem_object *obj;
402 struct drm_i915_gem_object *obj_priv;
404 obj = ring->status_page.obj;
407 obj_priv = to_intel_bo(obj);
409 kunmap(obj_priv->pages[0]);
410 i915_gem_object_unpin(obj);
411 drm_gem_object_unreference(obj);
412 ring->status_page.obj = NULL;
414 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
417 static int init_status_page(struct drm_device *dev,
418 struct intel_ring_buffer *ring)
420 drm_i915_private_t *dev_priv = dev->dev_private;
421 struct drm_gem_object *obj;
422 struct drm_i915_gem_object *obj_priv;
425 obj = i915_gem_alloc_object(dev, 4096);
427 DRM_ERROR("Failed to allocate status page\n");
431 obj_priv = to_intel_bo(obj);
432 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
434 ret = i915_gem_object_pin(obj, 4096);
439 ring->status_page.gfx_addr = obj_priv->gtt_offset;
440 ring->status_page.page_addr = kmap(obj_priv->pages[0]);
441 if (ring->status_page.page_addr == NULL) {
442 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
445 ring->status_page.obj = obj;
446 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
448 ring->setup_status_page(dev, ring);
449 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
450 ring->name, ring->status_page.gfx_addr);
455 i915_gem_object_unpin(obj);
457 drm_gem_object_unreference(obj);
463 int intel_init_ring_buffer(struct drm_device *dev,
464 struct intel_ring_buffer *ring)
467 struct drm_i915_gem_object *obj_priv;
468 struct drm_gem_object *obj;
471 if (I915_NEED_GFX_HWS(dev)) {
472 ret = init_status_page(dev, ring);
477 obj = i915_gem_alloc_object(dev, ring->size);
479 DRM_ERROR("Failed to allocate ringbuffer\n");
484 ring->gem_object = obj;
486 ret = i915_gem_object_pin(obj, ring->alignment);
488 drm_gem_object_unreference(obj);
492 obj_priv = to_intel_bo(obj);
493 ring->map.size = ring->size;
494 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
499 drm_core_ioremap_wc(&ring->map, dev);
500 if (ring->map.handle == NULL) {
501 DRM_ERROR("Failed to map ringbuffer.\n");
502 i915_gem_object_unpin(obj);
503 drm_gem_object_unreference(obj);
508 ring->virtual_start = ring->map.handle;
509 ret = ring->init(dev, ring);
511 intel_cleanup_ring_buffer(dev, ring);
515 if (!drm_core_check_feature(dev, DRIVER_MODESET))
516 i915_kernel_lost_context(dev);
518 ring->head = ring->get_head(dev, ring);
519 ring->tail = ring->get_tail(dev, ring);
520 ring->space = ring->head - (ring->tail + 8);
522 ring->space += ring->size;
524 INIT_LIST_HEAD(&ring->active_list);
525 INIT_LIST_HEAD(&ring->request_list);
528 cleanup_status_page(dev, ring);
532 void intel_cleanup_ring_buffer(struct drm_device *dev,
533 struct intel_ring_buffer *ring)
535 if (ring->gem_object == NULL)
538 drm_core_ioremapfree(&ring->map, dev);
540 i915_gem_object_unpin(ring->gem_object);
541 drm_gem_object_unreference(ring->gem_object);
542 ring->gem_object = NULL;
543 cleanup_status_page(dev, ring);
546 int intel_wrap_ring_buffer(struct drm_device *dev,
547 struct intel_ring_buffer *ring)
551 rem = ring->size - ring->tail;
553 if (ring->space < rem) {
554 int ret = intel_wait_ring_buffer(dev, ring, rem);
559 virt = (unsigned int *)(ring->virtual_start + ring->tail);
569 int intel_wait_ring_buffer(struct drm_device *dev,
570 struct intel_ring_buffer *ring, int n)
574 trace_i915_ring_wait_begin (dev);
575 end = jiffies + 3 * HZ;
577 ring->head = ring->get_head(dev, ring);
578 ring->space = ring->head - (ring->tail + 8);
580 ring->space += ring->size;
581 if (ring->space >= n) {
582 trace_i915_ring_wait_end (dev);
586 if (dev->primary->master) {
587 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
588 if (master_priv->sarea_priv)
589 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
592 } while (!time_after(jiffies, end));
593 trace_i915_ring_wait_end (dev);
597 void intel_ring_begin(struct drm_device *dev,
598 struct intel_ring_buffer *ring, int n)
600 if (unlikely(ring->tail + n > ring->size))
601 intel_wrap_ring_buffer(dev, ring);
602 if (unlikely(ring->space < n))
603 intel_wait_ring_buffer(dev, ring, n);
606 void intel_ring_emit(struct drm_device *dev,
607 struct intel_ring_buffer *ring, unsigned int data)
609 unsigned int *virt = ring->virtual_start + ring->tail;
612 ring->tail &= ring->size - 1;
616 void intel_ring_advance(struct drm_device *dev,
617 struct intel_ring_buffer *ring)
619 ring->advance_ring(dev, ring);
622 void intel_fill_struct(struct drm_device *dev,
623 struct intel_ring_buffer *ring,
627 unsigned int *virt = ring->virtual_start + ring->tail;
628 BUG_ON((len&~(4-1)) != 0);
629 intel_ring_begin(dev, ring, len);
630 memcpy(virt, data, len);
632 ring->tail &= ring->size - 1;
634 intel_ring_advance(dev, ring);
637 u32 intel_ring_get_seqno(struct drm_device *dev,
638 struct intel_ring_buffer *ring)
641 seqno = ring->next_seqno;
643 /* reserve 0 for non-seqno */
644 if (++ring->next_seqno == 0)
645 ring->next_seqno = 1;
649 struct intel_ring_buffer render_ring = {
650 .name = "render ring",
657 .ring_flag = I915_EXEC_RENDER,
658 .size = 32 * PAGE_SIZE,
659 .alignment = PAGE_SIZE,
660 .virtual_start = NULL,
667 .user_irq_refcount = 0,
669 .waiting_gem_seqno = 0,
670 .setup_status_page = render_setup_status_page,
671 .init = init_render_ring,
672 .get_head = render_ring_get_head,
673 .get_tail = render_ring_get_tail,
674 .get_active_head = render_ring_get_active_head,
675 .advance_ring = render_ring_advance_ring,
676 .flush = render_ring_flush,
677 .add_request = render_ring_add_request,
678 .get_gem_seqno = render_ring_get_gem_seqno,
679 .user_irq_get = render_ring_get_user_irq,
680 .user_irq_put = render_ring_put_user_irq,
681 .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
682 .status_page = {NULL, 0, NULL},