1 /* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
4 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
31 #include "drm_crtc_helper.h"
32 #include "intel_drv.h"
36 #define I915_DRV "i915_drv"
38 /* Really want an OS-independent resettable timer. Would like to have
39 * this loop run for (eg) 3 sec, but have the timer reset every time
40 * the head pointer changes, so that EBUSY only happens if the ring
41 * actually stalls for (eg) 3 seconds.
43 int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
45 drm_i915_private_t *dev_priv = dev->dev_private;
46 drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
47 u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
48 u32 last_acthd = I915_READ(acthd_reg);
50 u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
53 for (i = 0; i < 100000; i++) {
54 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
55 acthd = I915_READ(acthd_reg);
56 ring->space = ring->head - (ring->tail + 8);
58 ring->space += ring->Size;
62 if (dev->primary->master) {
63 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
64 if (master_priv->sarea_priv)
65 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
69 if (ring->head != last_head)
71 if (acthd != last_acthd)
74 last_head = ring->head;
76 msleep_interruptible(10);
83 /* As a ringbuffer is only allowed to wrap between instructions, fill
84 * the tail with NOOPs.
86 int i915_wrap_ring(struct drm_device *dev)
88 drm_i915_private_t *dev_priv = dev->dev_private;
89 volatile unsigned int *virt;
92 rem = dev_priv->ring.Size - dev_priv->ring.tail;
93 if (dev_priv->ring.space < rem) {
94 int ret = i915_wait_ring(dev, rem, __func__);
98 dev_priv->ring.space -= rem;
100 virt = (unsigned int *)
101 (dev_priv->ring.virtual_start + dev_priv->ring.tail);
106 dev_priv->ring.tail = 0;
112 * Sets up the hardware status page for devices that need a physical address
115 static int i915_init_phys_hws(struct drm_device *dev)
117 drm_i915_private_t *dev_priv = dev->dev_private;
118 /* Program Hardware Status Page */
119 dev_priv->status_page_dmah =
120 drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE, 0xffffffff);
122 if (!dev_priv->status_page_dmah) {
123 DRM_ERROR("Can not allocate hardware status page\n");
126 dev_priv->hw_status_page = dev_priv->status_page_dmah->vaddr;
127 dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr;
129 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
131 I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
132 DRM_DEBUG_DRIVER(I915_DRV, "Enabled hardware status page\n");
137 * Frees the hardware status page, whether it's a physical address or a virtual
138 * address set up by the X Server.
140 static void i915_free_hws(struct drm_device *dev)
142 drm_i915_private_t *dev_priv = dev->dev_private;
143 if (dev_priv->status_page_dmah) {
144 drm_pci_free(dev, dev_priv->status_page_dmah);
145 dev_priv->status_page_dmah = NULL;
148 if (dev_priv->status_gfx_addr) {
149 dev_priv->status_gfx_addr = 0;
150 drm_core_ioremapfree(&dev_priv->hws_map, dev);
153 /* Need to rewrite hardware status page */
154 I915_WRITE(HWS_PGA, 0x1ffff000);
157 void i915_kernel_lost_context(struct drm_device * dev)
159 drm_i915_private_t *dev_priv = dev->dev_private;
160 struct drm_i915_master_private *master_priv;
161 drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
164 * We should never lose context on the ring with modesetting
165 * as we don't expose it to userspace
167 if (drm_core_check_feature(dev, DRIVER_MODESET))
170 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
171 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
172 ring->space = ring->head - (ring->tail + 8);
174 ring->space += ring->Size;
176 if (!dev->primary->master)
179 master_priv = dev->primary->master->driver_priv;
180 if (ring->head == ring->tail && master_priv->sarea_priv)
181 master_priv->sarea_priv->perf_boxes |= I915_BOX_RING_EMPTY;
184 static int i915_dma_cleanup(struct drm_device * dev)
186 drm_i915_private_t *dev_priv = dev->dev_private;
187 /* Make sure interrupts are disabled here because the uninstall ioctl
188 * may not have been called from userspace and after dev_private
189 * is freed, it's too late.
191 if (dev->irq_enabled)
192 drm_irq_uninstall(dev);
194 if (dev_priv->ring.virtual_start) {
195 drm_core_ioremapfree(&dev_priv->ring.map, dev);
196 dev_priv->ring.virtual_start = NULL;
197 dev_priv->ring.map.handle = NULL;
198 dev_priv->ring.map.size = 0;
201 /* Clear the HWS virtual address at teardown */
202 if (I915_NEED_GFX_HWS(dev))
208 static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
210 drm_i915_private_t *dev_priv = dev->dev_private;
211 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
213 master_priv->sarea = drm_getsarea(dev);
214 if (master_priv->sarea) {
215 master_priv->sarea_priv = (drm_i915_sarea_t *)
216 ((u8 *)master_priv->sarea->handle + init->sarea_priv_offset);
218 DRM_DEBUG_DRIVER(I915_DRV,
219 "sarea not found assuming DRI2 userspace\n");
222 if (init->ring_size != 0) {
223 if (dev_priv->ring.ring_obj != NULL) {
224 i915_dma_cleanup(dev);
225 DRM_ERROR("Client tried to initialize ringbuffer in "
230 dev_priv->ring.Size = init->ring_size;
232 dev_priv->ring.map.offset = init->ring_start;
233 dev_priv->ring.map.size = init->ring_size;
234 dev_priv->ring.map.type = 0;
235 dev_priv->ring.map.flags = 0;
236 dev_priv->ring.map.mtrr = 0;
238 drm_core_ioremap_wc(&dev_priv->ring.map, dev);
240 if (dev_priv->ring.map.handle == NULL) {
241 i915_dma_cleanup(dev);
242 DRM_ERROR("can not ioremap virtual address for"
248 dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
250 dev_priv->cpp = init->cpp;
251 dev_priv->back_offset = init->back_offset;
252 dev_priv->front_offset = init->front_offset;
253 dev_priv->current_page = 0;
254 if (master_priv->sarea_priv)
255 master_priv->sarea_priv->pf_current_page = 0;
257 /* Allow hardware batchbuffers unless told otherwise.
259 dev_priv->allow_batchbuffer = 1;
264 static int i915_dma_resume(struct drm_device * dev)
266 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
268 DRM_DEBUG_DRIVER(I915_DRV, "%s\n", __func__);
270 if (dev_priv->ring.map.handle == NULL) {
271 DRM_ERROR("can not ioremap virtual address for"
276 /* Program Hardware Status Page */
277 if (!dev_priv->hw_status_page) {
278 DRM_ERROR("Can not find hardware status page\n");
281 DRM_DEBUG_DRIVER(I915_DRV, "hw status page @ %p\n",
282 dev_priv->hw_status_page);
284 if (dev_priv->status_gfx_addr != 0)
285 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
287 I915_WRITE(HWS_PGA, dev_priv->dma_status_page);
288 DRM_DEBUG_DRIVER(I915_DRV, "Enabled hardware status page\n");
293 static int i915_dma_init(struct drm_device *dev, void *data,
294 struct drm_file *file_priv)
296 drm_i915_init_t *init = data;
299 switch (init->func) {
301 retcode = i915_initialize(dev, init);
303 case I915_CLEANUP_DMA:
304 retcode = i915_dma_cleanup(dev);
306 case I915_RESUME_DMA:
307 retcode = i915_dma_resume(dev);
317 /* Implement basically the same security restrictions as hardware does
318 * for MI_BATCH_NON_SECURE. These can be made stricter at any time.
320 * Most of the calculations below involve calculating the size of a
321 * particular instruction. It's important to get the size right as
322 * that tells us where the next instruction to check is. Any illegal
323 * instruction detected will be given a size of zero, which is a
324 * signal to abort the rest of the buffer.
326 static int do_validate_cmd(int cmd)
328 switch (((cmd >> 29) & 0x7)) {
330 switch ((cmd >> 23) & 0x3f) {
332 return 1; /* MI_NOOP */
334 return 1; /* MI_FLUSH */
336 return 0; /* disallow everything else */
340 return 0; /* reserved */
342 return (cmd & 0xff) + 2; /* 2d commands */
344 if (((cmd >> 24) & 0x1f) <= 0x18)
347 switch ((cmd >> 24) & 0x1f) {
351 switch ((cmd >> 16) & 0xff) {
353 return (cmd & 0x1f) + 2;
355 return (cmd & 0xf) + 2;
357 return (cmd & 0xffff) + 2;
361 return (cmd & 0xffff) + 1;
365 if ((cmd & (1 << 23)) == 0) /* inline vertices */
366 return (cmd & 0x1ffff) + 2;
367 else if (cmd & (1 << 17)) /* indirect random */
368 if ((cmd & 0xffff) == 0)
369 return 0; /* unknown length, too hard */
371 return (((cmd & 0xffff) + 1) / 2) + 1;
373 return 2; /* indirect sequential */
384 static int validate_cmd(int cmd)
386 int ret = do_validate_cmd(cmd);
388 /* printk("validate_cmd( %x ): %d\n", cmd, ret); */
393 static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
395 drm_i915_private_t *dev_priv = dev->dev_private;
399 if ((dwords+1) * sizeof(int) >= dev_priv->ring.Size - 8)
402 BEGIN_LP_RING((dwords+1)&~1);
404 for (i = 0; i < dwords;) {
409 if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords)
428 i915_emit_box(struct drm_device *dev,
429 struct drm_clip_rect *boxes,
430 int i, int DR1, int DR4)
432 drm_i915_private_t *dev_priv = dev->dev_private;
433 struct drm_clip_rect box = boxes[i];
436 if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) {
437 DRM_ERROR("Bad box %d,%d..%d,%d\n",
438 box.x1, box.y1, box.x2, box.y2);
444 OUT_RING(GFX_OP_DRAWRECT_INFO_I965);
445 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
446 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
451 OUT_RING(GFX_OP_DRAWRECT_INFO);
453 OUT_RING((box.x1 & 0xffff) | (box.y1 << 16));
454 OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16));
463 /* XXX: Emitting the counter should really be moved to part of the IRQ
464 * emit. For now, do it in both places:
467 static void i915_emit_breadcrumb(struct drm_device *dev)
469 drm_i915_private_t *dev_priv = dev->dev_private;
470 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
474 if (dev_priv->counter > 0x7FFFFFFFUL)
475 dev_priv->counter = 0;
476 if (master_priv->sarea_priv)
477 master_priv->sarea_priv->last_enqueue = dev_priv->counter;
480 OUT_RING(MI_STORE_DWORD_INDEX);
481 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
482 OUT_RING(dev_priv->counter);
487 static int i915_dispatch_cmdbuffer(struct drm_device * dev,
488 drm_i915_cmdbuffer_t *cmd,
489 struct drm_clip_rect *cliprects,
492 int nbox = cmd->num_cliprects;
493 int i = 0, count, ret;
496 DRM_ERROR("alignment");
500 i915_kernel_lost_context(dev);
502 count = nbox ? nbox : 1;
504 for (i = 0; i < count; i++) {
506 ret = i915_emit_box(dev, cliprects, i,
512 ret = i915_emit_cmds(dev, cmdbuf, cmd->sz / 4);
517 i915_emit_breadcrumb(dev);
521 static int i915_dispatch_batchbuffer(struct drm_device * dev,
522 drm_i915_batchbuffer_t * batch,
523 struct drm_clip_rect *cliprects)
525 drm_i915_private_t *dev_priv = dev->dev_private;
526 int nbox = batch->num_cliprects;
530 if ((batch->start | batch->used) & 0x7) {
531 DRM_ERROR("alignment");
535 i915_kernel_lost_context(dev);
537 count = nbox ? nbox : 1;
539 for (i = 0; i < count; i++) {
541 int ret = i915_emit_box(dev, cliprects, i,
542 batch->DR1, batch->DR4);
547 if (!IS_I830(dev) && !IS_845G(dev)) {
550 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965);
551 OUT_RING(batch->start);
553 OUT_RING(MI_BATCH_BUFFER_START | (2 << 6));
554 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
559 OUT_RING(MI_BATCH_BUFFER);
560 OUT_RING(batch->start | MI_BATCH_NON_SECURE);
561 OUT_RING(batch->start + batch->used - 4);
567 i915_emit_breadcrumb(dev);
572 static int i915_dispatch_flip(struct drm_device * dev)
574 drm_i915_private_t *dev_priv = dev->dev_private;
575 struct drm_i915_master_private *master_priv =
576 dev->primary->master->driver_priv;
579 if (!master_priv->sarea_priv)
582 DRM_DEBUG_DRIVER(I915_DRV, "%s: page=%d pfCurrentPage=%d\n",
584 dev_priv->current_page,
585 master_priv->sarea_priv->pf_current_page);
587 i915_kernel_lost_context(dev);
590 OUT_RING(MI_FLUSH | MI_READ_FLUSH);
595 OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP);
597 if (dev_priv->current_page == 0) {
598 OUT_RING(dev_priv->back_offset);
599 dev_priv->current_page = 1;
601 OUT_RING(dev_priv->front_offset);
602 dev_priv->current_page = 0;
608 OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP);
612 master_priv->sarea_priv->last_enqueue = dev_priv->counter++;
615 OUT_RING(MI_STORE_DWORD_INDEX);
616 OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
617 OUT_RING(dev_priv->counter);
621 master_priv->sarea_priv->pf_current_page = dev_priv->current_page;
625 static int i915_quiescent(struct drm_device * dev)
627 drm_i915_private_t *dev_priv = dev->dev_private;
629 i915_kernel_lost_context(dev);
630 return i915_wait_ring(dev, dev_priv->ring.Size - 8, __func__);
633 static int i915_flush_ioctl(struct drm_device *dev, void *data,
634 struct drm_file *file_priv)
638 RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
640 mutex_lock(&dev->struct_mutex);
641 ret = i915_quiescent(dev);
642 mutex_unlock(&dev->struct_mutex);
647 static int i915_batchbuffer(struct drm_device *dev, void *data,
648 struct drm_file *file_priv)
650 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
651 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
652 drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
653 master_priv->sarea_priv;
654 drm_i915_batchbuffer_t *batch = data;
656 struct drm_clip_rect *cliprects = NULL;
658 if (!dev_priv->allow_batchbuffer) {
659 DRM_ERROR("Batchbuffer ioctl disabled\n");
663 DRM_DEBUG_DRIVER(I915_DRV,
664 "i915 batchbuffer, start %x used %d cliprects %d\n",
665 batch->start, batch->used, batch->num_cliprects);
667 RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
669 if (batch->num_cliprects < 0)
672 if (batch->num_cliprects) {
673 cliprects = kcalloc(batch->num_cliprects,
674 sizeof(struct drm_clip_rect),
676 if (cliprects == NULL)
679 ret = copy_from_user(cliprects, batch->cliprects,
680 batch->num_cliprects *
681 sizeof(struct drm_clip_rect));
686 mutex_lock(&dev->struct_mutex);
687 ret = i915_dispatch_batchbuffer(dev, batch, cliprects);
688 mutex_unlock(&dev->struct_mutex);
691 sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
699 static int i915_cmdbuffer(struct drm_device *dev, void *data,
700 struct drm_file *file_priv)
702 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
703 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
704 drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
705 master_priv->sarea_priv;
706 drm_i915_cmdbuffer_t *cmdbuf = data;
707 struct drm_clip_rect *cliprects = NULL;
711 DRM_DEBUG_DRIVER(I915_DRV,
712 "i915 cmdbuffer, buf %p sz %d cliprects %d\n",
713 cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
715 RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
717 if (cmdbuf->num_cliprects < 0)
720 batch_data = kmalloc(cmdbuf->sz, GFP_KERNEL);
721 if (batch_data == NULL)
724 ret = copy_from_user(batch_data, cmdbuf->buf, cmdbuf->sz);
726 goto fail_batch_free;
728 if (cmdbuf->num_cliprects) {
729 cliprects = kcalloc(cmdbuf->num_cliprects,
730 sizeof(struct drm_clip_rect), GFP_KERNEL);
731 if (cliprects == NULL)
732 goto fail_batch_free;
734 ret = copy_from_user(cliprects, cmdbuf->cliprects,
735 cmdbuf->num_cliprects *
736 sizeof(struct drm_clip_rect));
741 mutex_lock(&dev->struct_mutex);
742 ret = i915_dispatch_cmdbuffer(dev, cmdbuf, cliprects, batch_data);
743 mutex_unlock(&dev->struct_mutex);
745 DRM_ERROR("i915_dispatch_cmdbuffer failed\n");
750 sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
760 static int i915_flip_bufs(struct drm_device *dev, void *data,
761 struct drm_file *file_priv)
765 DRM_DEBUG_DRIVER(I915_DRV, "%s\n", __func__);
767 RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
769 mutex_lock(&dev->struct_mutex);
770 ret = i915_dispatch_flip(dev);
771 mutex_unlock(&dev->struct_mutex);
776 static int i915_getparam(struct drm_device *dev, void *data,
777 struct drm_file *file_priv)
779 drm_i915_private_t *dev_priv = dev->dev_private;
780 drm_i915_getparam_t *param = data;
784 DRM_ERROR("called with no initialization\n");
788 switch (param->param) {
789 case I915_PARAM_IRQ_ACTIVE:
790 value = dev->pdev->irq ? 1 : 0;
792 case I915_PARAM_ALLOW_BATCHBUFFER:
793 value = dev_priv->allow_batchbuffer ? 1 : 0;
795 case I915_PARAM_LAST_DISPATCH:
796 value = READ_BREADCRUMB(dev_priv);
798 case I915_PARAM_CHIPSET_ID:
799 value = dev->pci_device;
801 case I915_PARAM_HAS_GEM:
802 value = dev_priv->has_gem;
804 case I915_PARAM_NUM_FENCES_AVAIL:
805 value = dev_priv->num_fence_regs - dev_priv->fence_reg_start;
808 DRM_DEBUG_DRIVER(I915_DRV, "Unknown parameter %d\n",
813 if (DRM_COPY_TO_USER(param->value, &value, sizeof(int))) {
814 DRM_ERROR("DRM_COPY_TO_USER failed\n");
821 static int i915_setparam(struct drm_device *dev, void *data,
822 struct drm_file *file_priv)
824 drm_i915_private_t *dev_priv = dev->dev_private;
825 drm_i915_setparam_t *param = data;
828 DRM_ERROR("called with no initialization\n");
832 switch (param->param) {
833 case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
835 case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
836 dev_priv->tex_lru_log_granularity = param->value;
838 case I915_SETPARAM_ALLOW_BATCHBUFFER:
839 dev_priv->allow_batchbuffer = param->value;
841 case I915_SETPARAM_NUM_USED_FENCES:
842 if (param->value > dev_priv->num_fence_regs ||
845 /* Userspace can use first N regs */
846 dev_priv->fence_reg_start = param->value;
849 DRM_DEBUG_DRIVER(I915_DRV, "unknown parameter %d\n",
857 static int i915_set_status_page(struct drm_device *dev, void *data,
858 struct drm_file *file_priv)
860 drm_i915_private_t *dev_priv = dev->dev_private;
861 drm_i915_hws_addr_t *hws = data;
863 if (!I915_NEED_GFX_HWS(dev))
867 DRM_ERROR("called with no initialization\n");
871 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
872 WARN(1, "tried to set status page when mode setting active\n");
876 DRM_DEBUG("set status page addr 0x%08x\n", (u32)hws->addr);
878 dev_priv->status_gfx_addr = hws->addr & (0x1ffff<<12);
880 dev_priv->hws_map.offset = dev->agp->base + hws->addr;
881 dev_priv->hws_map.size = 4*1024;
882 dev_priv->hws_map.type = 0;
883 dev_priv->hws_map.flags = 0;
884 dev_priv->hws_map.mtrr = 0;
886 drm_core_ioremap_wc(&dev_priv->hws_map, dev);
887 if (dev_priv->hws_map.handle == NULL) {
888 i915_dma_cleanup(dev);
889 dev_priv->status_gfx_addr = 0;
890 DRM_ERROR("can not ioremap virtual address for"
891 " G33 hw status page\n");
894 dev_priv->hw_status_page = dev_priv->hws_map.handle;
896 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
897 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
898 DRM_DEBUG_DRIVER(I915_DRV, "load hws HWS_PGA with gfx mem 0x%x\n",
899 dev_priv->status_gfx_addr);
900 DRM_DEBUG_DRIVER(I915_DRV, "load hws at %p\n",
901 dev_priv->hw_status_page);
906 * i915_probe_agp - get AGP bootup configuration
908 * @aperture_size: returns AGP aperture configured size
909 * @preallocated_size: returns size of BIOS preallocated AGP space
911 * Since Intel integrated graphics are UMA, the BIOS has to set aside
912 * some RAM for the framebuffer at early boot. This code figures out
913 * how much was set aside so we can use it for our own purposes.
915 static int i915_probe_agp(struct drm_device *dev, uint32_t *aperture_size,
916 uint32_t *preallocated_size)
918 struct pci_dev *bridge_dev;
920 unsigned long overhead;
921 unsigned long stolen;
923 bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0,0));
925 DRM_ERROR("bridge device not found\n");
929 /* Get the fb aperture size and "stolen" memory amount. */
930 pci_read_config_word(bridge_dev, INTEL_GMCH_CTRL, &tmp);
931 pci_dev_put(bridge_dev);
933 *aperture_size = 1024 * 1024;
934 *preallocated_size = 1024 * 1024;
936 switch (dev->pdev->device) {
937 case PCI_DEVICE_ID_INTEL_82830_CGC:
938 case PCI_DEVICE_ID_INTEL_82845G_IG:
939 case PCI_DEVICE_ID_INTEL_82855GM_IG:
940 case PCI_DEVICE_ID_INTEL_82865_IG:
941 if ((tmp & INTEL_GMCH_MEM_MASK) == INTEL_GMCH_MEM_64M)
942 *aperture_size *= 64;
944 *aperture_size *= 128;
947 /* 9xx supports large sizes, just look at the length */
948 *aperture_size = pci_resource_len(dev->pdev, 2);
953 * Some of the preallocated space is taken by the GTT
954 * and popup. GTT is 1K per MB of aperture size, and popup is 4K.
956 if (IS_G4X(dev) || IS_IGD(dev) || IS_IGDNG(dev))
959 overhead = (*aperture_size / 1024) + 4096;
961 switch (tmp & INTEL_GMCH_GMS_MASK) {
962 case INTEL_855_GMCH_GMS_DISABLED:
963 DRM_ERROR("video memory is disabled\n");
965 case INTEL_855_GMCH_GMS_STOLEN_1M:
966 stolen = 1 * 1024 * 1024;
968 case INTEL_855_GMCH_GMS_STOLEN_4M:
969 stolen = 4 * 1024 * 1024;
971 case INTEL_855_GMCH_GMS_STOLEN_8M:
972 stolen = 8 * 1024 * 1024;
974 case INTEL_855_GMCH_GMS_STOLEN_16M:
975 stolen = 16 * 1024 * 1024;
977 case INTEL_855_GMCH_GMS_STOLEN_32M:
978 stolen = 32 * 1024 * 1024;
980 case INTEL_915G_GMCH_GMS_STOLEN_48M:
981 stolen = 48 * 1024 * 1024;
983 case INTEL_915G_GMCH_GMS_STOLEN_64M:
984 stolen = 64 * 1024 * 1024;
986 case INTEL_GMCH_GMS_STOLEN_128M:
987 stolen = 128 * 1024 * 1024;
989 case INTEL_GMCH_GMS_STOLEN_256M:
990 stolen = 256 * 1024 * 1024;
992 case INTEL_GMCH_GMS_STOLEN_96M:
993 stolen = 96 * 1024 * 1024;
995 case INTEL_GMCH_GMS_STOLEN_160M:
996 stolen = 160 * 1024 * 1024;
998 case INTEL_GMCH_GMS_STOLEN_224M:
999 stolen = 224 * 1024 * 1024;
1001 case INTEL_GMCH_GMS_STOLEN_352M:
1002 stolen = 352 * 1024 * 1024;
1005 DRM_ERROR("unexpected GMCH_GMS value: 0x%02x\n",
1006 tmp & INTEL_GMCH_GMS_MASK);
1009 *preallocated_size = stolen - overhead;
1014 static int i915_load_modeset_init(struct drm_device *dev,
1015 unsigned long prealloc_size,
1016 unsigned long agp_size)
1018 struct drm_i915_private *dev_priv = dev->dev_private;
1019 int fb_bar = IS_I9XX(dev) ? 2 : 0;
1022 dev->mode_config.fb_base = drm_get_resource_start(dev, fb_bar) &
1025 if (IS_MOBILE(dev) || IS_I9XX(dev))
1026 dev_priv->cursor_needs_physical = true;
1028 dev_priv->cursor_needs_physical = false;
1030 if (IS_I965G(dev) || IS_G33(dev))
1031 dev_priv->cursor_needs_physical = false;
1033 /* Basic memrange allocator for stolen space (aka vram) */
1034 drm_mm_init(&dev_priv->vram, 0, prealloc_size);
1036 /* Let GEM Manage from end of prealloc space to end of aperture.
1038 * However, leave one page at the end still bound to the scratch page.
1039 * There are a number of places where the hardware apparently
1040 * prefetches past the end of the object, and we've seen multiple
1041 * hangs with the GPU head pointer stuck in a batchbuffer bound
1042 * at the last page of the aperture. One page should be enough to
1043 * keep any prefetching inside of the aperture.
1045 i915_gem_do_init(dev, prealloc_size, agp_size - 4096);
1047 ret = i915_gem_init_ringbuffer(dev);
1051 /* Allow hardware batchbuffers unless told otherwise.
1053 dev_priv->allow_batchbuffer = 1;
1055 ret = intel_init_bios(dev);
1057 DRM_INFO("failed to find VBIOS tables\n");
1059 ret = drm_irq_install(dev);
1061 goto destroy_ringbuffer;
1063 /* Always safe in the mode setting case. */
1064 /* FIXME: do pre/post-mode set stuff in core KMS code */
1065 dev->vblank_disable_allowed = 1;
1068 * Initialize the hardware status page IRQ location.
1071 I915_WRITE(INSTPM, (1 << 5) | (1 << 21));
1073 intel_modeset_init(dev);
1075 drm_helper_initial_config(dev);
1080 i915_gem_cleanup_ringbuffer(dev);
1085 int i915_master_create(struct drm_device *dev, struct drm_master *master)
1087 struct drm_i915_master_private *master_priv;
1089 master_priv = kzalloc(sizeof(*master_priv), GFP_KERNEL);
1093 master->driver_priv = master_priv;
1097 void i915_master_destroy(struct drm_device *dev, struct drm_master *master)
1099 struct drm_i915_master_private *master_priv = master->driver_priv;
1106 master->driver_priv = NULL;
1109 static void i915_get_mem_freq(struct drm_device *dev)
1111 drm_i915_private_t *dev_priv = dev->dev_private;
1117 tmp = I915_READ(CLKCFG);
1119 switch (tmp & CLKCFG_FSB_MASK) {
1120 case CLKCFG_FSB_533:
1121 dev_priv->fsb_freq = 533; /* 133*4 */
1123 case CLKCFG_FSB_800:
1124 dev_priv->fsb_freq = 800; /* 200*4 */
1126 case CLKCFG_FSB_667:
1127 dev_priv->fsb_freq = 667; /* 167*4 */
1129 case CLKCFG_FSB_400:
1130 dev_priv->fsb_freq = 400; /* 100*4 */
1134 switch (tmp & CLKCFG_MEM_MASK) {
1135 case CLKCFG_MEM_533:
1136 dev_priv->mem_freq = 533;
1138 case CLKCFG_MEM_667:
1139 dev_priv->mem_freq = 667;
1141 case CLKCFG_MEM_800:
1142 dev_priv->mem_freq = 800;
1148 * i915_driver_load - setup chip and create an initial config
1150 * @flags: startup flags
1152 * The driver load routine has to do several things:
1153 * - drive output discovery via intel_modeset_init()
1154 * - initialize the memory manager
1155 * - allocate initial config memory
1156 * - setup the DRM framebuffer with the allocated memory
1158 int i915_driver_load(struct drm_device *dev, unsigned long flags)
1160 struct drm_i915_private *dev_priv = dev->dev_private;
1161 resource_size_t base, size;
1162 int ret = 0, mmio_bar = IS_I9XX(dev) ? 0 : 1;
1163 uint32_t agp_size, prealloc_size;
1165 /* i915 has 4 more counters */
1167 dev->types[6] = _DRM_STAT_IRQ;
1168 dev->types[7] = _DRM_STAT_PRIMARY;
1169 dev->types[8] = _DRM_STAT_SECONDARY;
1170 dev->types[9] = _DRM_STAT_DMA;
1172 dev_priv = kzalloc(sizeof(drm_i915_private_t), GFP_KERNEL);
1173 if (dev_priv == NULL)
1176 dev->dev_private = (void *)dev_priv;
1177 dev_priv->dev = dev;
1179 /* Add register map (needed for suspend/resume) */
1180 base = drm_get_resource_start(dev, mmio_bar);
1181 size = drm_get_resource_len(dev, mmio_bar);
1183 dev_priv->regs = ioremap(base, size);
1184 if (!dev_priv->regs) {
1185 DRM_ERROR("failed to map registers\n");
1190 dev_priv->mm.gtt_mapping =
1191 io_mapping_create_wc(dev->agp->base,
1192 dev->agp->agp_info.aper_size * 1024*1024);
1193 if (dev_priv->mm.gtt_mapping == NULL) {
1198 /* Set up a WC MTRR for non-PAT systems. This is more common than
1199 * one would think, because the kernel disables PAT on first
1200 * generation Core chips because WC PAT gets overridden by a UC
1201 * MTRR if present. Even if a UC MTRR isn't present.
1203 dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base,
1204 dev->agp->agp_info.aper_size *
1206 MTRR_TYPE_WRCOMB, 1);
1207 if (dev_priv->mm.gtt_mtrr < 0) {
1208 DRM_INFO("MTRR allocation failed. Graphics "
1209 "performance may suffer.\n");
1212 ret = i915_probe_agp(dev, &agp_size, &prealloc_size);
1216 dev_priv->wq = create_workqueue("i915");
1217 if (dev_priv->wq == NULL) {
1218 DRM_ERROR("Failed to create our workqueue.\n");
1223 /* enable GEM by default */
1224 dev_priv->has_gem = 1;
1226 if (prealloc_size > agp_size * 3 / 4) {
1227 DRM_ERROR("Detected broken video BIOS with %d/%dkB of video "
1229 prealloc_size / 1024, agp_size / 1024);
1230 DRM_ERROR("Disabling GEM. (try reducing stolen memory or "
1231 "updating the BIOS to fix).\n");
1232 dev_priv->has_gem = 0;
1235 dev->driver->get_vblank_counter = i915_get_vblank_counter;
1236 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
1237 if (IS_G4X(dev) || IS_IGDNG(dev)) {
1238 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
1239 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
1245 if (!I915_NEED_GFX_HWS(dev)) {
1246 ret = i915_init_phys_hws(dev);
1248 goto out_workqueue_free;
1251 i915_get_mem_freq(dev);
1253 /* On the 945G/GM, the chipset reports the MSI capability on the
1254 * integrated graphics even though the support isn't actually there
1255 * according to the published specs. It doesn't appear to function
1256 * correctly in testing on 945G.
1257 * This may be a side effect of MSI having been made available for PEG
1258 * and the registers being closely associated.
1260 * According to chipset errata, on the 965GM, MSI interrupts may
1261 * be lost or delayed, but we use them anyways to avoid
1262 * stuck interrupts on some machines.
1264 if (!IS_I945G(dev) && !IS_I945GM(dev))
1265 pci_enable_msi(dev->pdev);
1267 spin_lock_init(&dev_priv->user_irq_lock);
1268 spin_lock_init(&dev_priv->error_lock);
1269 dev_priv->user_irq_refcount = 0;
1271 ret = drm_vblank_init(dev, I915_NUM_PIPE);
1274 (void) i915_driver_unload(dev);
1278 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1279 ret = i915_load_modeset_init(dev, prealloc_size, agp_size);
1281 DRM_ERROR("failed to init modeset\n");
1282 goto out_workqueue_free;
1286 /* Must be done after probing outputs */
1287 /* FIXME: verify on IGDNG */
1289 intel_opregion_init(dev, 0);
1294 destroy_workqueue(dev_priv->wq);
1296 io_mapping_free(dev_priv->mm.gtt_mapping);
1298 iounmap(dev_priv->regs);
1304 int i915_driver_unload(struct drm_device *dev)
1306 struct drm_i915_private *dev_priv = dev->dev_private;
1308 destroy_workqueue(dev_priv->wq);
1310 io_mapping_free(dev_priv->mm.gtt_mapping);
1311 if (dev_priv->mm.gtt_mtrr >= 0) {
1312 mtrr_del(dev_priv->mm.gtt_mtrr, dev->agp->base,
1313 dev->agp->agp_info.aper_size * 1024 * 1024);
1314 dev_priv->mm.gtt_mtrr = -1;
1317 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1318 drm_irq_uninstall(dev);
1321 if (dev->pdev->msi_enabled)
1322 pci_disable_msi(dev->pdev);
1324 if (dev_priv->regs != NULL)
1325 iounmap(dev_priv->regs);
1328 intel_opregion_free(dev, 0);
1330 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1331 intel_modeset_cleanup(dev);
1333 i915_gem_free_all_phys_object(dev);
1335 mutex_lock(&dev->struct_mutex);
1336 i915_gem_cleanup_ringbuffer(dev);
1337 mutex_unlock(&dev->struct_mutex);
1338 drm_mm_takedown(&dev_priv->vram);
1339 i915_gem_lastclose(dev);
1342 kfree(dev->dev_private);
1347 int i915_driver_open(struct drm_device *dev, struct drm_file *file_priv)
1349 struct drm_i915_file_private *i915_file_priv;
1351 DRM_DEBUG_DRIVER(I915_DRV, "\n");
1352 i915_file_priv = (struct drm_i915_file_private *)
1353 kmalloc(sizeof(*i915_file_priv), GFP_KERNEL);
1355 if (!i915_file_priv)
1358 file_priv->driver_priv = i915_file_priv;
1360 INIT_LIST_HEAD(&i915_file_priv->mm.request_list);
1366 * i915_driver_lastclose - clean up after all DRM clients have exited
1369 * Take care of cleaning up after all DRM clients have exited. In the
1370 * mode setting case, we want to restore the kernel's initial mode (just
1371 * in case the last client left us in a bad state).
1373 * Additionally, in the non-mode setting case, we'll tear down the AGP
1374 * and DMA structures, since the kernel won't be using them, and clea
1377 void i915_driver_lastclose(struct drm_device * dev)
1379 drm_i915_private_t *dev_priv = dev->dev_private;
1381 if (!dev_priv || drm_core_check_feature(dev, DRIVER_MODESET)) {
1386 i915_gem_lastclose(dev);
1388 if (dev_priv->agp_heap)
1389 i915_mem_takedown(&(dev_priv->agp_heap));
1391 i915_dma_cleanup(dev);
1394 void i915_driver_preclose(struct drm_device * dev, struct drm_file *file_priv)
1396 drm_i915_private_t *dev_priv = dev->dev_private;
1397 i915_gem_release(dev, file_priv);
1398 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1399 i915_mem_release(dev, file_priv, dev_priv->agp_heap);
1402 void i915_driver_postclose(struct drm_device *dev, struct drm_file *file_priv)
1404 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1406 kfree(i915_file_priv);
1409 struct drm_ioctl_desc i915_ioctls[] = {
1410 DRM_IOCTL_DEF(DRM_I915_INIT, i915_dma_init, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1411 DRM_IOCTL_DEF(DRM_I915_FLUSH, i915_flush_ioctl, DRM_AUTH),
1412 DRM_IOCTL_DEF(DRM_I915_FLIP, i915_flip_bufs, DRM_AUTH),
1413 DRM_IOCTL_DEF(DRM_I915_BATCHBUFFER, i915_batchbuffer, DRM_AUTH),
1414 DRM_IOCTL_DEF(DRM_I915_IRQ_EMIT, i915_irq_emit, DRM_AUTH),
1415 DRM_IOCTL_DEF(DRM_I915_IRQ_WAIT, i915_irq_wait, DRM_AUTH),
1416 DRM_IOCTL_DEF(DRM_I915_GETPARAM, i915_getparam, DRM_AUTH),
1417 DRM_IOCTL_DEF(DRM_I915_SETPARAM, i915_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1418 DRM_IOCTL_DEF(DRM_I915_ALLOC, i915_mem_alloc, DRM_AUTH),
1419 DRM_IOCTL_DEF(DRM_I915_FREE, i915_mem_free, DRM_AUTH),
1420 DRM_IOCTL_DEF(DRM_I915_INIT_HEAP, i915_mem_init_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1421 DRM_IOCTL_DEF(DRM_I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH),
1422 DRM_IOCTL_DEF(DRM_I915_DESTROY_HEAP, i915_mem_destroy_heap, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
1423 DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE, i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
1424 DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE, i915_vblank_pipe_get, DRM_AUTH ),
1425 DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
1426 DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1427 DRM_IOCTL_DEF(DRM_I915_GEM_INIT, i915_gem_init_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1428 DRM_IOCTL_DEF(DRM_I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
1429 DRM_IOCTL_DEF(DRM_I915_GEM_PIN, i915_gem_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
1430 DRM_IOCTL_DEF(DRM_I915_GEM_UNPIN, i915_gem_unpin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
1431 DRM_IOCTL_DEF(DRM_I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH),
1432 DRM_IOCTL_DEF(DRM_I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH),
1433 DRM_IOCTL_DEF(DRM_I915_GEM_ENTERVT, i915_gem_entervt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1434 DRM_IOCTL_DEF(DRM_I915_GEM_LEAVEVT, i915_gem_leavevt_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1435 DRM_IOCTL_DEF(DRM_I915_GEM_CREATE, i915_gem_create_ioctl, 0),
1436 DRM_IOCTL_DEF(DRM_I915_GEM_PREAD, i915_gem_pread_ioctl, 0),
1437 DRM_IOCTL_DEF(DRM_I915_GEM_PWRITE, i915_gem_pwrite_ioctl, 0),
1438 DRM_IOCTL_DEF(DRM_I915_GEM_MMAP, i915_gem_mmap_ioctl, 0),
1439 DRM_IOCTL_DEF(DRM_I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, 0),
1440 DRM_IOCTL_DEF(DRM_I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, 0),
1441 DRM_IOCTL_DEF(DRM_I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, 0),
1442 DRM_IOCTL_DEF(DRM_I915_GEM_SET_TILING, i915_gem_set_tiling, 0),
1443 DRM_IOCTL_DEF(DRM_I915_GEM_GET_TILING, i915_gem_get_tiling, 0),
1444 DRM_IOCTL_DEF(DRM_I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, 0),
1445 DRM_IOCTL_DEF(DRM_I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
1448 int i915_max_ioctl = DRM_ARRAY_SIZE(i915_ioctls);
1451 * Determine if the device really is AGP or not.
1453 * All Intel graphics chipsets are treated as AGP, even if they are really
1456 * \param dev The device to be tested.
1459 * A value of 1 is always retured to indictate every i9x5 is AGP.
1461 int i915_driver_device_is_agp(struct drm_device * dev)