drivers/gpu/drm/i915/intel_ringbuffer.c

   1 /*
   2  * Copyright © 2008-2010 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  *
  23  * Authors:
  24  *    Eric Anholt <eric@anholt.net>
  25  *    Zou Nan hai <nanhai.zou@intel.com>
  26  *    Xiang Hai hao<haihao.xiang@intel.com>
  27  *
  28  */
  29
  30 #include "drmP.h"
  31 #include "drm.h"
  32 #include "i915_drv.h"
  33 #include "i915_drm.h"
  34 #include "i915_trace.h"
  35
  36 static void
  37 render_ring_flush(struct drm_device *dev,
  38                 struct intel_ring_buffer *ring,
  39                 u32     invalidate_domains,
  40                 u32     flush_domains)
  41 {
  42 #if WATCH_EXEC
  43         DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
  44                   invalidate_domains, flush_domains);
  45 #endif
  46         u32 cmd;
  47         trace_i915_gem_request_flush(dev, ring->next_seqno,
  48                                      invalidate_domains, flush_domains);
  49
  50         if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
  51                 /*
  52                  * read/write caches:
  53                  *
  54                  * I915_GEM_DOMAIN_RENDER is always invalidated, but is
  55                  * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
  56                  * also flushed at 2d versus 3d pipeline switches.
  57                  *
  58                  * read-only caches:
  59                  *
  60                  * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
  61                  * MI_READ_FLUSH is set, and is always flushed on 965.
  62                  *
  63                  * I915_GEM_DOMAIN_COMMAND may not exist?
  64                  *
  65                  * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
  66                  * invalidated when MI_EXE_FLUSH is set.
  67                  *
  68                  * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
  69                  * invalidated with every MI_FLUSH.
  70                  *
  71                  * TLBs:
  72                  *
  73                  * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
  74                  * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
  75                  * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
  76                  * are flushed at any MI_FLUSH.
  77                  */
  78
  79                 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
  80                 if ((invalidate_domains|flush_domains) &
  81                     I915_GEM_DOMAIN_RENDER)
  82                         cmd &= ~MI_NO_WRITE_FLUSH;
  83                 if (!IS_I965G(dev)) {
  84                         /*
  85                          * On the 965, the sampler cache always gets flushed
  86                          * and this bit is reserved.
  87                          */
  88                         if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
  89                                 cmd |= MI_READ_FLUSH;
  90                 }
  91                 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
  92                         cmd |= MI_EXE_FLUSH;
  93
  94 #if WATCH_EXEC
  95                 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
  96 #endif
  97                 intel_ring_begin(dev, ring, 8);
  98                 intel_ring_emit(dev, ring, cmd);
  99                 intel_ring_emit(dev, ring, MI_NOOP);
 100                 intel_ring_advance(dev, ring);
 101         }
 102 }
 103
 104 static unsigned int render_ring_get_head(struct drm_device *dev,
 105                 struct intel_ring_buffer *ring)
 106 {
 107         drm_i915_private_t *dev_priv = dev->dev_private;
 108         return I915_READ(PRB0_HEAD) & HEAD_ADDR;
 109 }
 110
 111 static unsigned int render_ring_get_tail(struct drm_device *dev,
 112                 struct intel_ring_buffer *ring)
 113 {
 114         drm_i915_private_t *dev_priv = dev->dev_private;
 115         return I915_READ(PRB0_TAIL) & TAIL_ADDR;
 116 }
 117
 118 static unsigned int render_ring_get_active_head(struct drm_device *dev,
 119                 struct intel_ring_buffer *ring)
 120 {
 121         drm_i915_private_t *dev_priv = dev->dev_private;
 122         u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
 123
 124         return I915_READ(acthd_reg);
 125 }
 126
 127 static void render_ring_advance_ring(struct drm_device *dev,
 128                 struct intel_ring_buffer *ring)
 129 {
 130         drm_i915_private_t *dev_priv = dev->dev_private;
 131         I915_WRITE(PRB0_TAIL, ring->tail);
 132 }
 133
 134 static int init_ring_common(struct drm_device *dev,
 135                 struct intel_ring_buffer *ring)
 136 {
 137         u32 head;
 138         drm_i915_private_t *dev_priv = dev->dev_private;
 139         struct drm_i915_gem_object *obj_priv;
 140         obj_priv = to_intel_bo(ring->gem_object);
 141
 142         /* Stop the ring if it's running. */
 143         I915_WRITE(ring->regs.ctl, 0);
 144         I915_WRITE(ring->regs.head, 0);
 145         I915_WRITE(ring->regs.tail, 0);
 146
 147         /* Initialize the ring. */
 148         I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
 149         head = ring->get_head(dev, ring);
 150
 151         /* G45 ring initialization fails to reset head to zero */
 152         if (head != 0) {
 153                 DRM_ERROR("%s head not reset to zero "
 154                                 "ctl %08x head %08x tail %08x start %08x\n",
 155                                 ring->name,
 156                                 I915_READ(ring->regs.ctl),
 157                                 I915_READ(ring->regs.head),
 158                                 I915_READ(ring->regs.tail),
 159                                 I915_READ(ring->regs.start));
 160
 161                 I915_WRITE(ring->regs.head, 0);
 162
 163                 DRM_ERROR("%s head forced to zero "
 164                                 "ctl %08x head %08x tail %08x start %08x\n",
 165                                 ring->name,
 166                                 I915_READ(ring->regs.ctl),
 167                                 I915_READ(ring->regs.head),
 168                                 I915_READ(ring->regs.tail),
 169                                 I915_READ(ring->regs.start));
 170         }
 171
 172         I915_WRITE(ring->regs.ctl,
 173                         ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
 174                         | RING_NO_REPORT | RING_VALID);
 175
 176         head = I915_READ(ring->regs.head) & HEAD_ADDR;
 177         /* If the head is still not zero, the ring is dead */
 178         if (head != 0) {
 179                 DRM_ERROR("%s initialization failed "
 180                                 "ctl %08x head %08x tail %08x start %08x\n",
 181                                 ring->name,
 182                                 I915_READ(ring->regs.ctl),
 183                                 I915_READ(ring->regs.head),
 184                                 I915_READ(ring->regs.tail),
 185                                 I915_READ(ring->regs.start));
 186                 return -EIO;
 187         }
 188
 189         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 190                 i915_kernel_lost_context(dev);
 191         else {
 192                 ring->head = ring->get_head(dev, ring);
 193                 ring->tail = ring->get_tail(dev, ring);
 194                 ring->space = ring->head - (ring->tail + 8);
 195                 if (ring->space < 0)
 196                         ring->space += ring->size;
 197         }
 198         return 0;
 199 }
 200
 201 static int init_render_ring(struct drm_device *dev,
 202                 struct intel_ring_buffer *ring)
 203 {
 204         drm_i915_private_t *dev_priv = dev->dev_private;
 205         int ret = init_ring_common(dev, ring);
 206         if (IS_I9XX(dev) && !IS_GEN3(dev)) {
 207                 I915_WRITE(MI_MODE,
 208                                 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
 209         }
 210         return ret;
 211 }
 212
 213 #define PIPE_CONTROL_FLUSH(addr)                                        \
 214 do {                                                                    \
 215         OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |          \
 216                  PIPE_CONTROL_DEPTH_STALL);                             \
 217         OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT);                       \
 218         OUT_RING(0);                                                    \
 219         OUT_RING(0);                                                    \
 220 } while (0)
 221
 222 /**
 223  * Creates a new sequence number, emitting a write of it to the status page
 224  * plus an interrupt, which will trigger i915_user_interrupt_handler.
 225  *
 226  * Must be called with struct_lock held.
 227  *
 228  * Returned sequence numbers are nonzero on success.
 229  */
 230 static u32
 231 render_ring_add_request(struct drm_device *dev,
 232                 struct intel_ring_buffer *ring,
 233                 struct drm_file *file_priv,
 234                 u32 flush_domains)
 235 {
 236         u32 seqno;
 237         drm_i915_private_t *dev_priv = dev->dev_private;
 238         seqno = intel_ring_get_seqno(dev, ring);
 239         if (HAS_PIPE_CONTROL(dev)) {
 240                 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
 241
 242                 /*
 243                  * Workaround qword write incoherence by flushing the
 244                  * PIPE_NOTIFY buffers out to memory before requesting
 245                  * an interrupt.
 246                  */
 247                 BEGIN_LP_RING(32);
 248                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 249                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
 250                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 251                 OUT_RING(seqno);
 252                 OUT_RING(0);
 253                 PIPE_CONTROL_FLUSH(scratch_addr);
 254                 scratch_addr += 128; /* write to separate cachelines */
 255                 PIPE_CONTROL_FLUSH(scratch_addr);
 256                 scratch_addr += 128;
 257                 PIPE_CONTROL_FLUSH(scratch_addr);
 258                 scratch_addr += 128;
 259                 PIPE_CONTROL_FLUSH(scratch_addr);
 260                 scratch_addr += 128;
 261                 PIPE_CONTROL_FLUSH(scratch_addr);
 262                 scratch_addr += 128;
 263                 PIPE_CONTROL_FLUSH(scratch_addr);
 264                 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
 265                          PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
 266                          PIPE_CONTROL_NOTIFY);
 267                 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
 268                 OUT_RING(seqno);
 269                 OUT_RING(0);
 270                 ADVANCE_LP_RING();
 271         } else {
 272                 BEGIN_LP_RING(4);
 273                 OUT_RING(MI_STORE_DWORD_INDEX);
 274                 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
 275                 OUT_RING(seqno);
 276
 277                 OUT_RING(MI_USER_INTERRUPT);
 278                 ADVANCE_LP_RING();
 279         }
 280         return seqno;
 281 }
 282
 283 static u32
 284 render_ring_get_gem_seqno(struct drm_device *dev,
 285                 struct intel_ring_buffer *ring)
 286 {
 287         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 288         if (HAS_PIPE_CONTROL(dev))
 289                 return ((volatile u32 *)(dev_priv->seqno_page))[0];
 290         else
 291                 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 292 }
 293
 294 static void
 295 render_ring_get_user_irq(struct drm_device *dev,
 296                 struct intel_ring_buffer *ring)
 297 {
 298         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 299         unsigned long irqflags;
 300
 301         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 302         if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
 303                 if (HAS_PCH_SPLIT(dev))
 304                         ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 305                 else
 306                         i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
 307         }
 308         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 309 }
 310
 311 static void
 312 render_ring_put_user_irq(struct drm_device *dev,
 313                 struct intel_ring_buffer *ring)
 314 {
 315         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 316         unsigned long irqflags;
 317
 318         spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
 319         BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
 320         if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
 321                 if (HAS_PCH_SPLIT(dev))
 322                         ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
 323                 else
 324                         i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
 325         }
 326         spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
 327 }
 328
 329 static void render_setup_status_page(struct drm_device *dev,
 330         struct  intel_ring_buffer *ring)
 331 {
 332         drm_i915_private_t *dev_priv = dev->dev_private;
 333         if (IS_GEN6(dev)) {
 334                 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
 335                 I915_READ(HWS_PGA_GEN6); /* posting read */
 336         } else {
 337                 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
 338                 I915_READ(HWS_PGA); /* posting read */
 339         }
 340
 341 }
 342
 343 static int
 344 render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
 345                 struct intel_ring_buffer *ring,
 346                 struct drm_i915_gem_execbuffer2 *exec,
 347                 struct drm_clip_rect *cliprects,
 348                 uint64_t exec_offset)
 349 {
 350         drm_i915_private_t *dev_priv = dev->dev_private;
 351         int nbox = exec->num_cliprects;
 352         int i = 0, count;
 353         uint32_t exec_start, exec_len;
 354         exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
 355         exec_len = (uint32_t) exec->batch_len;
 356
 357         trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
 358
 359         count = nbox ? nbox : 1;
 360
 361         for (i = 0; i < count; i++) {
 362                 if (i < nbox) {
 363                         int ret = i915_emit_box(dev, cliprects, i,
 364                                                 exec->DR1, exec->DR4);
 365                         if (ret)
 366                                 return ret;
 367                 }
 368
 369                 if (IS_I830(dev) || IS_845G(dev)) {
 370                         intel_ring_begin(dev, ring, 4);
 371                         intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
 372                         intel_ring_emit(dev, ring,
 373                                         exec_start | MI_BATCH_NON_SECURE);
 374                         intel_ring_emit(dev, ring, exec_start + exec_len - 4);
 375                         intel_ring_emit(dev, ring, 0);
 376                 } else {
 377                         intel_ring_begin(dev, ring, 4);
 378                         if (IS_I965G(dev)) {
 379                                 intel_ring_emit(dev, ring,
 380                                                 MI_BATCH_BUFFER_START | (2 << 6)
 381                                                 | MI_BATCH_NON_SECURE_I965);
 382                                 intel_ring_emit(dev, ring, exec_start);
 383                         } else {
 384                                 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
 385                                                 | (2 << 6));
 386                                 intel_ring_emit(dev, ring, exec_start |
 387                                                 MI_BATCH_NON_SECURE);
 388                         }
 389                 }
 390                 intel_ring_advance(dev, ring);
 391         }
 392
 393         /* XXX breadcrumb */
 394         return 0;
 395 }
 396
 397 static void cleanup_status_page(struct drm_device *dev,
 398                 struct intel_ring_buffer *ring)
 399 {
 400         drm_i915_private_t *dev_priv = dev->dev_private;
 401         struct drm_gem_object *obj;
 402         struct drm_i915_gem_object *obj_priv;
 403
 404         obj = ring->status_page.obj;
 405         if (obj == NULL)
 406                 return;
 407         obj_priv = to_intel_bo(obj);
 408
 409         kunmap(obj_priv->pages[0]);
 410         i915_gem_object_unpin(obj);
 411         drm_gem_object_unreference(obj);
 412         ring->status_page.obj = NULL;
 413
 414         memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 415 }
 416
 417 static int init_status_page(struct drm_device *dev,
 418                 struct intel_ring_buffer *ring)
 419 {
 420         drm_i915_private_t *dev_priv = dev->dev_private;
 421         struct drm_gem_object *obj;
 422         struct drm_i915_gem_object *obj_priv;
 423         int ret;
 424
 425         obj = i915_gem_alloc_object(dev, 4096);
 426         if (obj == NULL) {
 427                 DRM_ERROR("Failed to allocate status page\n");
 428                 ret = -ENOMEM;
 429                 goto err;
 430         }
 431         obj_priv = to_intel_bo(obj);
 432         obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
 433
 434         ret = i915_gem_object_pin(obj, 4096);
 435         if (ret != 0) {
 436                 goto err_unref;
 437         }
 438
 439         ring->status_page.gfx_addr = obj_priv->gtt_offset;
 440         ring->status_page.page_addr = kmap(obj_priv->pages[0]);
 441         if (ring->status_page.page_addr == NULL) {
 442                 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 443                 goto err_unpin;
 444         }
 445         ring->status_page.obj = obj;
 446         memset(ring->status_page.page_addr, 0, PAGE_SIZE);
 447
 448         ring->setup_status_page(dev, ring);
 449         DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
 450                         ring->name, ring->status_page.gfx_addr);
 451
 452         return 0;
 453
 454 err_unpin:
 455         i915_gem_object_unpin(obj);
 456 err_unref:
 457         drm_gem_object_unreference(obj);
 458 err:
 459         return ret;
 460 }
 461
 462
 463 int intel_init_ring_buffer(struct drm_device *dev,
 464                 struct intel_ring_buffer *ring)
 465 {
 466         int ret;
 467         struct drm_i915_gem_object *obj_priv;
 468         struct drm_gem_object *obj;
 469         ring->dev = dev;
 470
 471         if (I915_NEED_GFX_HWS(dev)) {
 472                 ret = init_status_page(dev, ring);
 473                 if (ret)
 474                         return ret;
 475         }
 476
 477         obj = i915_gem_alloc_object(dev, ring->size);
 478         if (obj == NULL) {
 479                 DRM_ERROR("Failed to allocate ringbuffer\n");
 480                 ret = -ENOMEM;
 481                 goto cleanup;
 482         }
 483
 484         ring->gem_object = obj;
 485
 486         ret = i915_gem_object_pin(obj, ring->alignment);
 487         if (ret != 0) {
 488                 drm_gem_object_unreference(obj);
 489                 goto cleanup;
 490         }
 491
 492         obj_priv = to_intel_bo(obj);
 493         ring->map.size = ring->size;
 494         ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
 495         ring->map.type = 0;
 496         ring->map.flags = 0;
 497         ring->map.mtrr = 0;
 498
 499         drm_core_ioremap_wc(&ring->map, dev);
 500         if (ring->map.handle == NULL) {
 501                 DRM_ERROR("Failed to map ringbuffer.\n");
 502                 i915_gem_object_unpin(obj);
 503                 drm_gem_object_unreference(obj);
 504                 ret = -EINVAL;
 505                 goto cleanup;
 506         }
 507
 508         ring->virtual_start = ring->map.handle;
 509         ret = ring->init(dev, ring);
 510         if (ret != 0) {
 511                 intel_cleanup_ring_buffer(dev, ring);
 512                 return ret;
 513         }
 514
 515         if (!drm_core_check_feature(dev, DRIVER_MODESET))
 516                 i915_kernel_lost_context(dev);
 517         else {
 518                 ring->head = ring->get_head(dev, ring);
 519                 ring->tail = ring->get_tail(dev, ring);
 520                 ring->space = ring->head - (ring->tail + 8);
 521                 if (ring->space < 0)
 522                         ring->space += ring->size;
 523         }
 524         INIT_LIST_HEAD(&ring->active_list);
 525         INIT_LIST_HEAD(&ring->request_list);
 526         return ret;
 527 cleanup:
 528         cleanup_status_page(dev, ring);
 529         return ret;
 530 }
 531
 532 void intel_cleanup_ring_buffer(struct drm_device *dev,
 533                 struct intel_ring_buffer *ring)
 534 {
 535         if (ring->gem_object == NULL)
 536                 return;
 537
 538         drm_core_ioremapfree(&ring->map, dev);
 539
 540         i915_gem_object_unpin(ring->gem_object);
 541         drm_gem_object_unreference(ring->gem_object);
 542         ring->gem_object = NULL;
 543         cleanup_status_page(dev, ring);
 544 }
 545
 546 int intel_wrap_ring_buffer(struct drm_device *dev,
 547                 struct intel_ring_buffer *ring)
 548 {
 549         unsigned int *virt;
 550         int rem;
 551         rem = ring->size - ring->tail;
 552
 553         if (ring->space < rem) {
 554                 int ret = intel_wait_ring_buffer(dev, ring, rem);
 555                 if (ret)
 556                         return ret;
 557         }
 558
 559         virt = (unsigned int *)(ring->virtual_start + ring->tail);
 560         rem /= 4;
 561         while (rem--)
 562                 *virt++ = MI_NOOP;
 563
 564         ring->tail = 0;
 565
 566         return 0;
 567 }
 568
 569 int intel_wait_ring_buffer(struct drm_device *dev,
 570                 struct intel_ring_buffer *ring, int n)
 571 {
 572         unsigned long end;
 573
 574         trace_i915_ring_wait_begin (dev);
 575         end = jiffies + 3 * HZ;
 576         do {
 577                 ring->head = ring->get_head(dev, ring);
 578                 ring->space = ring->head - (ring->tail + 8);
 579                 if (ring->space < 0)
 580                         ring->space += ring->size;
 581                 if (ring->space >= n) {
 582                         trace_i915_ring_wait_end (dev);
 583                         return 0;
 584                 }
 585
 586                 if (dev->primary->master) {
 587                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 588                         if (master_priv->sarea_priv)
 589                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
 590                 }
 591                 yield();
 592         } while (!time_after(jiffies, end));
 593         trace_i915_ring_wait_end (dev);
 594         return -EBUSY;
 595 }
 596
 597 void intel_ring_begin(struct drm_device *dev,
 598                 struct intel_ring_buffer *ring, int n)
 599 {
 600         if (unlikely(ring->tail + n > ring->size))
 601                 intel_wrap_ring_buffer(dev, ring);
 602         if (unlikely(ring->space < n))
 603                 intel_wait_ring_buffer(dev, ring, n);
 604 }
 605
 606 void intel_ring_emit(struct drm_device *dev,
 607                 struct intel_ring_buffer *ring, unsigned int data)
 608 {
 609         unsigned int *virt = ring->virtual_start + ring->tail;
 610         *virt = data;
 611         ring->tail += 4;
 612         ring->tail &= ring->size - 1;
 613         ring->space -= 4;
 614 }
 615
 616 void intel_ring_advance(struct drm_device *dev,
 617                 struct intel_ring_buffer *ring)
 618 {
 619         ring->advance_ring(dev, ring);
 620 }
 621
 622 void intel_fill_struct(struct drm_device *dev,
 623                 struct intel_ring_buffer *ring,
 624                 void *data,
 625                 unsigned int len)
 626 {
 627         unsigned int *virt = ring->virtual_start + ring->tail;
 628         BUG_ON((len&~(4-1)) != 0);
 629         intel_ring_begin(dev, ring, len);
 630         memcpy(virt, data, len);
 631         ring->tail += len;
 632         ring->tail &= ring->size - 1;
 633         ring->space -= len;
 634         intel_ring_advance(dev, ring);
 635 }
 636
 637 u32 intel_ring_get_seqno(struct drm_device *dev,
 638                 struct intel_ring_buffer *ring)
 639 {
 640         u32 seqno;
 641         seqno = ring->next_seqno;
 642
 643         /* reserve 0 for non-seqno */
 644         if (++ring->next_seqno == 0)
 645                 ring->next_seqno = 1;
 646         return seqno;
 647 }
 648
 649 struct intel_ring_buffer render_ring = {
 650         .name                   = "render ring",
 651         .regs                   = {
 652                 .ctl = PRB0_CTL,
 653                 .head = PRB0_HEAD,
 654                 .tail = PRB0_TAIL,
 655                 .start = PRB0_START
 656         },
 657         .ring_flag              = I915_EXEC_RENDER,
 658         .size                   = 32 * PAGE_SIZE,
 659         .alignment              = PAGE_SIZE,
 660         .virtual_start          = NULL,
 661         .dev                    = NULL,
 662         .gem_object             = NULL,
 663         .head                   = 0,
 664         .tail                   = 0,
 665         .space                  = 0,
 666         .next_seqno             = 1,
 667         .user_irq_refcount      = 0,
 668         .irq_gem_seqno          = 0,
 669         .waiting_gem_seqno      = 0,
 670         .setup_status_page      = render_setup_status_page,
 671         .init                   = init_render_ring,
 672         .get_head               = render_ring_get_head,
 673         .get_tail               = render_ring_get_tail,
 674         .get_active_head        = render_ring_get_active_head,
 675         .advance_ring           = render_ring_advance_ring,
 676         .flush                  = render_ring_flush,
 677         .add_request            = render_ring_add_request,
 678         .get_gem_seqno          = render_ring_get_gem_seqno,
 679         .user_irq_get           = render_ring_get_user_irq,
 680         .user_irq_put           = render_ring_put_user_irq,
 681         .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
 682         .status_page            = {NULL, 0, NULL},
 683         .map                    = {0,}
 684 };