diff options
Diffstat (limited to '')
31 files changed, 7603 insertions, 5022 deletions
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index fdc833d5cc7b..0ae6a7c5020f 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -9,6 +9,8 @@ i915-y := i915_drv.o i915_dma.o i915_irq.o i915_mem.o \ i915_gem.o \ i915_gem_debug.o \ i915_gem_evict.o \ + i915_gem_execbuffer.o \ + i915_gem_gtt.o \ i915_gem_tiling.o \ i915_trace_points.o \ intel_display.o \ diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index 1f4f3ceb63c7..3601466c5502 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -32,6 +32,7 @@ #include "drmP.h" #include "drm.h" #include "intel_drv.h" +#include "intel_ringbuffer.h" #include "i915_drm.h" #include "i915_drv.h" @@ -72,7 +73,6 @@ static int i915_capabilities(struct seq_file *m, void *data) B(is_broadwater); B(is_crestline); B(has_fbc); - B(has_rc6); B(has_pipe_cxsr); B(has_hotplug); B(cursor_needs_physical); @@ -86,19 +86,19 @@ static int i915_capabilities(struct seq_file *m, void *data) return 0; } -static const char *get_pin_flag(struct drm_i915_gem_object *obj_priv) +static const char *get_pin_flag(struct drm_i915_gem_object *obj) { - if (obj_priv->user_pin_count > 0) + if (obj->user_pin_count > 0) return "P"; - else if (obj_priv->pin_count > 0) + else if (obj->pin_count > 0) return "p"; else return " "; } -static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv) +static const char *get_tiling_flag(struct drm_i915_gem_object *obj) { - switch (obj_priv->tiling_mode) { + switch (obj->tiling_mode) { default: case I915_TILING_NONE: return " "; case I915_TILING_X: return "X"; @@ -106,10 +106,19 @@ static const char *get_tiling_flag(struct drm_i915_gem_object *obj_priv) } } +static const char *agp_type_str(int type) +{ + switch (type) { + case 0: return " uncached"; + case 1: return " snooped"; + default: return ""; + } +} + static void describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) { - seq_printf(m, "%p: %s%s %8zd %08x %08x %d%s%s", + seq_printf(m, "%p: %s%s %8zd %04x %04x %d %d%s%s%s", &obj->base, get_pin_flag(obj), get_tiling_flag(obj), @@ -117,6 +126,8 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) obj->base.read_domains, obj->base.write_domain, obj->last_rendering_seqno, + obj->last_fenced_seqno, + agp_type_str(obj->agp_type == AGP_USER_CACHED_MEMORY), obj->dirty ? " dirty" : "", obj->madv == I915_MADV_DONTNEED ? " purgeable" : ""); if (obj->base.name) @@ -124,7 +135,17 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj) if (obj->fence_reg != I915_FENCE_REG_NONE) seq_printf(m, " (fence: %d)", obj->fence_reg); if (obj->gtt_space != NULL) - seq_printf(m, " (gtt_offset: %08x)", obj->gtt_offset); + seq_printf(m, " (gtt offset: %08x, size: %08x)", + obj->gtt_offset, (unsigned int)obj->gtt_space->size); + if (obj->pin_mappable || obj->fault_mappable) { + char s[3], *t = s; + if (obj->pin_mappable) + *t++ = 'p'; + if (obj->fault_mappable) + *t++ = 'f'; + *t = '\0'; + seq_printf(m, " (%s mappable)", s); + } if (obj->ring != NULL) seq_printf(m, " (%s)", obj->ring->name); } @@ -136,7 +157,7 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data) struct list_head *head; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; size_t total_obj_size, total_gtt_size; int count, ret; @@ -171,12 +192,12 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data) } total_obj_size = total_gtt_size = count = 0; - list_for_each_entry(obj_priv, head, mm_list) { + list_for_each_entry(obj, head, mm_list) { seq_printf(m, " "); - describe_obj(m, obj_priv); + describe_obj(m, obj); seq_printf(m, "\n"); - total_obj_size += obj_priv->base.size; - total_gtt_size += obj_priv->gtt_space->size; + total_obj_size += obj->base.size; + total_gtt_size += obj->gtt_space->size; count++; } mutex_unlock(&dev->struct_mutex); @@ -186,30 +207,116 @@ static int i915_gem_object_list_info(struct seq_file *m, void *data) return 0; } +#define count_objects(list, member) do { \ + list_for_each_entry(obj, list, member) { \ + size += obj->gtt_space->size; \ + ++count; \ + if (obj->map_and_fenceable) { \ + mappable_size += obj->gtt_space->size; \ + ++mappable_count; \ + } \ + } \ +} while(0) + static int i915_gem_object_info(struct seq_file *m, void* data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; struct drm_i915_private *dev_priv = dev->dev_private; + u32 count, mappable_count; + size_t size, mappable_size; + struct drm_i915_gem_object *obj; int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) return ret; - seq_printf(m, "%u objects\n", dev_priv->mm.object_count); - seq_printf(m, "%zu object bytes\n", dev_priv->mm.object_memory); - seq_printf(m, "%u pinned\n", dev_priv->mm.pin_count); - seq_printf(m, "%zu pin bytes\n", dev_priv->mm.pin_memory); - seq_printf(m, "%u objects in gtt\n", dev_priv->mm.gtt_count); - seq_printf(m, "%zu gtt bytes\n", dev_priv->mm.gtt_memory); - seq_printf(m, "%zu gtt total\n", dev_priv->mm.gtt_total); + seq_printf(m, "%u objects, %zu bytes\n", + dev_priv->mm.object_count, + dev_priv->mm.object_memory); + + size = count = mappable_size = mappable_count = 0; + count_objects(&dev_priv->mm.gtt_list, gtt_list); + seq_printf(m, "%u [%u] objects, %zu [%zu] bytes in gtt\n", + count, mappable_count, size, mappable_size); + + size = count = mappable_size = mappable_count = 0; + count_objects(&dev_priv->mm.active_list, mm_list); + count_objects(&dev_priv->mm.flushing_list, mm_list); + seq_printf(m, " %u [%u] active objects, %zu [%zu] bytes\n", + count, mappable_count, size, mappable_size); + + size = count = mappable_size = mappable_count = 0; + count_objects(&dev_priv->mm.pinned_list, mm_list); + seq_printf(m, " %u [%u] pinned objects, %zu [%zu] bytes\n", + count, mappable_count, size, mappable_size); + + size = count = mappable_size = mappable_count = 0; + count_objects(&dev_priv->mm.inactive_list, mm_list); + seq_printf(m, " %u [%u] inactive objects, %zu [%zu] bytes\n", + count, mappable_count, size, mappable_size); + + size = count = mappable_size = mappable_count = 0; + count_objects(&dev_priv->mm.deferred_free_list, mm_list); + seq_printf(m, " %u [%u] freed objects, %zu [%zu] bytes\n", + count, mappable_count, size, mappable_size); + + size = count = mappable_size = mappable_count = 0; + list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { + if (obj->fault_mappable) { + size += obj->gtt_space->size; + ++count; + } + if (obj->pin_mappable) { + mappable_size += obj->gtt_space->size; + ++mappable_count; + } + } + seq_printf(m, "%u pinned mappable objects, %zu bytes\n", + mappable_count, mappable_size); + seq_printf(m, "%u fault mappable objects, %zu bytes\n", + count, size); + + seq_printf(m, "%zu [%zu] gtt total\n", + dev_priv->mm.gtt_total, dev_priv->mm.mappable_gtt_total); mutex_unlock(&dev->struct_mutex); return 0; } +static int i915_gem_gtt_info(struct seq_file *m, void* data) +{ + struct drm_info_node *node = (struct drm_info_node *) m->private; + struct drm_device *dev = node->minor->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_object *obj; + size_t total_obj_size, total_gtt_size; + int count, ret; + + ret = mutex_lock_interruptible(&dev->struct_mutex); + if (ret) + return ret; + + total_obj_size = total_gtt_size = count = 0; + list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { + seq_printf(m, " "); + describe_obj(m, obj); + seq_printf(m, "\n"); + total_obj_size += obj->base.size; + total_gtt_size += obj->gtt_space->size; + count++; + } + + mutex_unlock(&dev->struct_mutex); + + seq_printf(m, "Total %d objects, %zu bytes, %zu GTT size\n", + count, total_obj_size, total_gtt_size); + + return 0; +} + static int i915_gem_pageflip_info(struct seq_file *m, void *data) { @@ -243,14 +350,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data) seq_printf(m, "%d prepares\n", work->pending); if (work->old_fb_obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(work->old_fb_obj); - if(obj_priv) - seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset ); + struct drm_i915_gem_object *obj = work->old_fb_obj; + if (obj) + seq_printf(m, "Old framebuffer gtt_offset 0x%08x\n", obj->gtt_offset); } if (work->pending_flip_obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(work->pending_flip_obj); - if(obj_priv) - seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj_priv->gtt_offset ); + struct drm_i915_gem_object *obj = work->pending_flip_obj; + if (obj) + seq_printf(m, "New framebuffer gtt_offset 0x%08x\n", obj->gtt_offset); } } spin_unlock_irqrestore(&dev->event_lock, flags); @@ -265,44 +372,80 @@ static int i915_gem_request_info(struct seq_file *m, void *data) struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_gem_request *gem_request; - int ret; + int ret, count; ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) return ret; - seq_printf(m, "Request:\n"); - list_for_each_entry(gem_request, &dev_priv->render_ring.request_list, - list) { - seq_printf(m, " %d @ %d\n", - gem_request->seqno, - (int) (jiffies - gem_request->emitted_jiffies)); + count = 0; + if (!list_empty(&dev_priv->ring[RCS].request_list)) { + seq_printf(m, "Render requests:\n"); + list_for_each_entry(gem_request, + &dev_priv->ring[RCS].request_list, + list) { + seq_printf(m, " %d @ %d\n", + gem_request->seqno, + (int) (jiffies - gem_request->emitted_jiffies)); + } + count++; + } + if (!list_empty(&dev_priv->ring[VCS].request_list)) { + seq_printf(m, "BSD requests:\n"); + list_for_each_entry(gem_request, + &dev_priv->ring[VCS].request_list, + list) { + seq_printf(m, " %d @ %d\n", + gem_request->seqno, + (int) (jiffies - gem_request->emitted_jiffies)); + } + count++; + } + if (!list_empty(&dev_priv->ring[BCS].request_list)) { + seq_printf(m, "BLT requests:\n"); + list_for_each_entry(gem_request, + &dev_priv->ring[BCS].request_list, + list) { + seq_printf(m, " %d @ %d\n", + gem_request->seqno, + (int) (jiffies - gem_request->emitted_jiffies)); + } + count++; } mutex_unlock(&dev->struct_mutex); + if (count == 0) + seq_printf(m, "No requests\n"); + return 0; } +static void i915_ring_seqno_info(struct seq_file *m, + struct intel_ring_buffer *ring) +{ + if (ring->get_seqno) { + seq_printf(m, "Current sequence (%s): %d\n", + ring->name, ring->get_seqno(ring)); + seq_printf(m, "Waiter sequence (%s): %d\n", + ring->name, ring->waiting_seqno); + seq_printf(m, "IRQ sequence (%s): %d\n", + ring->name, ring->irq_seqno); + } +} + static int i915_gem_seqno_info(struct seq_file *m, void *data) { struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - int ret; + int ret, i; ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) return ret; - if (dev_priv->render_ring.status_page.page_addr != NULL) { - seq_printf(m, "Current sequence: %d\n", - dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring)); - } else { - seq_printf(m, "Current sequence: hws uninitialized\n"); - } - seq_printf(m, "Waiter sequence: %d\n", - dev_priv->mm.waiting_gem_seqno); - seq_printf(m, "IRQ sequence: %d\n", dev_priv->mm.irq_gem_seqno); + for (i = 0; i < I915_NUM_RINGS; i++) + i915_ring_seqno_info(m, &dev_priv->ring[i]); mutex_unlock(&dev->struct_mutex); @@ -315,7 +458,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - int ret; + int ret, i; ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) @@ -354,16 +497,14 @@ static int i915_interrupt_info(struct seq_file *m, void *data) } seq_printf(m, "Interrupts received: %d\n", atomic_read(&dev_priv->irq_received)); - if (dev_priv->render_ring.status_page.page_addr != NULL) { - seq_printf(m, "Current sequence: %d\n", - dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring)); - } else { - seq_printf(m, "Current sequence: hws uninitialized\n"); + for (i = 0; i < I915_NUM_RINGS; i++) { + if (IS_GEN6(dev)) { + seq_printf(m, "Graphics Interrupt mask (%s): %08x\n", + dev_priv->ring[i].name, + I915_READ_IMR(&dev_priv->ring[i])); + } + i915_ring_seqno_info(m, &dev_priv->ring[i]); } - seq_printf(m, "Waiter sequence: %d\n", - dev_priv->mm.waiting_gem_seqno); - seq_printf(m, "IRQ sequence: %d\n", - dev_priv->mm.irq_gem_seqno); mutex_unlock(&dev->struct_mutex); return 0; @@ -383,29 +524,17 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data) seq_printf(m, "Reserved fences = %d\n", dev_priv->fence_reg_start); seq_printf(m, "Total fences = %d\n", dev_priv->num_fence_regs); for (i = 0; i < dev_priv->num_fence_regs; i++) { - struct drm_gem_object *obj = dev_priv->fence_regs[i].obj; + struct drm_i915_gem_object *obj = dev_priv->fence_regs[i].obj; - if (obj == NULL) { - seq_printf(m, "Fenced object[%2d] = unused\n", i); - } else { - struct drm_i915_gem_object *obj_priv; - - obj_priv = to_intel_bo(obj); - seq_printf(m, "Fenced object[%2d] = %p: %s " - "%08x %08zx %08x %s %08x %08x %d", - i, obj, get_pin_flag(obj_priv), - obj_priv->gtt_offset, - obj->size, obj_priv->stride, - get_tiling_flag(obj_priv), - obj->read_domains, obj->write_domain, - obj_priv->last_rendering_seqno); - if (obj->name) - seq_printf(m, " (name: %d)", obj->name); - seq_printf(m, "\n"); - } + seq_printf(m, "Fenced object[%2d] = ", i); + if (obj == NULL) + seq_printf(m, "unused"); + else + describe_obj(m, obj); + seq_printf(m, "\n"); } - mutex_unlock(&dev->struct_mutex); + mutex_unlock(&dev->struct_mutex); return 0; } @@ -414,10 +543,12 @@ static int i915_hws_info(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - int i; + struct intel_ring_buffer *ring; volatile u32 *hws; + int i; - hws = (volatile u32 *)dev_priv->render_ring.status_page.page_addr; + ring = &dev_priv->ring[(uintptr_t)node->info_ent->data]; + hws = (volatile u32 *)ring->status_page.page_addr; if (hws == NULL) return 0; @@ -431,14 +562,14 @@ static int i915_hws_info(struct seq_file *m, void *data) static void i915_dump_object(struct seq_file *m, struct io_mapping *mapping, - struct drm_i915_gem_object *obj_priv) + struct drm_i915_gem_object *obj) { int page, page_count, i; - page_count = obj_priv->base.size / PAGE_SIZE; + page_count = obj->base.size / PAGE_SIZE; for (page = 0; page < page_count; page++) { u32 *mem = io_mapping_map_wc(mapping, - obj_priv->gtt_offset + page * PAGE_SIZE); + obj->gtt_offset + page * PAGE_SIZE); for (i = 0; i < PAGE_SIZE; i += 4) seq_printf(m, "%08x : %08x\n", i, mem[i / 4]); io_mapping_unmap(mem); @@ -450,25 +581,21 @@ static int i915_batchbuffer_info(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) return ret; - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { - obj = &obj_priv->base; - if (obj->read_domains & I915_GEM_DOMAIN_COMMAND) { - seq_printf(m, "--- gtt_offset = 0x%08x\n", - obj_priv->gtt_offset); - i915_dump_object(m, dev_priv->mm.gtt_mapping, obj_priv); + list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { + if (obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) { + seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset); + i915_dump_object(m, dev_priv->mm.gtt_mapping, obj); } } mutex_unlock(&dev->struct_mutex); - return 0; } @@ -477,19 +604,21 @@ static int i915_ringbuffer_data(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring; int ret; ret = mutex_lock_interruptible(&dev->struct_mutex); if (ret) return ret; - if (!dev_priv->render_ring.gem_object) { + ring = &dev_priv->ring[(uintptr_t)node->info_ent->data]; + if (!ring->obj) { seq_printf(m, "No ringbuffer setup\n"); } else { - u8 *virt = dev_priv->render_ring.virtual_start; + u8 *virt = ring->virtual_start; uint32_t off; - for (off = 0; off < dev_priv->render_ring.size; off += 4) { + for (off = 0; off < ring->size; off += 4) { uint32_t *ptr = (uint32_t *)(virt + off); seq_printf(m, "%08x : %08x\n", off, *ptr); } @@ -504,19 +633,38 @@ static int i915_ringbuffer_info(struct seq_file *m, void *data) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - unsigned int head, tail; + struct intel_ring_buffer *ring; - head = I915_READ(PRB0_HEAD) & HEAD_ADDR; - tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; + ring = &dev_priv->ring[(uintptr_t)node->info_ent->data]; + if (ring->size == 0) + return 0; - seq_printf(m, "RingHead : %08x\n", head); - seq_printf(m, "RingTail : %08x\n", tail); - seq_printf(m, "RingSize : %08lx\n", dev_priv->render_ring.size); - seq_printf(m, "Acthd : %08x\n", I915_READ(INTEL_INFO(dev)->gen >= 4 ? ACTHD_I965 : ACTHD)); + seq_printf(m, "Ring %s:\n", ring->name); + seq_printf(m, " Head : %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR); + seq_printf(m, " Tail : %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR); + seq_printf(m, " Size : %08x\n", ring->size); + seq_printf(m, " Active : %08x\n", intel_ring_get_active_head(ring)); + seq_printf(m, " NOPID : %08x\n", I915_READ_NOPID(ring)); + if (IS_GEN6(dev)) { + seq_printf(m, " Sync 0 : %08x\n", I915_READ_SYNC_0(ring)); + seq_printf(m, " Sync 1 : %08x\n", I915_READ_SYNC_1(ring)); + } + seq_printf(m, " Control : %08x\n", I915_READ_CTL(ring)); + seq_printf(m, " Start : %08x\n", I915_READ_START(ring)); return 0; } +static const char *ring_str(int ring) +{ + switch (ring) { + case RING_RENDER: return " render"; + case RING_BSD: return " bsd"; + case RING_BLT: return " blt"; + default: return ""; + } +} + static const char *pin_flag(int pinned) { if (pinned > 0) @@ -547,6 +695,37 @@ static const char *purgeable_flag(int purgeable) return purgeable ? " purgeable" : ""; } +static void print_error_buffers(struct seq_file *m, + const char *name, + struct drm_i915_error_buffer *err, + int count) +{ + seq_printf(m, "%s [%d]:\n", name, count); + + while (count--) { + seq_printf(m, " %08x %8u %04x %04x %08x%s%s%s%s%s%s", + err->gtt_offset, + err->size, + err->read_domains, + err->write_domain, + err->seqno, + pin_flag(err->pinned), + tiling_flag(err->tiling), + dirty_flag(err->dirty), + purgeable_flag(err->purgeable), + ring_str(err->ring), + agp_type_str(err->agp_type)); + + if (err->name) + seq_printf(m, " (name: %d)", err->name); + if (err->fence_reg != I915_FENCE_REG_NONE) + seq_printf(m, " (fence: %d)", err->fence_reg); + + seq_printf(m, "\n"); + err++; + } +} + static int i915_error_state(struct seq_file *m, void *unused) { struct drm_info_node *node = (struct drm_info_node *) m->private; @@ -568,47 +747,54 @@ static int i915_error_state(struct seq_file *m, void *unused) error->time.tv_usec); seq_printf(m, "PCI ID: 0x%04x\n", dev->pci_device); seq_printf(m, "EIR: 0x%08x\n", error->eir); - seq_printf(m, " PGTBL_ER: 0x%08x\n", error->pgtbl_er); - seq_printf(m, " INSTPM: 0x%08x\n", error->instpm); + seq_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er); + if (INTEL_INFO(dev)->gen >= 6) { + seq_printf(m, "ERROR: 0x%08x\n", error->error); + seq_printf(m, "Blitter command stream:\n"); + seq_printf(m, " ACTHD: 0x%08x\n", error->bcs_acthd); + seq_printf(m, " IPEIR: 0x%08x\n", error->bcs_ipeir); + seq_printf(m, " IPEHR: 0x%08x\n", error->bcs_ipehr); + seq_printf(m, " INSTDONE: 0x%08x\n", error->bcs_instdone); + seq_printf(m, " seqno: 0x%08x\n", error->bcs_seqno); + seq_printf(m, "Video (BSD) command stream:\n"); + seq_printf(m, " ACTHD: 0x%08x\n", error->vcs_acthd); + seq_printf(m, " IPEIR: 0x%08x\n", error->vcs_ipeir); + seq_printf(m, " IPEHR: 0x%08x\n", error->vcs_ipehr); + seq_printf(m, " INSTDONE: 0x%08x\n", error->vcs_instdone); + seq_printf(m, " seqno: 0x%08x\n", error->vcs_seqno); + } + seq_printf(m, "Render command stream:\n"); + seq_printf(m, " ACTHD: 0x%08x\n", error->acthd); seq_printf(m, " IPEIR: 0x%08x\n", error->ipeir); seq_printf(m, " IPEHR: 0x%08x\n", error->ipehr); seq_printf(m, " INSTDONE: 0x%08x\n", error->instdone); - seq_printf(m, " ACTHD: 0x%08x\n", error->acthd); if (INTEL_INFO(dev)->gen >= 4) { - seq_printf(m, " INSTPS: 0x%08x\n", error->instps); seq_printf(m, " INSTDONE1: 0x%08x\n", error->instdone1); + seq_printf(m, " INSTPS: 0x%08x\n", error->instps); } - seq_printf(m, "seqno: 0x%08x\n", error->seqno); - - if (error->active_bo_count) { - seq_printf(m, "Buffers [%d]:\n", error->active_bo_count); - - for (i = 0; i < error->active_bo_count; i++) { - seq_printf(m, " %08x %8zd %08x %08x %08x%s%s%s%s", - error->active_bo[i].gtt_offset, - error->active_bo[i].size, - error->active_bo[i].read_domains, - error->active_bo[i].write_domain, - error->active_bo[i].seqno, - pin_flag(error->active_bo[i].pinned), - tiling_flag(error->active_bo[i].tiling), - dirty_flag(error->active_bo[i].dirty), - purgeable_flag(error->active_bo[i].purgeable)); - - if (error->active_bo[i].name) - seq_printf(m, " (name: %d)", error->active_bo[i].name); - if (error->active_bo[i].fence_reg != I915_FENCE_REG_NONE) - seq_printf(m, " (fence: %d)", error->active_bo[i].fence_reg); - - seq_printf(m, "\n"); - } - } + seq_printf(m, " INSTPM: 0x%08x\n", error->instpm); + seq_printf(m, " seqno: 0x%08x\n", error->seqno); + + for (i = 0; i < 16; i++) + seq_printf(m, " fence[%d] = %08llx\n", i, error->fence[i]); + + if (error->active_bo) + print_error_buffers(m, "Active", + error->active_bo, + error->active_bo_count); + + if (error->pinned_bo) + print_error_buffers(m, "Pinned", + error->pinned_bo, + error->pinned_bo_count); for (i = 0; i < ARRAY_SIZE(error->batchbuffer); i++) { if (error->batchbuffer[i]) { struct drm_i915_error_object *obj = error->batchbuffer[i]; - seq_printf(m, "--- gtt_offset = 0x%08x\n", obj->gtt_offset); + seq_printf(m, "%s --- gtt_offset = 0x%08x\n", + dev_priv->ring[i].name, + obj->gtt_offset); offset = 0; for (page = 0; page < obj->page_count; page++) { for (elt = 0; elt < PAGE_SIZE/4; elt++) { @@ -635,6 +821,9 @@ static int i915_error_state(struct seq_file *m, void *unused) if (error->overlay) intel_overlay_print_error_state(m, error->overlay); + if (error->display) + intel_display_print_error_state(m, dev, error->display); + out: spin_unlock_irqrestore(&dev_priv->error_lock, flags); @@ -658,15 +847,51 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused) struct drm_info_node *node = (struct drm_info_node *) m->private; struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; - u16 rgvswctl = I915_READ16(MEMSWCTL); - u16 rgvstat = I915_READ16(MEMSTAT_ILK); - seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); - seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f); - seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> - MEMSTAT_VID_SHIFT); - seq_printf(m, "Current P-state: %d\n", - (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); + if (IS_GEN5(dev)) { + u16 rgvswctl = I915_READ16(MEMSWCTL); + u16 rgvstat = I915_READ16(MEMSTAT_ILK); + + seq_printf(m, "Requested P-state: %d\n", (rgvswctl >> 8) & 0xf); + seq_printf(m, "Requested VID: %d\n", rgvswctl & 0x3f); + seq_printf(m, "Current VID: %d\n", (rgvstat & MEMSTAT_VID_MASK) >> + MEMSTAT_VID_SHIFT); + seq_printf(m, "Current P-state: %d\n", + (rgvstat & MEMSTAT_PSTATE_MASK) >> MEMSTAT_PSTATE_SHIFT); + } else if (IS_GEN6(dev)) { + u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); + u32 rp_state_limits = I915_READ(GEN6_RP_STATE_LIMITS); + u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + int max_freq; + + /* RPSTAT1 is in the GT power well */ + __gen6_force_wake_get(dev_priv); + + seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status); + seq_printf(m, "RPSTAT1: 0x%08x\n", I915_READ(GEN6_RPSTAT1)); + seq_printf(m, "Render p-state ratio: %d\n", + (gt_perf_status & 0xff00) >> 8); + seq_printf(m, "Render p-state VID: %d\n", + gt_perf_status & 0xff); + seq_printf(m, "Render p-state limit: %d\n", + rp_state_limits & 0xff); + + max_freq = (rp_state_cap & 0xff0000) >> 16; + seq_printf(m, "Lowest (RPN) frequency: %dMHz\n", + max_freq * 100); + + max_freq = (rp_state_cap & 0xff00) >> 8; + seq_printf(m, "Nominal (RP1) frequency: %dMHz\n", + max_freq * 100); + + max_freq = rp_state_cap & 0xff; + seq_printf(m, "Max non-overclocked (RP0) frequency: %dMHz\n", + max_freq * 100); + + __gen6_force_wake_put(dev_priv); + } else { + seq_printf(m, "no P-state info available\n"); + } return 0; } @@ -715,7 +940,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused) struct drm_device *dev = node->minor->dev; drm_i915_private_t *dev_priv = dev->dev_private; u32 rgvmodectl = I915_READ(MEMMODECTL); - u32 rstdbyctl = I915_READ(MCHBAR_RENDER_STANDBY); + u32 rstdbyctl = I915_READ(RSTDBYCTL); u16 crstandvid = I915_READ16(CRSTANDVID); seq_printf(m, "HD boost: %s\n", (rgvmodectl & MEMMODE_BOOST_EN) ? @@ -738,6 +963,30 @@ static int i915_drpc_info(struct seq_file *m, void *unused) seq_printf(m, "RS2 VID: %d\n", ((crstandvid >> 8) & 0x3f)); seq_printf(m, "Render standby enabled: %s\n", (rstdbyctl & RCX_SW_EXIT) ? "no" : "yes"); + seq_printf(m, "Current RS state: "); + switch (rstdbyctl & RSX_STATUS_MASK) { + case RSX_STATUS_ON: + seq_printf(m, "on\n"); + break; + case RSX_STATUS_RC1: + seq_printf(m, "RC1\n"); + break; + case RSX_STATUS_RC1E: + seq_printf(m, "RC1E\n"); + break; + case RSX_STATUS_RS1: + seq_printf(m, "RS1\n"); + break; + case RSX_STATUS_RS2: + seq_printf(m, "RS2 (RC6)\n"); + break; + case RSX_STATUS_RS3: + seq_printf(m, "RC3 (RC6+)\n"); + break; + default: + seq_printf(m, "unknown\n"); + break; + } return 0; } @@ -794,7 +1043,7 @@ static int i915_sr_status(struct seq_file *m, void *unused) drm_i915_private_t *dev_priv = dev->dev_private; bool sr_enabled = false; - if (IS_GEN5(dev)) + if (HAS_PCH_SPLIT(dev)) sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN; else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev)) sr_enabled = I915_READ(FW_BLC_SELF) & FW_BLC_SELF_EN; @@ -886,7 +1135,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) fb->base.height, fb->base.depth, fb->base.bits_per_pixel); - describe_obj(m, to_intel_bo(fb->obj)); + describe_obj(m, fb->obj); seq_printf(m, "\n"); list_for_each_entry(fb, &dev->mode_config.fb_list, base.head) { @@ -898,7 +1147,7 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data) fb->base.height, fb->base.depth, fb->base.bits_per_pixel); - describe_obj(m, to_intel_bo(fb->obj)); + describe_obj(m, fb->obj); seq_printf(m, "\n"); } @@ -943,7 +1192,6 @@ i915_wedged_write(struct file *filp, loff_t *ppos) { struct drm_device *dev = filp->private_data; - drm_i915_private_t *dev_priv = dev->dev_private; char buf[20]; int val = 1; @@ -959,12 +1207,7 @@ i915_wedged_write(struct file *filp, } DRM_INFO("Manually setting wedged to %d\n", val); - - atomic_set(&dev_priv->mm.wedged, val); - if (val) { - wake_up_all(&dev_priv->irq_queue); - queue_work(dev_priv->wq, &dev_priv->error_work); - } + i915_handle_error(dev, val); return cnt; } @@ -1018,6 +1261,7 @@ static int i915_wedged_create(struct dentry *root, struct drm_minor *minor) static struct drm_info_list i915_debugfs_list[] = { {"i915_capabilities", i915_capabilities, 0, 0}, {"i915_gem_objects", i915_gem_object_info, 0}, + {"i915_gem_gtt", i915_gem_gtt_info, 0}, {"i915_gem_active", i915_gem_object_list_info, 0, (void *) ACTIVE_LIST}, {"i915_gem_flushing", i915_gem_object_list_info, 0, (void *) FLUSHING_LIST}, {"i915_gem_inactive", i915_gem_object_list_info, 0, (void *) INACTIVE_LIST}, @@ -1028,9 +1272,15 @@ static struct drm_info_list i915_debugfs_list[] = { {"i915_gem_seqno", i915_gem_seqno_info, 0}, {"i915_gem_fence_regs", i915_gem_fence_regs_info, 0}, {"i915_gem_interrupt", i915_interrupt_info, 0}, - {"i915_gem_hws", i915_hws_info, 0}, - {"i915_ringbuffer_data", i915_ringbuffer_data, 0}, - {"i915_ringbuffer_info", i915_ringbuffer_info, 0}, + {"i915_gem_hws", i915_hws_info, 0, (void *)RCS}, + {"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS}, + {"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS}, + {"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS}, + {"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS}, + {"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS}, + {"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS}, + {"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS}, + {"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS}, {"i915_batchbuffers", i915_batchbuffer_info, 0}, {"i915_error_state", i915_error_state, 0}, {"i915_rstdby_delays", i915_rstdby_delays, 0}, diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c index cb900dc83d95..e33d9be7df3b 100644 --- a/drivers/gpu/drm/i915/i915_dma.c +++ b/drivers/gpu/drm/i915/i915_dma.c @@ -50,6 +50,8 @@ static int i915_init_phys_hws(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring = LP_RING(dev_priv); + /* Program Hardware Status Page */ dev_priv->status_page_dmah = drm_pci_alloc(dev, PAGE_SIZE, PAGE_SIZE); @@ -58,11 +60,10 @@ static int i915_init_phys_hws(struct drm_device *dev) DRM_ERROR("Can not allocate hardware status page\n"); return -ENOMEM; } - dev_priv->render_ring.status_page.page_addr - = dev_priv->status_page_dmah->vaddr; + ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; dev_priv->dma_status_page = dev_priv->status_page_dmah->busaddr; - memset(dev_priv->render_ring.status_page.page_addr, 0, PAGE_SIZE); + memset(ring->status_page.page_addr, 0, PAGE_SIZE); if (INTEL_INFO(dev)->gen >= 4) dev_priv->dma_status_page |= (dev_priv->dma_status_page >> 28) & @@ -80,13 +81,15 @@ static int i915_init_phys_hws(struct drm_device *dev) static void i915_free_hws(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring = LP_RING(dev_priv); + if (dev_priv->status_page_dmah) { drm_pci_free(dev, dev_priv->status_page_dmah); dev_priv->status_page_dmah = NULL; } - if (dev_priv->render_ring.status_page.gfx_addr) { - dev_priv->render_ring.status_page.gfx_addr = 0; + if (ring->status_page.gfx_addr) { + ring->status_page.gfx_addr = 0; drm_core_ioremapfree(&dev_priv->hws_map, dev); } @@ -98,7 +101,7 @@ void i915_kernel_lost_context(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv; - struct intel_ring_buffer *ring = &dev_priv->render_ring; + struct intel_ring_buffer *ring = LP_RING(dev_priv); /* * We should never lose context on the ring with modesetting @@ -107,8 +110,8 @@ void i915_kernel_lost_context(struct drm_device * dev) if (drm_core_check_feature(dev, DRIVER_MODESET)) return; - ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR; - ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR; + ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; + ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; ring->space = ring->head - (ring->tail + 8); if (ring->space < 0) ring->space += ring->size; @@ -124,6 +127,8 @@ void i915_kernel_lost_context(struct drm_device * dev) static int i915_dma_cleanup(struct drm_device * dev) { drm_i915_private_t *dev_priv = dev->dev_private; + int i; + /* Make sure interrupts are disabled here because the uninstall ioctl * may not have been called from userspace and after dev_private * is freed, it's too late. @@ -132,9 +137,8 @@ static int i915_dma_cleanup(struct drm_device * dev) drm_irq_uninstall(dev); mutex_lock(&dev->struct_mutex); - intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); - intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); - intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring); + for (i = 0; i < I915_NUM_RINGS; i++) + intel_cleanup_ring_buffer(&dev_priv->ring[i]); mutex_unlock(&dev->struct_mutex); /* Clear the HWS virtual address at teardown */ @@ -148,6 +152,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; + int ret; master_priv->sarea = drm_getsarea(dev); if (master_priv->sarea) { @@ -158,33 +163,22 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) } if (init->ring_size != 0) { - if (dev_priv->render_ring.gem_object != NULL) { + if (LP_RING(dev_priv)->obj != NULL) { i915_dma_cleanup(dev); DRM_ERROR("Client tried to initialize ringbuffer in " "GEM mode\n"); return -EINVAL; } - dev_priv->render_ring.size = init->ring_size; - - dev_priv->render_ring.map.offset = init->ring_start; - dev_priv->render_ring.map.size = init->ring_size; - dev_priv->render_ring.map.type = 0; - dev_priv->render_ring.map.flags = 0; - dev_priv->render_ring.map.mtrr = 0; - - drm_core_ioremap_wc(&dev_priv->render_ring.map, dev); - - if (dev_priv->render_ring.map.handle == NULL) { + ret = intel_render_ring_init_dri(dev, + init->ring_start, + init->ring_size); + if (ret) { i915_dma_cleanup(dev); - DRM_ERROR("can not ioremap virtual address for" - " ring buffer\n"); - return -ENOMEM; + return ret; } } - dev_priv->render_ring.virtual_start = dev_priv->render_ring.map.handle; - dev_priv->cpp = init->cpp; dev_priv->back_offset = init->back_offset; dev_priv->front_offset = init->front_offset; @@ -202,12 +196,10 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init) static int i915_dma_resume(struct drm_device * dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + struct intel_ring_buffer *ring = LP_RING(dev_priv); - struct intel_ring_buffer *ring; DRM_DEBUG_DRIVER("%s\n", __func__); - ring = &dev_priv->render_ring; - if (ring->map.handle == NULL) { DRM_ERROR("can not ioremap virtual address for" " ring buffer\n"); @@ -222,7 +214,7 @@ static int i915_dma_resume(struct drm_device * dev) DRM_DEBUG_DRIVER("hw status page @ %p\n", ring->status_page.page_addr); if (ring->status_page.gfx_addr != 0) - intel_ring_setup_status_page(dev, ring); + intel_ring_setup_status_page(ring); else I915_WRITE(HWS_PGA, dev_priv->dma_status_page); @@ -264,7 +256,7 @@ static int i915_dma_init(struct drm_device *dev, void *data, * instruction detected will be given a size of zero, which is a * signal to abort the rest of the buffer. */ -static int do_validate_cmd(int cmd) +static int validate_cmd(int cmd) { switch (((cmd >> 29) & 0x7)) { case 0x0: @@ -322,40 +314,27 @@ static int do_validate_cmd(int cmd) return 0; } -static int validate_cmd(int cmd) -{ - int ret = do_validate_cmd(cmd); - -/* printk("validate_cmd( %x ): %d\n", cmd, ret); */ - - return ret; -} - static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords) { drm_i915_private_t *dev_priv = dev->dev_private; - int i; + int i, ret; - if ((dwords+1) * sizeof(int) >= dev_priv->render_ring.size - 8) + if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8) return -EINVAL; - BEGIN_LP_RING((dwords+1)&~1); - for (i = 0; i < dwords;) { - int cmd, sz; - - cmd = buffer[i]; - - if ((sz = validate_cmd(cmd)) == 0 || i + sz > dwords) + int sz = validate_cmd(buffer[i]); + if (sz == 0 || i + sz > dwords) return -EINVAL; - - OUT_RING(cmd); - - while (++i, --sz) { - OUT_RING(buffer[i]); - } + i += sz; } + ret = BEGIN_LP_RING((dwords+1)&~1); + if (ret) + return ret; + + for (i = 0; i < dwords; i++) + OUT_RING(buffer[i]); if (dwords & 1) OUT_RING(0); @@ -366,34 +345,41 @@ static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords) int i915_emit_box(struct drm_device *dev, - struct drm_clip_rect *boxes, - int i, int DR1, int DR4) + struct drm_clip_rect *box, + int DR1, int DR4) { - struct drm_clip_rect box = boxes[i]; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; - if (box.y2 <= box.y1 || box.x2 <= box.x1 || box.y2 <= 0 || box.x2 <= 0) { + if (box->y2 <= box->y1 || box->x2 <= box->x1 || + box->y2 <= 0 || box->x2 <= 0) { DRM_ERROR("Bad box %d,%d..%d,%d\n", - box.x1, box.y1, box.x2, box.y2); + box->x1, box->y1, box->x2, box->y2); return -EINVAL; } if (INTEL_INFO(dev)->gen >= 4) { - BEGIN_LP_RING(4); + ret = BEGIN_LP_RING(4); + if (ret) + return ret; + OUT_RING(GFX_OP_DRAWRECT_INFO_I965); - OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); - OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); + OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); + OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); - ADVANCE_LP_RING(); } else { - BEGIN_LP_RING(6); + ret = BEGIN_LP_RING(6); + if (ret) + return ret; + OUT_RING(GFX_OP_DRAWRECT_INFO); OUT_RING(DR1); - OUT_RING((box.x1 & 0xffff) | (box.y1 << 16)); - OUT_RING(((box.x2 - 1) & 0xffff) | ((box.y2 - 1) << 16)); + OUT_RING((box->x1 & 0xffff) | (box->y1 << 16)); + OUT_RING(((box->x2 - 1) & 0xffff) | ((box->y2 - 1) << 16)); OUT_RING(DR4); OUT_RING(0); - ADVANCE_LP_RING(); } + ADVANCE_LP_RING(); return 0; } @@ -413,12 +399,13 @@ static void i915_emit_breadcrumb(struct drm_device *dev) if (master_priv->sarea_priv) master_priv->sarea_priv->last_enqueue = dev_priv->counter; - BEGIN_LP_RING(4); - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(dev_priv->counter); - OUT_RING(0); - ADVANCE_LP_RING(); + if (BEGIN_LP_RING(4) == 0) { + OUT_RING(MI_STORE_DWORD_INDEX); + OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + OUT_RING(dev_priv->counter); + OUT_RING(0); + ADVANCE_LP_RING(); + } } static int i915_dispatch_cmdbuffer(struct drm_device * dev, @@ -440,7 +427,7 @@ static int i915_dispatch_cmdbuffer(struct drm_device * dev, for (i = 0; i < count; i++) { if (i < nbox) { - ret = i915_emit_box(dev, cliprects, i, + ret = i915_emit_box(dev, &cliprects[i], cmd->DR1, cmd->DR4); if (ret) return ret; @@ -459,8 +446,9 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev, drm_i915_batchbuffer_t * batch, struct drm_clip_rect *cliprects) { + struct drm_i915_private *dev_priv = dev->dev_private; int nbox = batch->num_cliprects; - int i = 0, count; + int i, count, ret; if ((batch->start | batch->used) & 0x7) { DRM_ERROR("alignment"); @@ -470,17 +458,19 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev, i915_kernel_lost_context(dev); count = nbox ? nbox : 1; - for (i = 0; i < count; i++) { if (i < nbox) { - int ret = i915_emit_box(dev, cliprects, i, - batch->DR1, batch->DR4); + ret = i915_emit_box(dev, &cliprects[i], + batch->DR1, batch->DR4); if (ret) return ret; } if (!IS_I830(dev) && !IS_845G(dev)) { - BEGIN_LP_RING(2); + ret = BEGIN_LP_RING(2); + if (ret) + return ret; + if (INTEL_INFO(dev)->gen >= 4) { OUT_RING(MI_BATCH_BUFFER_START | (2 << 6) | MI_BATCH_NON_SECURE_I965); OUT_RING(batch->start); @@ -488,26 +478,29 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev, OUT_RING(MI_BATCH_BUFFER_START | (2 << 6)); OUT_RING(batch->start | MI_BATCH_NON_SECURE); } - ADVANCE_LP_RING(); } else { - BEGIN_LP_RING(4); + ret = BEGIN_LP_RING(4); + if (ret) + return ret; + OUT_RING(MI_BATCH_BUFFER); OUT_RING(batch->start | MI_BATCH_NON_SECURE); OUT_RING(batch->start + batch->used - 4); OUT_RING(0); - ADVANCE_LP_RING(); } + ADVANCE_LP_RING(); } if (IS_G4X(dev) || IS_GEN5(dev)) { - BEGIN_LP_RING(2); - OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP); - OUT_RING(MI_NOOP); - ADVANCE_LP_RING(); + if (BEGIN_LP_RING(2) == 0) { + OUT_RING(MI_FLUSH | MI_NO_WRITE_FLUSH | MI_INVALIDATE_ISP); + OUT_RING(MI_NOOP); + ADVANCE_LP_RING(); + } } - i915_emit_breadcrumb(dev); + i915_emit_breadcrumb(dev); return 0; } @@ -516,6 +509,7 @@ static int i915_dispatch_flip(struct drm_device * dev) drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; + int ret; if (!master_priv->sarea_priv) return -EINVAL; @@ -527,12 +521,13 @@ static int i915_dispatch_flip(struct drm_device * dev) i915_kernel_lost_context(dev); - BEGIN_LP_RING(2); + ret = BEGIN_LP_RING(10); + if (ret) + return ret; + OUT_RING(MI_FLUSH | MI_READ_FLUSH); OUT_RING(0); - ADVANCE_LP_RING(); - BEGIN_LP_RING(6); OUT_RING(CMD_OP_DISPLAYBUFFER_INFO | ASYNC_FLIP); OUT_RING(0); if (dev_priv->current_page == 0) { @@ -543,33 +538,32 @@ static int i915_dispatch_flip(struct drm_device * dev) dev_priv->current_page = 0; } OUT_RING(0); - ADVANCE_LP_RING(); - BEGIN_LP_RING(2); OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_PLANE_A_FLIP); OUT_RING(0); + ADVANCE_LP_RING(); master_priv->sarea_priv->last_enqueue = dev_priv->counter++; - BEGIN_LP_RING(4); - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(dev_priv->counter); - OUT_RING(0); - ADVANCE_LP_RING(); + if (BEGIN_LP_RING(4) == 0) { + OUT_RING(MI_STORE_DWORD_INDEX); + OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + OUT_RING(dev_priv->counter); + OUT_RING(0); + ADVANCE_LP_RING(); + } master_priv->sarea_priv->pf_current_page = dev_priv->current_page; return 0; } -static int i915_quiescent(struct drm_device * dev) +static int i915_quiescent(struct drm_device *dev) { - drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring = LP_RING(dev->dev_private); i915_kernel_lost_context(dev); - return intel_wait_ring_buffer(dev, &dev_priv->render_ring, - dev_priv->render_ring.size - 8); + return intel_wait_ring_buffer(ring, ring->size - 8); } static int i915_flush_ioctl(struct drm_device *dev, void *data, @@ -768,9 +762,15 @@ static int i915_getparam(struct drm_device *dev, void *data, case I915_PARAM_HAS_BLT: value = HAS_BLT(dev); break; + case I915_PARAM_HAS_RELAXED_FENCING: + value = 1; + break; case I915_PARAM_HAS_COHERENT_RINGS: value = 1; break; + case I915_PARAM_HAS_EXEC_CONSTANTS: + value = INTEL_INFO(dev)->gen >= 4; + break; default: DRM_DEBUG_DRIVER("Unknown parameter %d\n", param->param); @@ -826,7 +826,7 @@ static int i915_set_status_page(struct drm_device *dev, void *data, { drm_i915_private_t *dev_priv = dev->dev_private; drm_i915_hws_addr_t *hws = data; - struct intel_ring_buffer *ring = &dev_priv->render_ring; + struct intel_ring_buffer *ring = LP_RING(dev_priv); if (!I915_NEED_GFX_HWS(dev)) return -EINVAL; @@ -1005,73 +1005,47 @@ intel_teardown_mchbar(struct drm_device *dev) #define PTE_VALID (1 << 0) /** - * i915_gtt_to_phys - take a GTT address and turn it into a physical one + * i915_stolen_to_phys - take an offset into stolen memory and turn it into + * a physical one * @dev: drm device - * @gtt_addr: address to translate + * @offset: address to translate * - * Some chip functions require allocations from stolen space but need the - * physical address of the memory in question. We use this routine - * to get a physical address suitable for register programming from a given - * GTT address. + * Some chip functions require allocations from stolen space and need the + * physical address of the memory in question. */ -static unsigned long i915_gtt_to_phys(struct drm_device *dev, - unsigned long gtt_addr) +static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset) { - unsigned long *gtt; - unsigned long entry, phys; - int gtt_bar = IS_GEN2(dev) ? 1 : 0; - int gtt_offset, gtt_size; - - if (INTEL_INFO(dev)->gen >= 4) { - if (IS_G4X(dev) || INTEL_INFO(dev)->gen > 4) { - gtt_offset = 2*1024*1024; - gtt_size = 2*1024*1024; - } else { - gtt_offset = 512*1024; - gtt_size = 512*1024; - } + struct drm_i915_private *dev_priv = dev->dev_private; + struct pci_dev *pdev = dev_priv->bridge_dev; + u32 base; + +#if 0 + /* On the machines I have tested the Graphics Base of Stolen Memory + * is unreliable, so compute the base by subtracting the stolen memory + * from the Top of Low Usable DRAM which is where the BIOS places + * the graphics stolen memory. + */ + if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) { + /* top 32bits are reserved = 0 */ + pci_read_config_dword(pdev, 0xA4, &base); } else { - gtt_bar = 3; - gtt_offset = 0; - gtt_size = pci_resource_len(dev->pdev, gtt_bar); - } - - gtt = ioremap_wc(pci_resource_start(dev->pdev, gtt_bar) + gtt_offset, - gtt_size); - if (!gtt) { - DRM_ERROR("ioremap of GTT failed\n"); - return 0; - } - - entry = *(volatile u32 *)(gtt + (gtt_addr / 1024)); - - DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, PTE: 0x%08lx\n", gtt_addr, entry); - - /* Mask out these reserved bits on this hardware. */ - if (INTEL_INFO(dev)->gen < 4 && !IS_G33(dev)) - entry &= ~PTE_ADDRESS_MASK_HIGH; - - /* If it's not a mapping type we know, then bail. */ - if ((entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_UNCACHED && - (entry & PTE_MAPPING_TYPE_MASK) != PTE_MAPPING_TYPE_CACHED) { - iounmap(gtt); - return 0; - } - - if (!(entry & PTE_VALID)) { - DRM_ERROR("bad GTT entry in stolen space\n"); - iounmap(gtt); - return 0; + /* XXX presume 8xx is the same as i915 */ + pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base); + } +#else + if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) { + u16 val; + pci_read_config_word(pdev, 0xb0, &val); + base = val >> 4 << 20; + } else { + u8 val; + pci_read_config_byte(pdev, 0x9c, &val); + base = val >> 3 << 27; } + base -= dev_priv->mm.gtt->stolen_size; +#endif - iounmap(gtt); - - phys =(entry & PTE_ADDRESS_MASK) | - ((uint64_t)(entry & PTE_ADDRESS_MASK_HIGH) << (32 - 4)); - - DRM_DEBUG_DRIVER("GTT addr: 0x%08lx, phys addr: 0x%08lx\n", gtt_addr, phys); - - return phys; + return base + offset; } static void i915_warn_stolen(struct drm_device *dev) @@ -1087,54 +1061,35 @@ static void i915_setup_compression(struct drm_device *dev, int size) unsigned long cfb_base; unsigned long ll_base = 0; - /* Leave 1M for line length buffer & misc. */ - compressed_fb = drm_mm_search_free(&dev_priv->mm.vram, size, 4096, 0); - if (!compressed_fb) { - dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; - i915_warn_stolen(dev); - return; - } + compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0); + if (compressed_fb) + compressed_fb = drm_mm_get_block(compressed_fb, size, 4096); + if (!compressed_fb) + goto err; - compressed_fb = drm_mm_get_block(compressed_fb, size, 4096); - if (!compressed_fb) { - i915_warn_stolen(dev); - dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; - return; - } + cfb_base = i915_stolen_to_phys(dev, compressed_fb->start); + if (!cfb_base) + goto err_fb; - cfb_base = i915_gtt_to_phys(dev, compressed_fb->start); - if (!cfb_base) { - DRM_ERROR("failed to get stolen phys addr, disabling FBC\n"); - drm_mm_put_block(compressed_fb); - } - - if (!(IS_GM45(dev) || IS_IRONLAKE_M(dev))) { - compressed_llb = drm_mm_search_free(&dev_priv->mm.vram, 4096, - 4096, 0); - if (!compressed_llb) { - i915_warn_stolen(dev); - return; - } - - compressed_llb = drm_mm_get_block(compressed_llb, 4096, 4096); - if (!compressed_llb) { - i915_warn_stolen(dev); - return; - } + if (!(IS_GM45(dev) || HAS_PCH_SPLIT(dev))) { + compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen, + 4096, 4096, 0); + if (compressed_llb) + compressed_llb = drm_mm_get_block(compressed_llb, + 4096, 4096); + if (!compressed_llb) + goto err_fb; - ll_base = i915_gtt_to_phys(dev, compressed_llb->start); - if (!ll_base) { - DRM_ERROR("failed to get stolen phys addr, disabling FBC\n"); - drm_mm_put_block(compressed_fb); - drm_mm_put_block(compressed_llb); - } + ll_base = i915_stolen_to_phys(dev, compressed_llb->start); + if (!ll_base) + goto err_llb; } dev_priv->cfb_size = size; intel_disable_fbc(dev); dev_priv->compressed_fb = compressed_fb; - if (IS_IRONLAKE_M(dev)) + if (HAS_PCH_SPLIT(dev)) I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start); else if (IS_GM45(dev)) { I915_WRITE(DPFC_CB_BASE, compressed_fb->start); @@ -1144,8 +1099,17 @@ static void i915_setup_compression(struct drm_device *dev, int size) dev_priv->compressed_llb = compressed_llb; } - DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", cfb_base, - ll_base, size >> 20); + DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n", + cfb_base, ll_base, size >> 20); + return; + +err_llb: + drm_mm_put_block(compressed_llb); +err_fb: + drm_mm_put_block(compressed_fb); +err: + dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; + i915_warn_stolen(dev); } static void i915_cleanup_compression(struct drm_device *dev) @@ -1176,12 +1140,16 @@ static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_ pm_message_t pmm = { .event = PM_EVENT_SUSPEND }; if (state == VGA_SWITCHEROO_ON) { printk(KERN_INFO "i915: switched on\n"); + dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; /* i915 resume handler doesn't set to D0 */ pci_set_power_state(dev->pdev, PCI_D0); i915_resume(dev); + dev->switch_power_state = DRM_SWITCH_POWER_ON; } else { printk(KERN_ERR "i915: switched off\n"); + dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; i915_suspend(dev, pmm); + dev->switch_power_state = DRM_SWITCH_POWER_OFF; } } @@ -1196,17 +1164,20 @@ static bool i915_switcheroo_can_switch(struct pci_dev *pdev) return can_switch; } -static int i915_load_modeset_init(struct drm_device *dev, - unsigned long prealloc_size, - unsigned long agp_size) +static int i915_load_modeset_init(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + unsigned long prealloc_size, gtt_size, mappable_size; int ret = 0; - /* Basic memrange allocator for stolen space (aka mm.vram) */ - drm_mm_init(&dev_priv->mm.vram, 0, prealloc_size); + prealloc_size = dev_priv->mm.gtt->stolen_size; + gtt_size = dev_priv->mm.gtt->gtt_total_entries << PAGE_SHIFT; + mappable_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; + + /* Basic memrange allocator for stolen space */ + drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size); - /* Let GEM Manage from end of prealloc space to end of aperture. + /* Let GEM Manage all of the aperture. * * However, leave one page at the end still bound to the scratch page. * There are a number of places where the hardware apparently @@ -1215,7 +1186,7 @@ static int i915_load_modeset_init(struct drm_device *dev, * at the last page of the aperture. One page should be enough to * keep any prefetching inside of the aperture. */ - i915_gem_do_init(dev, prealloc_size, agp_size - 4096); + i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE); mutex_lock(&dev->struct_mutex); ret = i915_gem_init_ringbuffer(dev); @@ -1227,31 +1198,39 @@ static int i915_load_modeset_init(struct drm_device *dev, if (I915_HAS_FBC(dev) && i915_powersave) { int cfb_size; - /* Try to get an 8M buffer... */ - if (prealloc_size > (9*1024*1024)) - cfb_size = 8*1024*1024; + /* Leave 1M for line length buffer & misc. */ + + /* Try to get a 32M buffer... */ + if (prealloc_size > (36*1024*1024)) + cfb_size = 32*1024*1024; else /* fall back to 7/8 of the stolen space */ cfb_size = prealloc_size * 7 / 8; i915_setup_compression(dev, cfb_size); } - /* Allow hardware batchbuffers unless told otherwise. - */ + /* Allow hardware batchbuffers unless told otherwise. */ dev_priv->allow_batchbuffer = 1; ret = intel_parse_bios(dev); if (ret) DRM_INFO("failed to find VBIOS tables\n"); - /* if we have > 1 VGA cards, then disable the radeon VGA resources */ + /* If we have > 1 VGA cards, then we need to arbitrate access + * to the common VGA resources. + * + * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA), + * then we do not take part in VGA arbitration and the + * vga_client_register() fails with -ENODEV. + */ ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode); - if (ret) + if (ret && ret != -ENODEV) goto cleanup_ringbuffer; intel_register_dsm_handler(); ret = vga_switcheroo_register_client(dev->pdev, i915_switcheroo_set_state, + NULL, i915_switcheroo_can_switch); if (ret) goto cleanup_vga_client; @@ -1426,152 +1405,12 @@ static void i915_ironlake_get_mem_freq(struct drm_device *dev) } } -struct v_table { - u8 vid; - unsigned long vd; /* in .1 mil */ - unsigned long vm; /* in .1 mil */ - u8 pvid; -}; - -static struct v_table v_table[] = { - { 0, 16125, 15000, 0x7f, }, - { 1, 16000, 14875, 0x7e, }, - { 2, 15875, 14750, 0x7d, }, - { 3, 15750, 14625, 0x7c, }, - { 4, 15625, 14500, 0x7b, }, - { 5, 15500, 14375, 0x7a, }, - { 6, 15375, 14250, 0x79, }, - { 7, 15250, 14125, 0x78, }, - { 8, 15125, 14000, 0x77, }, - { 9, 15000, 13875, 0x76, }, - { 10, 14875, 13750, 0x75, }, - { 11, 14750, 13625, 0x74, }, - { 12, 14625, 13500, 0x73, }, - { 13, 14500, 13375, 0x72, }, - { 14, 14375, 13250, 0x71, }, - { 15, 14250, 13125, 0x70, }, - { 16, 14125, 13000, 0x6f, }, - { 17, 14000, 12875, 0x6e, }, - { 18, 13875, 12750, 0x6d, }, - { 19, 13750, 12625, 0x6c, }, - { 20, 13625, 12500, 0x6b, }, - { 21, 13500, 12375, 0x6a, }, - { 22, 13375, 12250, 0x69, }, - { 23, 13250, 12125, 0x68, }, - { 24, 13125, 12000, 0x67, }, - { 25, 13000, 11875, 0x66, }, - { 26, 12875, 11750, 0x65, }, - { 27, 12750, 11625, 0x64, }, - { 28, 12625, 11500, 0x63, }, - { 29, 12500, 11375, 0x62, }, - { 30, 12375, 11250, 0x61, }, - { 31, 12250, 11125, 0x60, }, - { 32, 12125, 11000, 0x5f, }, - { 33, 12000, 10875, 0x5e, }, - { 34, 11875, 10750, 0x5d, }, - { 35, 11750, 10625, 0x5c, }, - { 36, 11625, 10500, 0x5b, }, - { 37, 11500, 10375, 0x5a, }, - { 38, 11375, 10250, 0x59, }, - { 39, 11250, 10125, 0x58, }, - { 40, 11125, 10000, 0x57, }, - { 41, 11000, 9875, 0x56, }, - { 42, 10875, 9750, 0x55, }, - { 43, 10750, 9625, 0x54, }, - { 44, 10625, 9500, 0x53, }, - { 45, 10500, 9375, 0x52, }, - { 46, 10375, 9250, 0x51, }, - { 47, 10250, 9125, 0x50, }, - { 48, 10125, 9000, 0x4f, }, - { 49, 10000, 8875, 0x4e, }, - { 50, 9875, 8750, 0x4d, }, - { 51, 9750, 8625, 0x4c, }, - { 52, 9625, 8500, 0x4b, }, - { 53, 9500, 8375, 0x4a, }, - { 54, 9375, 8250, 0x49, }, - { 55, 9250, 8125, 0x48, }, - { 56, 9125, 8000, 0x47, }, - { 57, 9000, 7875, 0x46, }, - { 58, 8875, 7750, 0x45, }, - { 59, 8750, 7625, 0x44, }, - { 60, 8625, 7500, 0x43, }, - { 61, 8500, 7375, 0x42, }, - { 62, 8375, 7250, 0x41, }, - { 63, 8250, 7125, 0x40, }, - { 64, 8125, 7000, 0x3f, }, - { 65, 8000, 6875, 0x3e, }, - { 66, 7875, 6750, 0x3d, }, - { 67, 7750, 6625, 0x3c, }, - { 68, 7625, 6500, 0x3b, }, - { 69, 7500, 6375, 0x3a, }, - { 70, 7375, 6250, 0x39, }, - { 71, 7250, 6125, 0x38, }, - { 72, 7125, 6000, 0x37, }, - { 73, 7000, 5875, 0x36, }, - { 74, 6875, 5750, 0x35, }, - { 75, 6750, 5625, 0x34, }, - { 76, 6625, 5500, 0x33, }, - { 77, 6500, 5375, 0x32, }, - { 78, 6375, 5250, 0x31, }, - { 79, 6250, 5125, 0x30, }, - { 80, 6125, 5000, 0x2f, }, - { 81, 6000, 4875, 0x2e, }, - { 82, 5875, 4750, 0x2d, }, - { 83, 5750, 4625, 0x2c, }, - { 84, 5625, 4500, 0x2b, }, - { 85, 5500, 4375, 0x2a, }, - { 86, 5375, 4250, 0x29, }, - { 87, 5250, 4125, 0x28, }, - { 88, 5125, 4000, 0x27, }, - { 89, 5000, 3875, 0x26, }, - { 90, 4875, 3750, 0x25, }, - { 91, 4750, 3625, 0x24, }, - { 92, 4625, 3500, 0x23, }, - { 93, 4500, 3375, 0x22, }, - { 94, 4375, 3250, 0x21, }, - { 95, 4250, 3125, 0x20, }, - { 96, 4125, 3000, 0x1f, }, - { 97, 4125, 3000, 0x1e, }, - { 98, 4125, 3000, 0x1d, }, - { 99, 4125, 3000, 0x1c, }, - { 100, 4125, 3000, 0x1b, }, - { 101, 4125, 3000, 0x1a, }, - { 102, 4125, 3000, 0x19, }, - { 103, 4125, 3000, 0x18, }, - { 104, 4125, 3000, 0x17, }, - { 105, 4125, 3000, 0x16, }, - { 106, 4125, 3000, 0x15, }, - { 107, 4125, 3000, 0x14, }, - { 108, 4125, 3000, 0x13, }, - { 109, 4125, 3000, 0x12, }, - { 110, 4125, 3000, 0x11, }, - { 111, 4125, 3000, 0x10, }, - { 112, 4125, 3000, 0x0f, }, - { 113, 4125, 3000, 0x0e, }, - { 114, 4125, 3000, 0x0d, }, - { 115, 4125, 3000, 0x0c, }, - { 116, 4125, 3000, 0x0b, }, - { 117, 4125, 3000, 0x0a, }, - { 118, 4125, 3000, 0x09, }, - { 119, 4125, 3000, 0x08, }, - { 120, 1125, 0, 0x07, }, - { 121, 1000, 0, 0x06, }, - { 122, 875, 0, 0x05, }, - { 123, 750, 0, 0x04, }, - { 124, 625, 0, 0x03, }, - { 125, 500, 0, 0x02, }, - { 126, 375, 0, 0x01, }, - { 127, 0, 0, 0x00, }, -}; - -struct cparams { - int i; - int t; - int m; - int c; -}; - -static struct cparams cparams[] = { +static const struct cparams { + u16 i; + u16 t; + u16 m; + u16 c; +} cparams[] = { { 1, 1333, 301, 28664 }, { 1, 1066, 294, 24460 }, { 1, 800, 294, 25192 }, @@ -1637,21 +1476,145 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv) return ((m * x) / 127) - b; } -static unsigned long pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) +static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid) { - unsigned long val = 0; - int i; - - for (i = 0; i < ARRAY_SIZE(v_table); i++) { - if (v_table[i].pvid == pxvid) { - if (IS_MOBILE(dev_priv->dev)) - val = v_table[i].vm; - else - val = v_table[i].vd; - } - } - - return val; + static const struct v_table { + u16 vd; /* in .1 mil */ + u16 vm; /* in .1 mil */ + } v_table[] = { + { 0, 0, }, + { 375, 0, }, + { 500, 0, }, + { 625, 0, }, + { 750, 0, }, + { 875, 0, }, + { 1000, 0, }, + { 1125, 0, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4125, 3000, }, + { 4250, 3125, }, + { 4375, 3250, }, + { 4500, 3375, }, + { 4625, 3500, }, + { 4750, 3625, }, + { 4875, 3750, }, + { 5000, 3875, }, + { 5125, 4000, }, + { 5250, 4125, }, + { 5375, 4250, }, + { 5500, 4375, }, + { 5625, 4500, }, + { 5750, 4625, }, + { 5875, 4750, }, + { 6000, 4875, }, + { 6125, 5000, }, + { 6250, 5125, }, + { 6375, 5250, }, + { 6500, 5375, }, + { 6625, 5500, }, + { 6750, 5625, }, + { 6875, 5750, }, + { 7000, 5875, }, + { 7125, 6000, }, + { 7250, 6125, }, + { 7375, 6250, }, + { 7500, 6375, }, + { 7625, 6500, }, + { 7750, 6625, }, + { 7875, 6750, }, + { 8000, 6875, }, + { 8125, 7000, }, + { 8250, 7125, }, + { 8375, 7250, }, + { 8500, 7375, }, + { 8625, 7500, }, + { 8750, 7625, }, + { 8875, 7750, }, + { 9000, 7875, }, + { 9125, 8000, }, + { 9250, 8125, }, + { 9375, 8250, }, + { 9500, 8375, }, + { 9625, 8500, }, + { 9750, 8625, }, + { 9875, 8750, }, + { 10000, 8875, }, + { 10125, 9000, }, + { 10250, 9125, }, + { 10375, 9250, }, + { 10500, 9375, }, + { 10625, 9500, }, + { 10750, 9625, }, + { 10875, 9750, }, + { 11000, 9875, }, + { 11125, 10000, }, + { 11250, 10125, }, + { 11375, 10250, }, + { 11500, 10375, }, + { 11625, 10500, }, + { 11750, 10625, }, + { 11875, 10750, }, + { 12000, 10875, }, + { 12125, 11000, }, + { 12250, 11125, }, + { 12375, 11250, }, + { 12500, 11375, }, + { 12625, 11500, }, + { 12750, 11625, }, + { 12875, 11750, }, + { 13000, 11875, }, + { 13125, 12000, }, + { 13250, 12125, }, + { 13375, 12250, }, + { 13500, 12375, }, + { 13625, 12500, }, + { 13750, 12625, }, + { 13875, 12750, }, + { 14000, 12875, }, + { 14125, 13000, }, + { 14250, 13125, }, + { 14375, 13250, }, + { 14500, 13375, }, + { 14625, 13500, }, + { 14750, 13625, }, + { 14875, 13750, }, + { 15000, 13875, }, + { 15125, 14000, }, + { 15250, 14125, }, + { 15375, 14250, }, + { 15500, 14375, }, + { 15625, 14500, }, + { 15750, 14625, }, + { 15875, 14750, }, + { 16000, 14875, }, + { 16125, 15000, }, + }; + if (dev_priv->info->is_mobile) + return v_table[pxvid].vm; + else + return v_table[pxvid].vd; } void i915_update_gfx_val(struct drm_i915_private *dev_priv) @@ -1905,9 +1868,9 @@ ips_ping_for_i915_load(void) int i915_driver_load(struct drm_device *dev, unsigned long flags) { struct drm_i915_private *dev_priv; - resource_size_t base, size; int ret = 0, mmio_bar; - uint32_t agp_size, prealloc_size; + uint32_t agp_size; + /* i915 has 4 more counters */ dev->counters += 4; dev->types[6] = _DRM_STAT_IRQ; @@ -1923,11 +1886,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) dev_priv->dev = dev; dev_priv->info = (struct intel_device_info *) flags; - /* Add register map (needed for suspend/resume) */ - mmio_bar = IS_GEN2(dev) ? 1 : 0; - base = pci_resource_start(dev->pdev, mmio_bar); - size = pci_resource_len(dev->pdev, mmio_bar); - if (i915_get_bridge_dev(dev)) { ret = -EIO; goto free_priv; @@ -1937,16 +1895,36 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) if (IS_GEN2(dev)) dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30)); - dev_priv->regs = ioremap(base, size); + /* 965GM sometimes incorrectly writes to hardware status page (HWS) + * using 32bit addressing, overwriting memory if HWS is located + * above 4GB. + * + * The documentation also mentions an issue with undefined + * behaviour if any general state is accessed within a page above 4GB, + * which also needs to be handled carefully. + */ + if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) + dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32)); + + mmio_bar = IS_GEN2(dev) ? 1 : 0; + dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, 0); if (!dev_priv->regs) { DRM_ERROR("failed to map registers\n"); ret = -EIO; goto put_bridge; } + dev_priv->mm.gtt = intel_gtt_get(); + if (!dev_priv->mm.gtt) { + DRM_ERROR("Failed to initialize GTT\n"); + ret = -ENODEV; + goto out_iomapfree; + } + + agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; + dev_priv->mm.gtt_mapping = - io_mapping_create_wc(dev->agp->base, - dev->agp->agp_info.aper_size * 1024*1024); + io_mapping_create_wc(dev->agp->base, agp_size); if (dev_priv->mm.gtt_mapping == NULL) { ret = -EIO; goto out_rmmap; @@ -1958,24 +1936,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) * MTRR if present. Even if a UC MTRR isn't present. */ dev_priv->mm.gtt_mtrr = mtrr_add(dev->agp->base, - dev->agp->agp_info.aper_size * - 1024 * 1024, + agp_size, MTRR_TYPE_WRCOMB, 1); if (dev_priv->mm.gtt_mtrr < 0) { DRM_INFO("MTRR allocation failed. Graphics " "performance may suffer.\n"); } - dev_priv->mm.gtt = intel_gtt_get(); - if (!dev_priv->mm.gtt) { - DRM_ERROR("Failed to initialize GTT\n"); - ret = -ENODEV; - goto out_iomapfree; - } - - prealloc_size = dev_priv->mm.gtt->gtt_stolen_entries << PAGE_SHIFT; - agp_size = dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; - /* The i915 workqueue is primarily used for batched retirement of * requests (and thus managing bo) once the task has been completed * by the GPU. i915_gem_retire_requests() is called directly when we @@ -1983,7 +1950,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) * bo. * * It is also used for periodic low-priority events, such as - * idle-timers and hangcheck. + * idle-timers and recording error state. * * All tasks on the workqueue are expected to acquire the dev mutex * so there is no point in running more than one instance of the @@ -2001,22 +1968,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) /* enable GEM by default */ dev_priv->has_gem = 1; - if (prealloc_size > agp_size * 3 / 4) { - DRM_ERROR("Detected broken video BIOS with %d/%dkB of video " - "memory stolen.\n", - prealloc_size / 1024, agp_size / 1024); - DRM_ERROR("Disabling GEM. (try reducing stolen memory or " - "updating the BIOS to fix).\n"); - dev_priv->has_gem = 0; - } - - if (dev_priv->has_gem == 0 && - drm_core_check_feature(dev, DRIVER_MODESET)) { - DRM_ERROR("kernel modesetting requires GEM, disabling driver.\n"); - ret = -ENODEV; - goto out_iomapfree; - } - dev->driver->get_vblank_counter = i915_get_vblank_counter; dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */ if (IS_G4X(dev) || IS_GEN5(dev) || IS_GEN6(dev)) { @@ -2037,8 +1988,8 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) /* Init HWS */ if (!I915_NEED_GFX_HWS(dev)) { ret = i915_init_phys_hws(dev); - if (ret != 0) - goto out_workqueue_free; + if (ret) + goto out_gem_unload; } if (IS_PINEVIEW(dev)) @@ -2060,16 +2011,13 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) if (!IS_I945G(dev) && !IS_I945GM(dev)) pci_enable_msi(dev->pdev); - spin_lock_init(&dev_priv->user_irq_lock); + spin_lock_init(&dev_priv->irq_lock); spin_lock_init(&dev_priv->error_lock); dev_priv->trace_irq_seqno = 0; ret = drm_vblank_init(dev, I915_NUM_PIPE); - - if (ret) { - (void) i915_driver_unload(dev); - return ret; - } + if (ret) + goto out_gem_unload; /* Start out suspended */ dev_priv->mm.suspended = 1; @@ -2077,10 +2025,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) intel_detect_pch(dev); if (drm_core_check_feature(dev, DRIVER_MODESET)) { - ret = i915_load_modeset_init(dev, prealloc_size, agp_size); + ret = i915_load_modeset_init(dev); if (ret < 0) { DRM_ERROR("failed to init modeset\n"); - goto out_workqueue_free; + goto out_gem_unload; } } @@ -2100,12 +2048,17 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags) return 0; -out_workqueue_free: +out_gem_unload: + if (dev->pdev->msi_enabled) + pci_disable_msi(dev->pdev); + + intel_teardown_gmbus(dev); + intel_teardown_mchbar(dev); destroy_workqueue(dev_priv->wq); out_iomapfree: io_mapping_free(dev_priv->mm.gtt_mapping); out_rmmap: - iounmap(dev_priv->regs); + pci_iounmap(dev->pdev, dev_priv->regs); put_bridge: pci_dev_put(dev_priv->bridge_dev); free_priv: @@ -2122,6 +2075,9 @@ int i915_driver_unload(struct drm_device *dev) i915_mch_dev = NULL; spin_unlock(&mchdev_lock); + if (dev_priv->mm.inactive_shrinker.shrink) + unregister_shrinker(&dev_priv->mm.inactive_shrinker); + mutex_lock(&dev->struct_mutex); ret = i915_gpu_idle(dev); if (ret) @@ -2179,7 +2135,7 @@ int i915_driver_unload(struct drm_device *dev) mutex_unlock(&dev->struct_mutex); if (I915_HAS_FBC(dev) && i915_powersave) i915_cleanup_compression(dev); - drm_mm_takedown(&dev_priv->mm.vram); + drm_mm_takedown(&dev_priv->mm.stolen); intel_cleanup_overlay(dev); @@ -2188,7 +2144,7 @@ int i915_driver_unload(struct drm_device *dev) } if (dev_priv->regs != NULL) - iounmap(dev_priv->regs); + pci_iounmap(dev->pdev, dev_priv->regs); intel_teardown_gmbus(dev); intel_teardown_mchbar(dev); diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index f737960712e6..0ad533f06af9 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -46,15 +46,24 @@ module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400); unsigned int i915_powersave = 1; module_param_named(powersave, i915_powersave, int, 0600); +unsigned int i915_enable_rc6 = 0; +module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0600); + unsigned int i915_lvds_downclock = 0; module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400); +unsigned int i915_panel_use_ssc = 1; +module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600); + +bool i915_try_reset = true; +module_param_named(reset, i915_try_reset, bool, 0600); + static struct drm_driver driver; extern int intel_agp_enabled; #define INTEL_VGA_DEVICE(id, info) { \ .class = PCI_CLASS_DISPLAY_VGA << 8, \ - .class_mask = 0xffff00, \ + .class_mask = 0xff0000, \ .vendor = 0x8086, \ .device = id, \ .subvendor = PCI_ANY_ID, \ @@ -111,7 +120,7 @@ static const struct intel_device_info intel_i965g_info = { static const struct intel_device_info intel_i965gm_info = { .gen = 4, .is_crestline = 1, - .is_mobile = 1, .has_fbc = 1, .has_rc6 = 1, .has_hotplug = 1, + .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1, .has_overlay = 1, .supports_tv = 1, }; @@ -130,7 +139,7 @@ static const struct intel_device_info intel_g45_info = { static const struct intel_device_info intel_gm45_info = { .gen = 4, .is_g4x = 1, - .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_rc6 = 1, + .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1, .has_pipe_cxsr = 1, .has_hotplug = 1, .supports_tv = 1, .has_bsd_ring = 1, @@ -150,7 +159,7 @@ static const struct intel_device_info intel_ironlake_d_info = { static const struct intel_device_info intel_ironlake_m_info = { .gen = 5, .is_mobile = 1, - .need_gfx_hws = 1, .has_rc6 = 1, .has_hotplug = 1, + .need_gfx_hws = 1, .has_hotplug = 1, .has_fbc = 0, /* disabled due to buggy hardware */ .has_bsd_ring = 1, }; @@ -165,6 +174,7 @@ static const struct intel_device_info intel_sandybridge_d_info = { static const struct intel_device_info intel_sandybridge_m_info = { .gen = 6, .is_mobile = 1, .need_gfx_hws = 1, .has_hotplug = 1, + .has_fbc = 1, .has_bsd_ring = 1, .has_blt_ring = 1, }; @@ -244,10 +254,34 @@ void intel_detect_pch (struct drm_device *dev) } } +void __gen6_force_wake_get(struct drm_i915_private *dev_priv) +{ + int count; + + count = 0; + while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1)) + udelay(10); + + I915_WRITE_NOTRACE(FORCEWAKE, 1); + POSTING_READ(FORCEWAKE); + + count = 0; + while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK) & 1) == 0) + udelay(10); +} + +void __gen6_force_wake_put(struct drm_i915_private *dev_priv) +{ + I915_WRITE_NOTRACE(FORCEWAKE, 0); + POSTING_READ(FORCEWAKE); +} + static int i915_drm_freeze(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + drm_kms_helper_poll_disable(dev); + pci_save_state(dev->pdev); /* If KMS is active, we do the leavevt stuff here */ @@ -284,7 +318,9 @@ int i915_suspend(struct drm_device *dev, pm_message_t state) if (state.event == PM_EVENT_PRETHAW) return 0; - drm_kms_helper_poll_disable(dev); + + if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; error = i915_drm_freeze(dev); if (error) @@ -304,6 +340,12 @@ static int i915_drm_thaw(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int error = 0; + if (drm_core_check_feature(dev, DRIVER_MODESET)) { + mutex_lock(&dev->struct_mutex); + i915_gem_restore_gtt_mappings(dev); + mutex_unlock(&dev->struct_mutex); + } + i915_restore_state(dev); intel_opregion_setup(dev); @@ -315,10 +357,14 @@ static int i915_drm_thaw(struct drm_device *dev) error = i915_gem_init_ringbuffer(dev); mutex_unlock(&dev->struct_mutex); + drm_mode_config_reset(dev); drm_irq_install(dev); /* Resume the modeset for every activated CRTC */ drm_helper_resume_force_mode(dev); + + if (IS_IRONLAKE_M(dev)) + ironlake_enable_rc6(dev); } intel_opregion_init(dev); @@ -332,6 +378,9 @@ int i915_resume(struct drm_device *dev) { int ret; + if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; + if (pci_enable_device(dev->pdev)) return -EIO; @@ -405,6 +454,14 @@ static int ironlake_do_reset(struct drm_device *dev, u8 flags) return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); } +static int gen6_do_reset(struct drm_device *dev, u8 flags) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(GEN6_GDRST, GEN6_GRDOM_FULL); + return wait_for((I915_READ(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500); +} + /** * i965_reset - reset chip after a hang * @dev: drm device to reset @@ -431,7 +488,11 @@ int i915_reset(struct drm_device *dev, u8 flags) bool need_display = true; int ret; - mutex_lock(&dev->struct_mutex); + if (!i915_try_reset) + return 0; + + if (!mutex_trylock(&dev->struct_mutex)) + return -EBUSY; i915_gem_reset(dev); @@ -439,6 +500,9 @@ int i915_reset(struct drm_device *dev, u8 flags) if (get_seconds() - dev_priv->last_gpu_reset < 5) { DRM_ERROR("GPU hanging too fast, declaring wedged!\n"); } else switch (INTEL_INFO(dev)->gen) { + case 6: + ret = gen6_do_reset(dev, flags); + break; case 5: ret = ironlake_do_reset(dev, flags); break; @@ -472,11 +536,17 @@ int i915_reset(struct drm_device *dev, u8 flags) */ if (drm_core_check_feature(dev, DRIVER_MODESET) || !dev_priv->mm.suspended) { - struct intel_ring_buffer *ring = &dev_priv->render_ring; dev_priv->mm.suspended = 0; - ring->init(dev, ring); + + dev_priv->ring[RCS].init(&dev_priv->ring[RCS]); + if (HAS_BSD(dev)) + dev_priv->ring[VCS].init(&dev_priv->ring[VCS]); + if (HAS_BLT(dev)) + dev_priv->ring[BCS].init(&dev_priv->ring[BCS]); + mutex_unlock(&dev->struct_mutex); drm_irq_uninstall(dev); + drm_mode_config_reset(dev); drm_irq_install(dev); mutex_lock(&dev->struct_mutex); } @@ -501,6 +571,14 @@ int i915_reset(struct drm_device *dev, u8 flags) static int __devinit i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { + /* Only bind to function 0 of the device. Early generations + * used function 1 as a placeholder for multi-head. This causes + * us confusion instead, especially on the systems where both + * functions have the same PCI-ID! + */ + if (PCI_FUNC(pdev->devfn)) + return -ENODEV; + return drm_get_pci_dev(pdev, ent, &driver); } @@ -523,6 +601,9 @@ static int i915_pm_suspend(struct device *dev) return -ENODEV; } + if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF) + return 0; + error = i915_drm_freeze(drm_dev); if (error) return error; @@ -606,6 +687,8 @@ static struct drm_driver driver = { .device_is_agp = i915_driver_device_is_agp, .enable_vblank = i915_enable_vblank, .disable_vblank = i915_disable_vblank, + .get_vblank_timestamp = i915_get_vblank_timestamp, + .get_scanout_position = i915_get_crtc_scanoutpos, .irq_preinstall = i915_driver_irq_preinstall, .irq_postinstall = i915_driver_irq_postinstall, .irq_uninstall = i915_driver_irq_uninstall, @@ -661,8 +744,6 @@ static int __init i915_init(void) driver.num_ioctls = i915_max_ioctl; - i915_gem_shrinker_init(); - /* * If CONFIG_DRM_I915_KMS is set, default to KMS unless * explicitly disabled with the module pararmeter. @@ -684,17 +765,14 @@ static int __init i915_init(void) driver.driver_features &= ~DRIVER_MODESET; #endif - if (!(driver.driver_features & DRIVER_MODESET)) { - driver.suspend = i915_suspend; - driver.resume = i915_resume; - } + if (!(driver.driver_features & DRIVER_MODESET)) + driver.get_vblank_timestamp = NULL; return drm_init(&driver); } static void __exit i915_exit(void) { - i915_gem_shrinker_exit(); drm_exit(&driver); } diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 409826da3099..65dfe81d0035 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -89,7 +89,7 @@ struct drm_i915_gem_phys_object { int id; struct page **page_list; drm_dma_handle_t *handle; - struct drm_gem_object *cur_obj; + struct drm_i915_gem_object *cur_obj; }; struct mem_block { @@ -124,9 +124,9 @@ struct drm_i915_master_private { #define I915_FENCE_REG_NONE -1 struct drm_i915_fence_reg { - struct drm_gem_object *obj; struct list_head lru_list; - bool gpu; + struct drm_i915_gem_object *obj; + uint32_t setup_seqno; }; struct sdvo_device_mapping { @@ -139,6 +139,8 @@ struct sdvo_device_mapping { u8 ddc_pin; }; +struct intel_display_error_state; + struct drm_i915_error_state { u32 eir; u32 pgtbl_er; @@ -148,32 +150,47 @@ struct drm_i915_error_state { u32 ipehr; u32 instdone; u32 acthd; + u32 error; /* gen6+ */ + u32 bcs_acthd; /* gen6+ blt engine */ + u32 bcs_ipehr; + u32 bcs_ipeir; + u32 bcs_instdone; + u32 bcs_seqno; + u32 vcs_acthd; /* gen6+ bsd engine */ + u32 vcs_ipehr; + u32 vcs_ipeir; + u32 vcs_instdone; + u32 vcs_seqno; u32 instpm; u32 instps; u32 instdone1; u32 seqno; u64 bbaddr; + u64 fence[16]; struct timeval time; struct drm_i915_error_object { int page_count; u32 gtt_offset; u32 *pages[0]; - } *ringbuffer, *batchbuffer[2]; + } *ringbuffer, *batchbuffer[I915_NUM_RINGS]; struct drm_i915_error_buffer { - size_t size; + u32 size; u32 name; u32 seqno; u32 gtt_offset; u32 read_domains; u32 write_domain; - u32 fence_reg; + s32 fence_reg:5; s32 pinned:2; u32 tiling:2; u32 dirty:1; u32 purgeable:1; - } *active_bo; - u32 active_bo_count; + u32 ring:4; + u32 agp_type:1; + } *active_bo, *pinned_bo; + u32 active_bo_count, pinned_bo_count; struct intel_overlay_error_state *overlay; + struct intel_display_error_state *display; }; struct drm_i915_display_funcs { @@ -207,7 +224,6 @@ struct intel_device_info { u8 is_broadwater : 1; u8 is_crestline : 1; u8 has_fbc : 1; - u8 has_rc6 : 1; u8 has_pipe_cxsr : 1; u8 has_hotplug : 1; u8 cursor_needs_physical : 1; @@ -243,6 +259,7 @@ typedef struct drm_i915_private { const struct intel_device_info *info; int has_gem; + int relative_constants_mode; void __iomem *regs; @@ -253,20 +270,15 @@ typedef struct drm_i915_private { } *gmbus; struct pci_dev *bridge_dev; - struct intel_ring_buffer render_ring; - struct intel_ring_buffer bsd_ring; - struct intel_ring_buffer blt_ring; + struct intel_ring_buffer ring[I915_NUM_RINGS]; uint32_t next_seqno; drm_dma_handle_t *status_page_dmah; - void *seqno_page; dma_addr_t dma_status_page; uint32_t counter; - unsigned int seqno_gfx_addr; drm_local_map_t hws_map; - struct drm_gem_object *seqno_obj; - struct drm_gem_object *pwrctx; - struct drm_gem_object *renderctx; + struct drm_i915_gem_object *pwrctx; + struct drm_i915_gem_object *renderctx; struct resource mch_res; @@ -275,25 +287,17 @@ typedef struct drm_i915_private { int front_offset; int current_page; int page_flipping; -#define I915_DEBUG_READ (1<<0) -#define I915_DEBUG_WRITE (1<<1) - unsigned long debug_flags; - wait_queue_head_t irq_queue; atomic_t irq_received; - /** Protects user_irq_refcount and irq_mask_reg */ - spinlock_t user_irq_lock; u32 trace_irq_seqno; + + /* protects the irq masks */ + spinlock_t irq_lock; /** Cached value of IMR to avoid reads in updating the bitfield */ - u32 irq_mask_reg; u32 pipestat[2]; - /** splitted irq regs for graphics and display engine on Ironlake, - irq_mask_reg is still used for display irq. */ - u32 gt_irq_mask_reg; - u32 gt_irq_enable_reg; - u32 de_irq_enable_reg; - u32 pch_irq_mask_reg; - u32 pch_irq_enable_reg; + u32 irq_mask; + u32 gt_irq_mask; + u32 pch_irq_mask; u32 hotplug_supported_mask; struct work_struct hotplug_work; @@ -306,7 +310,7 @@ typedef struct drm_i915_private { int num_pipe; /* For hangcheck timer */ -#define DRM_I915_HANGCHECK_PERIOD 250 /* in ms */ +#define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ struct timer_list hangcheck_timer; int hangcheck_count; uint32_t last_acthd; @@ -329,6 +333,7 @@ typedef struct drm_i915_private { /* LVDS info */ int backlight_level; /* restore backlight to this value */ + bool backlight_enabled; struct drm_display_mode *panel_fixed_mode; struct drm_display_mode *lfp_lvds_vbt_mode; /* if any */ struct drm_display_mode *sdvo_lvds_vbt_mode; /* if any */ @@ -530,23 +535,24 @@ typedef struct drm_i915_private { struct { /** Bridge to intel-gtt-ko */ - struct intel_gtt *gtt; + const struct intel_gtt *gtt; /** Memory allocator for GTT stolen memory */ - struct drm_mm vram; + struct drm_mm stolen; /** Memory allocator for GTT */ struct drm_mm gtt_space; + /** List of all objects in gtt_space. Used to restore gtt + * mappings on resume */ + struct list_head gtt_list; + + /** Usable portion of the GTT for GEM */ + unsigned long gtt_start; + unsigned long gtt_mappable_end; + unsigned long gtt_end; struct io_mapping *gtt_mapping; int gtt_mtrr; - /** - * Membership on list of all loaded devices, used to evict - * inactive buffers under memory pressure. - * - * Modifications should only be done whilst holding the - * shrink_list_lock spinlock. - */ - struct list_head shrink_list; + struct shrinker inactive_shrinker; /** * List of objects currently involved in rendering. @@ -609,16 +615,6 @@ typedef struct drm_i915_private { struct delayed_work retire_work; /** - * Waiting sequence number, if any - */ - uint32_t waiting_gem_seqno; - - /** - * Last seq seen at irq time - */ - uint32_t irq_gem_seqno; - - /** * Flag if the X Server, and thus DRM, is not currently in * control of the device. * @@ -645,16 +641,11 @@ typedef struct drm_i915_private { /* storage for physical objects */ struct drm_i915_gem_phys_object *phys_objs[I915_MAX_PHYS_OBJECT]; - uint32_t flush_rings; - /* accounting, useful for userland debugging */ - size_t object_memory; - size_t pin_memory; - size_t gtt_memory; size_t gtt_total; + size_t mappable_gtt_total; + size_t object_memory; u32 object_count; - u32 pin_count; - u32 gtt_count; } mm; struct sdvo_device_mapping sdvo_mappings[2]; /* indicate whether the LVDS_BORDER should be enabled or not */ @@ -688,14 +679,14 @@ typedef struct drm_i915_private { u8 fmax; u8 fstart; - u64 last_count1; - unsigned long last_time1; - u64 last_count2; - struct timespec last_time2; - unsigned long gfx_power; - int c_m; - int r_t; - u8 corr; + u64 last_count1; + unsigned long last_time1; + u64 last_count2; + struct timespec last_time2; + unsigned long gfx_power; + int c_m; + int r_t; + u8 corr; spinlock_t *mchdev_lock; enum no_fbc_reason no_fbc_reason; @@ -709,20 +700,20 @@ typedef struct drm_i915_private { struct intel_fbdev *fbdev; } drm_i915_private_t; -/** driver private structure attached to each drm_gem_object */ struct drm_i915_gem_object { struct drm_gem_object base; /** Current space allocated to this object in the GTT, if any. */ struct drm_mm_node *gtt_space; + struct list_head gtt_list; /** This object's place on the active/flushing/inactive lists */ struct list_head ring_list; struct list_head mm_list; /** This object's place on GPU write list */ struct list_head gpu_write_list; - /** This object's place on eviction list */ - struct list_head evict_list; + /** This object's place in the batchbuffer or on the eviction list */ + struct list_head exec_list; /** * This is set if the object is on the active or flushing lists @@ -738,6 +729,12 @@ struct drm_i915_gem_object { unsigned int dirty : 1; /** + * This is set if the object has been written to since the last + * GPU flush. + */ + unsigned int pending_gpu_write : 1; + + /** * Fence register bits (if any) for this object. Will be set * as needed when mapped into the GTT. * Protected by dev->struct_mutex. @@ -747,29 +744,15 @@ struct drm_i915_gem_object { signed int fence_reg : 5; /** - * Used for checking the object doesn't appear more than once - * in an execbuffer object list. - */ - unsigned int in_execbuffer : 1; - - /** * Advice: are the backing pages purgeable? */ unsigned int madv : 2; /** - * Refcount for the pages array. With the current locking scheme, there - * are at most two concurrent users: Binding a bo to the gtt and - * pwrite/pread using physical addresses. So two bits for a maximum - * of two users are enough. - */ - unsigned int pages_refcount : 2; -#define DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT 0x3 - - /** * Current tiling mode for the object. */ unsigned int tiling_mode : 2; + unsigned int tiling_changed : 1; /** How many users have pinned this object in GTT space. The following * users can each hold at most one reference: pwrite/pread, pin_ioctl @@ -783,28 +766,55 @@ struct drm_i915_gem_object { unsigned int pin_count : 4; #define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf - /** AGP memory structure for our GTT binding. */ - DRM_AGP_MEM *agp_mem; + /** + * Is the object at the current location in the gtt mappable and + * fenceable? Used to avoid costly recalculations. + */ + unsigned int map_and_fenceable : 1; + + /** + * Whether the current gtt mapping needs to be mappable (and isn't just + * mappable by accident). Track pin and fault separate for a more + * accurate mappable working set. + */ + unsigned int fault_mappable : 1; + unsigned int pin_mappable : 1; + + /* + * Is the GPU currently using a fence to access this buffer, + */ + unsigned int pending_fenced_gpu_access:1; + unsigned int fenced_gpu_access:1; struct page **pages; /** - * Current offset of the object in GTT space. - * - * This is the same as gtt_space->start + * DMAR support */ - uint32_t gtt_offset; + struct scatterlist *sg_list; + int num_sg; - /* Which ring is refering to is this object */ - struct intel_ring_buffer *ring; + /** + * Used for performing relocations during execbuffer insertion. + */ + struct hlist_node exec_node; + unsigned long exec_handle; + struct drm_i915_gem_exec_object2 *exec_entry; /** - * Fake offset for use by mmap(2) + * Current offset of the object in GTT space. + * + * This is the same as gtt_space->start */ - uint64_t mmap_offset; + uint32_t gtt_offset; /** Breadcrumb of last rendering to the buffer. */ uint32_t last_rendering_seqno; + struct intel_ring_buffer *ring; + + /** Breadcrumb of last fenced GPU access to the buffer. */ + uint32_t last_fenced_seqno; + struct intel_ring_buffer *last_fenced_ring; /** Current tiling stride for the object, if it's tiled. */ uint32_t stride; @@ -880,11 +890,75 @@ enum intel_chip_family { CHIP_I965 = 0x08, }; +#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info) + +#define IS_I830(dev) ((dev)->pci_device == 0x3577) +#define IS_845G(dev) ((dev)->pci_device == 0x2562) +#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) +#define IS_I865G(dev) ((dev)->pci_device == 0x2572) +#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) +#define IS_I915GM(dev) ((dev)->pci_device == 0x2592) +#define IS_I945G(dev) ((dev)->pci_device == 0x2772) +#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) +#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) +#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) +#define IS_GM45(dev) ((dev)->pci_device == 0x2A42) +#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x) +#define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001) +#define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011) +#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) +#define IS_G33(dev) (INTEL_INFO(dev)->is_g33) +#define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042) +#define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046) +#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) + +#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2) +#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3) +#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4) +#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5) +#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6) + +#define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring) +#define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring) +#define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) + +#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) +#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) + +/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte + * rows, which changed the alignment requirements and fence programming. + */ +#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \ + IS_I915GM(dev))) +#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) +#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev)) +#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev)) +#define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev)) +#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) +#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) +/* dsparb controlled by hw only */ +#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev)) + +#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) +#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) +#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) + +#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev)) +#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev)) + +#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type) +#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT) +#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX) + +#include "i915_trace.h" + extern struct drm_ioctl_desc i915_ioctls[]; extern int i915_max_ioctl; extern unsigned int i915_fbpercrtc; extern unsigned int i915_powersave; extern unsigned int i915_lvds_downclock; +extern unsigned int i915_panel_use_ssc; +extern unsigned int i915_enable_rc6; extern int i915_suspend(struct drm_device *dev, pm_message_t state); extern int i915_resume(struct drm_device *dev); @@ -907,8 +981,8 @@ extern int i915_driver_device_is_agp(struct drm_device * dev); extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); extern int i915_emit_box(struct drm_device *dev, - struct drm_clip_rect *boxes, - int i, int DR1, int DR4); + struct drm_clip_rect *box, + int DR1, int DR4); extern int i915_reset(struct drm_device *dev, u8 flags); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); @@ -918,6 +992,7 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv); /* i915_irq.c */ void i915_hangcheck_elapsed(unsigned long data); +void i915_handle_error(struct drm_device *dev, bool wedged); extern int i915_irq_emit(struct drm_device *dev, void *data, struct drm_file *file_priv); extern int i915_irq_wait(struct drm_device *dev, void *data, @@ -939,12 +1014,6 @@ extern u32 i915_get_vblank_counter(struct drm_device *dev, int crtc); extern u32 gm45_get_vblank_counter(struct drm_device *dev, int crtc); extern int i915_vblank_swap(struct drm_device *dev, void *data, struct drm_file *file_priv); -extern void i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask); -extern void i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask); -extern void ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, - u32 mask); -extern void ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, - u32 mask); void i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); @@ -953,6 +1022,13 @@ void i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask); void intel_enable_asle (struct drm_device *dev); +int i915_get_vblank_timestamp(struct drm_device *dev, int crtc, + int *max_error, + struct timeval *vblank_time, + unsigned flags); + +int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, + int *vpos, int *hpos); #ifdef CONFIG_DEBUG_FS extern void i915_destroy_error_state(struct drm_device *dev); @@ -1017,15 +1093,28 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); void i915_gem_load(struct drm_device *dev); int i915_gem_init_object(struct drm_gem_object *obj); -struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, - size_t size); +int __must_check i915_gem_flush_ring(struct drm_device *dev, + struct intel_ring_buffer *ring, + uint32_t invalidate_domains, + uint32_t flush_domains); +struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, + size_t size); void i915_gem_free_object(struct drm_gem_object *obj); -int i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment); -void i915_gem_object_unpin(struct drm_gem_object *obj); -int i915_gem_object_unbind(struct drm_gem_object *obj); -void i915_gem_release_mmap(struct drm_gem_object *obj); +int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj, + uint32_t alignment, + bool map_and_fenceable); +void i915_gem_object_unpin(struct drm_i915_gem_object *obj); +int __must_check i915_gem_object_unbind(struct drm_i915_gem_object *obj); +void i915_gem_release_mmap(struct drm_i915_gem_object *obj); void i915_gem_lastclose(struct drm_device *dev); +int __must_check i915_mutex_lock_interruptible(struct drm_device *dev); +int __must_check i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, + bool interruptible); +void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *ring, + u32 seqno); + /** * Returns true if seq1 is later than seq2. */ @@ -1035,73 +1124,88 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2) return (int32_t)(seq1 - seq2) >= 0; } -int i915_gem_object_get_fence_reg(struct drm_gem_object *obj, - bool interruptible); -int i915_gem_object_put_fence_reg(struct drm_gem_object *obj, - bool interruptible); +static inline u32 +i915_gem_next_request_seqno(struct drm_device *dev, + struct intel_ring_buffer *ring) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + return ring->outstanding_lazy_request = dev_priv->next_seqno; +} + +int __must_check i915_gem_object_get_fence(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined, + bool interruptible); +int __must_check i915_gem_object_put_fence(struct drm_i915_gem_object *obj); + void i915_gem_retire_requests(struct drm_device *dev); void i915_gem_reset(struct drm_device *dev); -void i915_gem_clflush_object(struct drm_gem_object *obj); -int i915_gem_object_set_domain(struct drm_gem_object *obj, - uint32_t read_domains, - uint32_t write_domain); -int i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj, - bool interruptible); -int i915_gem_init_ringbuffer(struct drm_device *dev); +void i915_gem_clflush_object(struct drm_i915_gem_object *obj); +int __must_check i915_gem_object_set_domain(struct drm_i915_gem_object *obj, + uint32_t read_domains, + uint32_t write_domain); +int __must_check i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj, + bool interruptible); +int __must_check i915_gem_init_ringbuffer(struct drm_device *dev); void i915_gem_cleanup_ringbuffer(struct drm_device *dev); -int i915_gem_do_init(struct drm_device *dev, unsigned long start, - unsigned long end); -int i915_gpu_idle(struct drm_device *dev); -int i915_gem_idle(struct drm_device *dev); -uint32_t i915_add_request(struct drm_device *dev, - struct drm_file *file_priv, - struct drm_i915_gem_request *request, - struct intel_ring_buffer *ring); -int i915_do_wait_request(struct drm_device *dev, - uint32_t seqno, - bool interruptible, - struct intel_ring_buffer *ring); +void i915_gem_do_init(struct drm_device *dev, + unsigned long start, + unsigned long mappable_end, + unsigned long end); +int __must_check i915_gpu_idle(struct drm_device *dev); +int __must_check i915_gem_idle(struct drm_device *dev); +int __must_check i915_add_request(struct drm_device *dev, + struct drm_file *file_priv, + struct drm_i915_gem_request *request, + struct intel_ring_buffer *ring); +int __must_check i915_do_wait_request(struct drm_device *dev, + uint32_t seqno, + bool interruptible, + struct intel_ring_buffer *ring); int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); -int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, - int write); -int i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, - bool pipelined); +int __must_check +i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, + bool write); +int __must_check +i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined); int i915_gem_attach_phys_object(struct drm_device *dev, - struct drm_gem_object *obj, + struct drm_i915_gem_object *obj, int id, int align); void i915_gem_detach_phys_object(struct drm_device *dev, - struct drm_gem_object *obj); + struct drm_i915_gem_object *obj); void i915_gem_free_all_phys_object(struct drm_device *dev); -void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv); +void i915_gem_release(struct drm_device *dev, struct drm_file *file); -void i915_gem_shrinker_init(void); -void i915_gem_shrinker_exit(void); +/* i915_gem_gtt.c */ +void i915_gem_restore_gtt_mappings(struct drm_device *dev); +int __must_check i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj); +void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj); /* i915_gem_evict.c */ -int i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment); -int i915_gem_evict_everything(struct drm_device *dev); -int i915_gem_evict_inactive(struct drm_device *dev); +int __must_check i915_gem_evict_something(struct drm_device *dev, int min_size, + unsigned alignment, bool mappable); +int __must_check i915_gem_evict_everything(struct drm_device *dev, + bool purgeable_only); +int __must_check i915_gem_evict_inactive(struct drm_device *dev, + bool purgeable_only); /* i915_gem_tiling.c */ void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj); -bool i915_tiling_ok(struct drm_device *dev, int stride, int size, - int tiling_mode); -bool i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, - int tiling_mode); +void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); +void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); /* i915_gem_debug.c */ -void i915_gem_dump_object(struct drm_gem_object *obj, int len, +void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len, const char *where, uint32_t mark); #if WATCH_LISTS int i915_verify_lists(struct drm_device *dev); #else #define i915_verify_lists(dev) 0 #endif -void i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle); -void i915_gem_dump_object(struct drm_gem_object *obj, int len, +void i915_gem_object_check_coherency(struct drm_i915_gem_object *obj, + int handle); +void i915_gem_dump_object(struct drm_i915_gem_object *obj, int len, const char *where, uint32_t mark); /* i915_debugfs.c */ @@ -1163,6 +1267,8 @@ extern void intel_disable_fbc(struct drm_device *dev); extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval); extern bool intel_fbc_enabled(struct drm_device *dev); extern bool ironlake_set_drps(struct drm_device *dev, u8 val); +extern void ironlake_enable_rc6(struct drm_device *dev); +extern void gen6_set_rps(struct drm_device *dev, u8 val); extern void intel_detect_pch (struct drm_device *dev); extern int intel_trans_dp_port_sel (struct drm_crtc *crtc); @@ -1170,79 +1276,120 @@ extern int intel_trans_dp_port_sel (struct drm_crtc *crtc); #ifdef CONFIG_DEBUG_FS extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev); extern void intel_overlay_print_error_state(struct seq_file *m, struct intel_overlay_error_state *error); + +extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev); +extern void intel_display_print_error_state(struct seq_file *m, + struct drm_device *dev, + struct intel_display_error_state *error); #endif +#define LP_RING(d) (&((struct drm_i915_private *)(d))->ring[RCS]) + +#define BEGIN_LP_RING(n) \ + intel_ring_begin(LP_RING(dev_priv), (n)) + +#define OUT_RING(x) \ + intel_ring_emit(LP_RING(dev_priv), x) + +#define ADVANCE_LP_RING() \ + intel_ring_advance(LP_RING(dev_priv)) + /** * Lock test for when it's just for synchronization of ring access. * * In that case, we don't need to do it when GEM is initialized as nobody else * has access to the ring. */ -#define RING_LOCK_TEST_WITH_RETURN(dev, file_priv) do { \ - if (((drm_i915_private_t *)dev->dev_private)->render_ring.gem_object \ - == NULL) \ - LOCK_TEST_WITH_RETURN(dev, file_priv); \ +#define RING_LOCK_TEST_WITH_RETURN(dev, file) do { \ + if (LP_RING(dev->dev_private)->obj == NULL) \ + LOCK_TEST_WITH_RETURN(dev, file); \ } while (0) -static inline u32 i915_read(struct drm_i915_private *dev_priv, u32 reg) + +#define __i915_read(x, y) \ +static inline u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \ + u##x val = read##y(dev_priv->regs + reg); \ + trace_i915_reg_rw('R', reg, val, sizeof(val)); \ + return val; \ +} +__i915_read(8, b) +__i915_read(16, w) +__i915_read(32, l) +__i915_read(64, q) +#undef __i915_read + +#define __i915_write(x, y) \ +static inline void i915_write##x(struct drm_i915_private *dev_priv, u32 reg, u##x val) { \ + trace_i915_reg_rw('W', reg, val, sizeof(val)); \ + write##y(val, dev_priv->regs + reg); \ +} +__i915_write(8, b) +__i915_write(16, w) +__i915_write(32, l) +__i915_write(64, q) +#undef __i915_write + +#define I915_READ8(reg) i915_read8(dev_priv, (reg)) +#define I915_WRITE8(reg, val) i915_write8(dev_priv, (reg), (val)) + +#define I915_READ16(reg) i915_read16(dev_priv, (reg)) +#define I915_WRITE16(reg, val) i915_write16(dev_priv, (reg), (val)) +#define I915_READ16_NOTRACE(reg) readw(dev_priv->regs + (reg)) +#define I915_WRITE16_NOTRACE(reg, val) writew(val, dev_priv->regs + (reg)) + +#define I915_READ(reg) i915_read32(dev_priv, (reg)) +#define I915_WRITE(reg, val) i915_write32(dev_priv, (reg), (val)) +#define I915_READ_NOTRACE(reg) readl(dev_priv->regs + (reg)) +#define I915_WRITE_NOTRACE(reg, val) writel(val, dev_priv->regs + (reg)) + +#define I915_WRITE64(reg, val) i915_write64(dev_priv, (reg), (val)) +#define I915_READ64(reg) i915_read64(dev_priv, (reg)) + +#define POSTING_READ(reg) (void)I915_READ_NOTRACE(reg) +#define POSTING_READ16(reg) (void)I915_READ16_NOTRACE(reg) + + +/* On SNB platform, before reading ring registers forcewake bit + * must be set to prevent GT core from power down and stale values being + * returned. + */ +void __gen6_force_wake_get(struct drm_i915_private *dev_priv); +void __gen6_force_wake_put (struct drm_i915_private *dev_priv); +static inline u32 i915_safe_read(struct drm_i915_private *dev_priv, u32 reg) { u32 val; - val = readl(dev_priv->regs + reg); - if (dev_priv->debug_flags & I915_DEBUG_READ) - printk(KERN_ERR "read 0x%08x from 0x%08x\n", val, reg); + if (dev_priv->info->gen >= 6) { + __gen6_force_wake_get(dev_priv); + val = I915_READ(reg); + __gen6_force_wake_put(dev_priv); + } else + val = I915_READ(reg); + return val; } -static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg, - u32 val) +static inline void +i915_write(struct drm_i915_private *dev_priv, u32 reg, u64 val, int len) { - writel(val, dev_priv->regs + reg); - if (dev_priv->debug_flags & I915_DEBUG_WRITE) - printk(KERN_ERR "wrote 0x%08x to 0x%08x\n", val, reg); + /* Trace down the write operation before the real write */ + trace_i915_reg_rw('W', reg, val, len); + switch (len) { + case 8: + writeq(val, dev_priv->regs + reg); + break; + case 4: + writel(val, dev_priv->regs + reg); + break; + case 2: + writew(val, dev_priv->regs + reg); + break; + case 1: + writeb(val, dev_priv->regs + reg); + break; + } } -#define I915_READ(reg) i915_read(dev_priv, (reg)) -#define I915_WRITE(reg, val) i915_write(dev_priv, (reg), (val)) -#define I915_READ16(reg) readw(dev_priv->regs + (reg)) -#define I915_WRITE16(reg, val) writel(val, dev_priv->regs + (reg)) -#define I915_READ8(reg) readb(dev_priv->regs + (reg)) -#define I915_WRITE8(reg, val) writeb(val, dev_priv->regs + (reg)) -#define I915_WRITE64(reg, val) writeq(val, dev_priv->regs + (reg)) -#define I915_READ64(reg) readq(dev_priv->regs + (reg)) -#define POSTING_READ(reg) (void)I915_READ(reg) -#define POSTING_READ16(reg) (void)I915_READ16(reg) - -#define I915_DEBUG_ENABLE_IO() (dev_priv->debug_flags |= I915_DEBUG_READ | \ - I915_DEBUG_WRITE) -#define I915_DEBUG_DISABLE_IO() (dev_priv->debug_flags &= ~(I915_DEBUG_READ | \ - I915_DEBUG_WRITE)) - -#define I915_VERBOSE 0 - -#define BEGIN_LP_RING(n) do { \ - drm_i915_private_t *dev_priv__ = dev->dev_private; \ - if (I915_VERBOSE) \ - DRM_DEBUG(" BEGIN_LP_RING %x\n", (int)(n)); \ - intel_ring_begin(dev, &dev_priv__->render_ring, (n)); \ -} while (0) - - -#define OUT_RING(x) do { \ - drm_i915_private_t *dev_priv__ = dev->dev_private; \ - if (I915_VERBOSE) \ - DRM_DEBUG(" OUT_RING %x\n", (int)(x)); \ - intel_ring_emit(dev, &dev_priv__->render_ring, x); \ -} while (0) - -#define ADVANCE_LP_RING() do { \ - drm_i915_private_t *dev_priv__ = dev->dev_private; \ - if (I915_VERBOSE) \ - DRM_DEBUG("ADVANCE_LP_RING %x\n", \ - dev_priv__->render_ring.tail); \ - intel_ring_advance(dev, &dev_priv__->render_ring); \ -} while(0) - /** * Reads a dword out of the status page, which is written to from the command * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or @@ -1259,72 +1406,9 @@ static inline void i915_write(struct drm_i915_private *dev_priv, u32 reg, * The area from dword 0x20 to 0x3ff is available for driver usage. */ #define READ_HWSP(dev_priv, reg) (((volatile u32 *)\ - (dev_priv->render_ring.status_page.page_addr))[reg]) + (LP_RING(dev_priv)->status_page.page_addr))[reg]) #define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX) #define I915_GEM_HWS_INDEX 0x20 #define I915_BREADCRUMB_INDEX 0x21 -#define INTEL_INFO(dev) (((struct drm_i915_private *) (dev)->dev_private)->info) - -#define IS_I830(dev) ((dev)->pci_device == 0x3577) -#define IS_845G(dev) ((dev)->pci_device == 0x2562) -#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) -#define IS_I865G(dev) ((dev)->pci_device == 0x2572) -#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) -#define IS_I915GM(dev) ((dev)->pci_device == 0x2592) -#define IS_I945G(dev) ((dev)->pci_device == 0x2772) -#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) -#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) -#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) -#define IS_GM45(dev) ((dev)->pci_device == 0x2A42) -#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x) -#define IS_PINEVIEW_G(dev) ((dev)->pci_device == 0xa001) -#define IS_PINEVIEW_M(dev) ((dev)->pci_device == 0xa011) -#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) -#define IS_G33(dev) (INTEL_INFO(dev)->is_g33) -#define IS_IRONLAKE_D(dev) ((dev)->pci_device == 0x0042) -#define IS_IRONLAKE_M(dev) ((dev)->pci_device == 0x0046) -#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) - -#define IS_GEN2(dev) (INTEL_INFO(dev)->gen == 2) -#define IS_GEN3(dev) (INTEL_INFO(dev)->gen == 3) -#define IS_GEN4(dev) (INTEL_INFO(dev)->gen == 4) -#define IS_GEN5(dev) (INTEL_INFO(dev)->gen == 5) -#define IS_GEN6(dev) (INTEL_INFO(dev)->gen == 6) - -#define HAS_BSD(dev) (INTEL_INFO(dev)->has_bsd_ring) -#define HAS_BLT(dev) (INTEL_INFO(dev)->has_blt_ring) -#define I915_NEED_GFX_HWS(dev) (INTEL_INFO(dev)->need_gfx_hws) - -#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) -#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) - -/* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte - * rows, which changed the alignment requirements and fence programming. - */ -#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \ - IS_I915GM(dev))) -#define SUPPORTS_DIGITAL_OUTPUTS(dev) (!IS_GEN2(dev) && !IS_PINEVIEW(dev)) -#define SUPPORTS_INTEGRATED_HDMI(dev) (IS_G4X(dev) || IS_GEN5(dev)) -#define SUPPORTS_INTEGRATED_DP(dev) (IS_G4X(dev) || IS_GEN5(dev)) -#define SUPPORTS_EDP(dev) (IS_IRONLAKE_M(dev)) -#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) -#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) -/* dsparb controlled by hw only */ -#define DSPARB_HWCONTROL(dev) (IS_G4X(dev) || IS_IRONLAKE(dev)) - -#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) -#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) -#define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) -#define I915_HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6) - -#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev)) -#define HAS_PIPE_CONTROL(dev) (IS_GEN5(dev) || IS_GEN6(dev)) - -#define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type) -#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT) -#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX) - -#define PRIMARY_RINGBUFFER_SIZE (128*1024) - #endif diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 275ec6ed43ae..cf4f74c7c6fb 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -34,38 +34,31 @@ #include <linux/slab.h> #include <linux/swap.h> #include <linux/pci.h> -#include <linux/intel-gtt.h> - -static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj); - -static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj); -static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj); -static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj); -static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, - int write); -static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, - uint64_t offset, - uint64_t size); -static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj); -static int i915_gem_object_wait_rendering(struct drm_gem_object *obj, - bool interruptible); -static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, - unsigned alignment); -static void i915_gem_clear_fence_reg(struct drm_gem_object *obj); -static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, + +static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj); +static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj); +static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj); +static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, + bool write); +static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, + uint64_t offset, + uint64_t size); +static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj); +static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, + unsigned alignment, + bool map_and_fenceable); +static void i915_gem_clear_fence_reg(struct drm_device *dev, + struct drm_i915_fence_reg *reg); +static int i915_gem_phys_pwrite(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv); -static void i915_gem_free_object_tail(struct drm_gem_object *obj); + struct drm_file *file); +static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj); -static int -i915_gem_object_get_pages(struct drm_gem_object *obj, - gfp_t gfpmask); +static int i915_gem_inactive_shrink(struct shrinker *shrinker, + int nr_to_scan, + gfp_t gfp_mask); -static void -i915_gem_object_put_pages(struct drm_gem_object *obj); - -static LIST_HEAD(shrink_list); -static DEFINE_SPINLOCK(shrink_list_lock); /* some bookkeeping */ static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv, @@ -82,34 +75,6 @@ static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, dev_priv->mm.object_memory -= size; } -static void i915_gem_info_add_gtt(struct drm_i915_private *dev_priv, - size_t size) -{ - dev_priv->mm.gtt_count++; - dev_priv->mm.gtt_memory += size; -} - -static void i915_gem_info_remove_gtt(struct drm_i915_private *dev_priv, - size_t size) -{ - dev_priv->mm.gtt_count--; - dev_priv->mm.gtt_memory -= size; -} - -static void i915_gem_info_add_pin(struct drm_i915_private *dev_priv, - size_t size) -{ - dev_priv->mm.pin_count++; - dev_priv->mm.pin_memory += size; -} - -static void i915_gem_info_remove_pin(struct drm_i915_private *dev_priv, - size_t size) -{ - dev_priv->mm.pin_count--; - dev_priv->mm.pin_memory -= size; -} - int i915_gem_check_is_wedged(struct drm_device *dev) { @@ -140,7 +105,7 @@ i915_gem_check_is_wedged(struct drm_device *dev) return -EIO; } -static int i915_mutex_lock_interruptible(struct drm_device *dev) +int i915_mutex_lock_interruptible(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; int ret; @@ -163,75 +128,80 @@ static int i915_mutex_lock_interruptible(struct drm_device *dev) } static inline bool -i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv) +i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) { - return obj_priv->gtt_space && - !obj_priv->active && - obj_priv->pin_count == 0; + return obj->gtt_space && !obj->active && obj->pin_count == 0; } -int i915_gem_do_init(struct drm_device *dev, - unsigned long start, - unsigned long end) +void i915_gem_do_init(struct drm_device *dev, + unsigned long start, + unsigned long mappable_end, + unsigned long end) { drm_i915_private_t *dev_priv = dev->dev_private; - if (start >= end || - (start & (PAGE_SIZE - 1)) != 0 || - (end & (PAGE_SIZE - 1)) != 0) { - return -EINVAL; - } - - drm_mm_init(&dev_priv->mm.gtt_space, start, - end - start); + drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); + dev_priv->mm.gtt_start = start; + dev_priv->mm.gtt_mappable_end = mappable_end; + dev_priv->mm.gtt_end = end; dev_priv->mm.gtt_total = end - start; + dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; - return 0; + /* Take over this portion of the GTT */ + intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); } int i915_gem_init_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_init *args = data; - int ret; + + if (args->gtt_start >= args->gtt_end || + (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) + return -EINVAL; mutex_lock(&dev->struct_mutex); - ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end); + i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); mutex_unlock(&dev->struct_mutex); - return ret; + return 0; } int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_get_aperture *args = data; + struct drm_i915_gem_object *obj; + size_t pinned; if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; + pinned = 0; mutex_lock(&dev->struct_mutex); - args->aper_size = dev_priv->mm.gtt_total; - args->aper_available_size = args->aper_size - dev_priv->mm.pin_memory; + list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) + pinned += obj->gtt_space->size; mutex_unlock(&dev->struct_mutex); + args->aper_size = dev_priv->mm.gtt_total; + args->aper_available_size = args->aper_size -pinned; + return 0; } - /** * Creates a new mm object and returns a handle to it. */ int i915_gem_create_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_create *args = data; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; int ret; u32 handle; @@ -242,45 +212,28 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data, if (obj == NULL) return -ENOMEM; - ret = drm_gem_handle_create(file_priv, obj, &handle); + ret = drm_gem_handle_create(file, &obj->base, &handle); if (ret) { - drm_gem_object_release(obj); - i915_gem_info_remove_obj(dev->dev_private, obj->size); + drm_gem_object_release(&obj->base); + i915_gem_info_remove_obj(dev->dev_private, obj->base.size); kfree(obj); return ret; } /* drop reference from allocate - handle holds it now */ - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); trace_i915_gem_object_create(obj); args->handle = handle; return 0; } -static inline int -fast_shmem_read(struct page **pages, - loff_t page_base, int page_offset, - char __user *data, - int length) -{ - char *vaddr; - int ret; - - vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]); - ret = __copy_to_user_inatomic(data, vaddr + page_offset, length); - kunmap_atomic(vaddr); - - return ret; -} - -static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj) +static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) { - drm_i915_private_t *dev_priv = obj->dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + drm_i915_private_t *dev_priv = obj->base.dev->dev_private; return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && - obj_priv->tiling_mode != I915_TILING_NONE; + obj->tiling_mode != I915_TILING_NONE; } static inline void @@ -356,38 +309,51 @@ slow_shmem_bit17_copy(struct page *gpu_page, * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow(). */ static int -i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_shmem_pread_fast(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pread *args, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; ssize_t remain; - loff_t offset, page_base; + loff_t offset; char __user *user_data; int page_offset, page_length; user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; - obj_priv = to_intel_bo(obj); offset = args->offset; while (remain > 0) { + struct page *page; + char *vaddr; + int ret; + /* Operation in this page * - * page_base = page offset within aperture * page_offset = offset within page * page_length = bytes to copy for this page */ - page_base = (offset & ~(PAGE_SIZE-1)); page_offset = offset & (PAGE_SIZE-1); page_length = remain; if ((page_offset + remain) > PAGE_SIZE) page_length = PAGE_SIZE - page_offset; - if (fast_shmem_read(obj_priv->pages, - page_base, page_offset, - user_data, page_length)) + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + + vaddr = kmap_atomic(page); + ret = __copy_to_user_inatomic(user_data, + vaddr + page_offset, + page_length); + kunmap_atomic(vaddr); + + mark_page_accessed(page); + page_cache_release(page); + if (ret) return -EFAULT; remain -= page_length; @@ -398,30 +364,6 @@ i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj, return 0; } -static int -i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj) -{ - int ret; - - ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN); - - /* If we've insufficient memory to map in the pages, attempt - * to make some space by throwing out some old buffers. - */ - if (ret == -ENOMEM) { - struct drm_device *dev = obj->dev; - - ret = i915_gem_evict_something(dev, obj->size, - i915_gem_get_gtt_alignment(obj)); - if (ret) - return ret; - - ret = i915_gem_object_get_pages(obj, 0); - } - - return ret; -} - /** * This is the fallback shmem pread path, which allocates temporary storage * in kernel space to copy_to_user into outside of the struct_mutex, so we @@ -429,18 +371,19 @@ i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj) * and not take page faults. */ static int -i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_shmem_pread_slow(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pread *args, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; struct mm_struct *mm = current->mm; struct page **user_pages; ssize_t remain; loff_t offset, pinned_pages, i; loff_t first_data_page, last_data_page, num_pages; - int shmem_page_index, shmem_page_offset; - int data_page_index, data_page_offset; + int shmem_page_offset; + int data_page_index, data_page_offset; int page_length; int ret; uint64_t data_ptr = args->data_ptr; @@ -479,19 +422,18 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - obj_priv = to_intel_bo(obj); offset = args->offset; while (remain > 0) { + struct page *page; + /* Operation in this page * - * shmem_page_index = page number within shmem file * shmem_page_offset = offset within page in shmem file * data_page_index = page number in get_user_pages return * data_page_offset = offset with data_page_index page. * page_length = bytes to copy for this page */ - shmem_page_index = offset / PAGE_SIZE; shmem_page_offset = offset & ~PAGE_MASK; data_page_index = data_ptr / PAGE_SIZE - first_data_page; data_page_offset = data_ptr & ~PAGE_MASK; @@ -502,8 +444,13 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, if ((data_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - data_page_offset; + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + if (do_bit17_swizzling) { - slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], + slow_shmem_bit17_copy(page, shmem_page_offset, user_pages[data_page_index], data_page_offset, @@ -512,11 +459,14 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, } else { slow_shmem_copy(user_pages[data_page_index], data_page_offset, - obj_priv->pages[shmem_page_index], + page, shmem_page_offset, page_length); } + mark_page_accessed(page); + page_cache_release(page); + remain -= page_length; data_ptr += page_length; offset += page_length; @@ -525,6 +475,7 @@ i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj, out: for (i = 0; i < pinned_pages; i++) { SetPageDirty(user_pages[i]); + mark_page_accessed(user_pages[i]); page_cache_release(user_pages[i]); } drm_free_large(user_pages); @@ -539,11 +490,10 @@ out: */ int i915_gem_pread_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_pread *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret = 0; if (args->size == 0) @@ -563,39 +513,33 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); /* Bounds check source. */ - if (args->offset > obj->size || args->size > obj->size - args->offset) { + if (args->offset > obj->base.size || + args->size > obj->base.size - args->offset) { ret = -EINVAL; goto out; } - ret = i915_gem_object_get_pages_or_evict(obj); - if (ret) - goto out; - ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset, args->size); if (ret) - goto out_put; + goto out; ret = -EFAULT; if (!i915_gem_object_needs_bit17_swizzle(obj)) - ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv); + ret = i915_gem_shmem_pread_fast(dev, obj, args, file); if (ret == -EFAULT) - ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv); + ret = i915_gem_shmem_pread_slow(dev, obj, args, file); -out_put: - i915_gem_object_put_pages(obj); out: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -645,32 +589,16 @@ slow_kernel_write(struct io_mapping *mapping, io_mapping_unmap(dst_vaddr); } -static inline int -fast_shmem_write(struct page **pages, - loff_t page_base, int page_offset, - char __user *data, - int length) -{ - char *vaddr; - int ret; - - vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT]); - ret = __copy_from_user_inatomic(vaddr + page_offset, data, length); - kunmap_atomic(vaddr); - - return ret; -} - /** * This is the fast pwrite path, where we copy the data directly from the * user into the GTT, uncached. */ static int -i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_gtt_pwrite_fast(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); drm_i915_private_t *dev_priv = dev->dev_private; ssize_t remain; loff_t offset, page_base; @@ -680,8 +608,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; - obj_priv = to_intel_bo(obj); - offset = obj_priv->gtt_offset + args->offset; + offset = obj->gtt_offset + args->offset; while (remain > 0) { /* Operation in this page @@ -721,11 +648,11 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit). */ static int -i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_gtt_pwrite_slow(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); drm_i915_private_t *dev_priv = dev->dev_private; ssize_t remain; loff_t gtt_page_base, offset; @@ -762,12 +689,15 @@ i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, goto out_unpin_pages; } - ret = i915_gem_object_set_to_gtt_domain(obj, 1); + ret = i915_gem_object_set_to_gtt_domain(obj, true); + if (ret) + goto out_unpin_pages; + + ret = i915_gem_object_put_fence(obj); if (ret) goto out_unpin_pages; - obj_priv = to_intel_bo(obj); - offset = obj_priv->gtt_offset + args->offset; + offset = obj->gtt_offset + args->offset; while (remain > 0) { /* Operation in this page @@ -813,39 +743,58 @@ out_unpin_pages: * copy_from_user into the kmapped pages backing the object. */ static int -i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_shmem_pwrite_fast(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; ssize_t remain; - loff_t offset, page_base; + loff_t offset; char __user *user_data; int page_offset, page_length; user_data = (char __user *) (uintptr_t) args->data_ptr; remain = args->size; - obj_priv = to_intel_bo(obj); offset = args->offset; - obj_priv->dirty = 1; + obj->dirty = 1; while (remain > 0) { + struct page *page; + char *vaddr; + int ret; + /* Operation in this page * - * page_base = page offset within aperture * page_offset = offset within page * page_length = bytes to copy for this page */ - page_base = (offset & ~(PAGE_SIZE-1)); page_offset = offset & (PAGE_SIZE-1); page_length = remain; if ((page_offset + remain) > PAGE_SIZE) page_length = PAGE_SIZE - page_offset; - if (fast_shmem_write(obj_priv->pages, - page_base, page_offset, - user_data, page_length)) + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + + vaddr = kmap_atomic(page, KM_USER0); + ret = __copy_from_user_inatomic(vaddr + page_offset, + user_data, + page_length); + kunmap_atomic(vaddr, KM_USER0); + + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + + /* If we get a fault while copying data, then (presumably) our + * source page isn't available. Return the error and we'll + * retry in the slow path. + */ + if (ret) return -EFAULT; remain -= page_length; @@ -864,17 +813,18 @@ i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj, * struct_mutex is held. */ static int -i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_shmem_pwrite_slow(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; struct mm_struct *mm = current->mm; struct page **user_pages; ssize_t remain; loff_t offset, pinned_pages, i; loff_t first_data_page, last_data_page, num_pages; - int shmem_page_index, shmem_page_offset; + int shmem_page_offset; int data_page_index, data_page_offset; int page_length; int ret; @@ -912,20 +862,19 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); - obj_priv = to_intel_bo(obj); offset = args->offset; - obj_priv->dirty = 1; + obj->dirty = 1; while (remain > 0) { + struct page *page; + /* Operation in this page * - * shmem_page_index = page number within shmem file * shmem_page_offset = offset within page in shmem file * data_page_index = page number in get_user_pages return * data_page_offset = offset with data_page_index page. * page_length = bytes to copy for this page */ - shmem_page_index = offset / PAGE_SIZE; shmem_page_offset = offset & ~PAGE_MASK; data_page_index = data_ptr / PAGE_SIZE - first_data_page; data_page_offset = data_ptr & ~PAGE_MASK; @@ -936,21 +885,32 @@ i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj, if ((data_page_offset + page_length) > PAGE_SIZE) page_length = PAGE_SIZE - data_page_offset; + page = read_cache_page_gfp(mapping, offset >> PAGE_SHIFT, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) { + ret = PTR_ERR(page); + goto out; + } + if (do_bit17_swizzling) { - slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index], + slow_shmem_bit17_copy(page, shmem_page_offset, user_pages[data_page_index], data_page_offset, page_length, 0); } else { - slow_shmem_copy(obj_priv->pages[shmem_page_index], + slow_shmem_copy(page, shmem_page_offset, user_pages[data_page_index], data_page_offset, page_length); } + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + remain -= page_length; data_ptr += page_length; offset += page_length; @@ -974,8 +934,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { struct drm_i915_gem_pwrite *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; if (args->size == 0) @@ -995,15 +954,15 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - obj = drm_gem_object_lookup(dev, file, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); /* Bounds check destination. */ - if (args->offset > obj->size || args->size > obj->size - args->offset) { + if (args->offset > obj->base.size || + args->size > obj->base.size - args->offset) { ret = -EINVAL; goto out; } @@ -1014,16 +973,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, * pread/pwrite currently are reading and writing from the CPU * perspective, requiring manual detiling by the client. */ - if (obj_priv->phys_obj) + if (obj->phys_obj) ret = i915_gem_phys_pwrite(dev, obj, args, file); - else if (obj_priv->tiling_mode == I915_TILING_NONE && - obj_priv->gtt_space && - obj->write_domain != I915_GEM_DOMAIN_CPU) { - ret = i915_gem_object_pin(obj, 0); + else if (obj->gtt_space && + obj->base.write_domain != I915_GEM_DOMAIN_CPU) { + ret = i915_gem_object_pin(obj, 0, true); if (ret) goto out; - ret = i915_gem_object_set_to_gtt_domain(obj, 1); + ret = i915_gem_object_set_to_gtt_domain(obj, true); + if (ret) + goto out_unpin; + + ret = i915_gem_object_put_fence(obj); if (ret) goto out_unpin; @@ -1034,26 +996,19 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, out_unpin: i915_gem_object_unpin(obj); } else { - ret = i915_gem_object_get_pages_or_evict(obj); - if (ret) - goto out; - ret = i915_gem_object_set_to_cpu_domain(obj, 1); if (ret) - goto out_put; + goto out; ret = -EFAULT; if (!i915_gem_object_needs_bit17_swizzle(obj)) ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file); if (ret == -EFAULT) ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file); - -out_put: - i915_gem_object_put_pages(obj); } out: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -1065,12 +1020,10 @@ unlock: */ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { - struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_set_domain *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; uint32_t read_domains = args->read_domains; uint32_t write_domain = args->write_domain; int ret; @@ -1095,28 +1048,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); - - intel_mark_busy(dev, obj); if (read_domains & I915_GEM_DOMAIN_GTT) { ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); - /* Update the LRU on the fence for the CPU access that's - * about to occur. - */ - if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_fence_reg *reg = - &dev_priv->fence_regs[obj_priv->fence_reg]; - list_move_tail(®->lru_list, - &dev_priv->mm.fence_list); - } - /* Silently promote "you're not bound, there was nothing to do" * to success, since the client was just asking us to * make sure everything was done. @@ -1127,11 +1067,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); } - /* Maintain LRU order of "inactive" objects */ - if (ret == 0 && i915_gem_object_is_inactive(obj_priv)) - list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); - - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -1142,10 +1078,10 @@ unlock: */ int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_sw_finish *args = data; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; int ret = 0; if (!(dev->driver->driver_features & DRIVER_GEM)) @@ -1155,17 +1091,17 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } /* Pinned buffers may be scanout, so flush the cache */ - if (to_intel_bo(obj)->pin_count) + if (obj->pin_count) i915_gem_object_flush_cpu_write_domain(obj); - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -1180,8 +1116,9 @@ unlock: */ int i915_gem_mmap_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_mmap *args = data; struct drm_gem_object *obj; loff_t offset; @@ -1190,10 +1127,15 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, if (!(dev->driver->driver_features & DRIVER_GEM)) return -ENODEV; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = drm_gem_object_lookup(dev, file, args->handle); if (obj == NULL) return -ENOENT; + if (obj->size > dev_priv->mm.gtt_mappable_end) { + drm_gem_object_unreference_unlocked(obj); + return -E2BIG; + } + offset = args->offset; down_write(¤t->mm->mmap_sem); @@ -1228,10 +1170,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data, */ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { - struct drm_gem_object *obj = vma->vm_private_data; - struct drm_device *dev = obj->dev; + struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data); + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); pgoff_t page_offset; unsigned long pfn; int ret = 0; @@ -1243,27 +1184,35 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf) /* Now bind it into the GTT if needed */ mutex_lock(&dev->struct_mutex); - if (!obj_priv->gtt_space) { - ret = i915_gem_object_bind_to_gtt(obj, 0); - if (ret) - goto unlock; - ret = i915_gem_object_set_to_gtt_domain(obj, write); + if (!obj->map_and_fenceable) { + ret = i915_gem_object_unbind(obj); if (ret) goto unlock; } - - /* Need a new fence register? */ - if (obj_priv->tiling_mode != I915_TILING_NONE) { - ret = i915_gem_object_get_fence_reg(obj, true); + if (!obj->gtt_space) { + ret = i915_gem_object_bind_to_gtt(obj, 0, true); if (ret) goto unlock; } - if (i915_gem_object_is_inactive(obj_priv)) - list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); + ret = i915_gem_object_set_to_gtt_domain(obj, write); + if (ret) + goto unlock; + + if (obj->tiling_mode == I915_TILING_NONE) + ret = i915_gem_object_put_fence(obj); + else + ret = i915_gem_object_get_fence(obj, NULL, true); + if (ret) + goto unlock; + + if (i915_gem_object_is_inactive(obj)) + list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); + + obj->fault_mappable = true; - pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) + + pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) + page_offset; /* Finally, remap it using the new GTT offset */ @@ -1272,11 +1221,12 @@ unlock: mutex_unlock(&dev->struct_mutex); switch (ret) { + case -EAGAIN: + set_need_resched(); case 0: case -ERESTARTSYS: return VM_FAULT_NOPAGE; case -ENOMEM: - case -EAGAIN: return VM_FAULT_OOM; default: return VM_FAULT_SIGBUS; @@ -1295,37 +1245,39 @@ unlock: * This routine allocates and attaches a fake offset for @obj. */ static int -i915_gem_create_mmap_offset(struct drm_gem_object *obj) +i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; struct drm_gem_mm *mm = dev->mm_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); struct drm_map_list *list; struct drm_local_map *map; int ret = 0; /* Set the object up for mmap'ing */ - list = &obj->map_list; + list = &obj->base.map_list; list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL); if (!list->map) return -ENOMEM; map = list->map; map->type = _DRM_GEM; - map->size = obj->size; + map->size = obj->base.size; map->handle = obj; /* Get a DRM GEM mmap offset allocated... */ list->file_offset_node = drm_mm_search_free(&mm->offset_manager, - obj->size / PAGE_SIZE, 0, 0); + obj->base.size / PAGE_SIZE, + 0, 0); if (!list->file_offset_node) { - DRM_ERROR("failed to allocate offset for bo %d\n", obj->name); + DRM_ERROR("failed to allocate offset for bo %d\n", + obj->base.name); ret = -ENOSPC; goto out_free_list; } list->file_offset_node = drm_mm_get_block(list->file_offset_node, - obj->size / PAGE_SIZE, 0); + obj->base.size / PAGE_SIZE, + 0); if (!list->file_offset_node) { ret = -ENOMEM; goto out_free_list; @@ -1338,16 +1290,13 @@ i915_gem_create_mmap_offset(struct drm_gem_object *obj) goto out_free_mm; } - /* By now we should be all set, any drm_mmap request on the offset - * below will get to our mmap & fault handler */ - obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT; - return 0; out_free_mm: drm_mm_put_block(list->file_offset_node); out_free_list: kfree(list->map); + list->map = NULL; return ret; } @@ -1367,38 +1316,51 @@ out_free_list: * fixup by i915_gem_fault(). */ void -i915_gem_release_mmap(struct drm_gem_object *obj) +i915_gem_release_mmap(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + if (!obj->fault_mappable) + return; + + unmap_mapping_range(obj->base.dev->dev_mapping, + (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT, + obj->base.size, 1); - if (dev->dev_mapping) - unmap_mapping_range(dev->dev_mapping, - obj_priv->mmap_offset, obj->size, 1); + obj->fault_mappable = false; } static void -i915_gem_free_mmap_offset(struct drm_gem_object *obj) +i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_device *dev = obj->base.dev; struct drm_gem_mm *mm = dev->mm_private; - struct drm_map_list *list; + struct drm_map_list *list = &obj->base.map_list; - list = &obj->map_list; drm_ht_remove_item(&mm->offset_hash, &list->hash); + drm_mm_put_block(list->file_offset_node); + kfree(list->map); + list->map = NULL; +} - if (list->file_offset_node) { - drm_mm_put_block(list->file_offset_node); - list->file_offset_node = NULL; - } +static uint32_t +i915_gem_get_gtt_size(struct drm_i915_gem_object *obj) +{ + struct drm_device *dev = obj->base.dev; + uint32_t size; - if (list->map) { - kfree(list->map); - list->map = NULL; - } + if (INTEL_INFO(dev)->gen >= 4 || + obj->tiling_mode == I915_TILING_NONE) + return obj->base.size; - obj_priv->mmap_offset = 0; + /* Previous chips need a power-of-two fence region when tiling */ + if (INTEL_INFO(dev)->gen == 3) + size = 1024*1024; + else + size = 512*1024; + + while (size < obj->base.size) + size <<= 1; + + return size; } /** @@ -1406,42 +1368,68 @@ i915_gem_free_mmap_offset(struct drm_gem_object *obj) * @obj: object to check * * Return the required GTT alignment for an object, taking into account - * potential fence register mapping if needed. + * potential fence register mapping. */ static uint32_t -i915_gem_get_gtt_alignment(struct drm_gem_object *obj) +i915_gem_get_gtt_alignment(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int start, i; + struct drm_device *dev = obj->base.dev; /* * Minimum alignment is 4k (GTT page size), but might be greater * if a fence register is needed for the object. */ - if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE) + if (INTEL_INFO(dev)->gen >= 4 || + obj->tiling_mode == I915_TILING_NONE) return 4096; /* * Previous chips need to be aligned to the size of the smallest * fence register that can contain the object. */ - if (INTEL_INFO(dev)->gen == 3) - start = 1024*1024; - else - start = 512*1024; + return i915_gem_get_gtt_size(obj); +} + +/** + * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an + * unfenced object + * @obj: object to check + * + * Return the required GTT alignment for an object, only taking into account + * unfenced tiled surface requirements. + */ +static uint32_t +i915_gem_get_unfenced_gtt_alignment(struct drm_i915_gem_object *obj) +{ + struct drm_device *dev = obj->base.dev; + int tile_height; - for (i = start; i < obj->size; i <<= 1) - ; + /* + * Minimum alignment is 4k (GTT page size) for sane hw. + */ + if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) || + obj->tiling_mode == I915_TILING_NONE) + return 4096; - return i; + /* + * Older chips need unfenced tiled buffers to be aligned to the left + * edge of an even tile row (where tile rows are counted as if the bo is + * placed in a fenced gtt region). + */ + if (IS_GEN2(dev) || + (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))) + tile_height = 32; + else + tile_height = 8; + + return tile_height * obj->stride * 2; } /** * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing * @dev: DRM device * @data: GTT mapping ioctl data - * @file_priv: GEM object info + * @file: GEM object info * * Simply returns the fake offset to userspace so it can mmap it. * The mmap call will end up in drm_gem_mmap(), which will set things @@ -1454,11 +1442,11 @@ i915_gem_get_gtt_alignment(struct drm_gem_object *obj) */ int i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_mmap_gtt *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; if (!(dev->driver->driver_features & DRIVER_GEM)) @@ -1468,130 +1456,196 @@ i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); - if (obj_priv->madv != I915_MADV_WILLNEED) { + if (obj->base.size > dev_priv->mm.gtt_mappable_end) { + ret = -E2BIG; + goto unlock; + } + + if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to mmap a purgeable buffer\n"); ret = -EINVAL; goto out; } - if (!obj_priv->mmap_offset) { + if (!obj->base.map_list.map) { ret = i915_gem_create_mmap_offset(obj); if (ret) goto out; } - args->offset = obj_priv->mmap_offset; - - /* - * Pull it into the GTT so that we have a page list (makes the - * initial fault faster and any subsequent flushing possible). - */ - if (!obj_priv->agp_mem) { - ret = i915_gem_object_bind_to_gtt(obj, 0); - if (ret) - goto out; - } + args->offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT; out: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; } +static int +i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, + gfp_t gfpmask) +{ + int page_count, i; + struct address_space *mapping; + struct inode *inode; + struct page *page; + + /* Get the list of pages out of our struct file. They'll be pinned + * at this point until we release them. + */ + page_count = obj->base.size / PAGE_SIZE; + BUG_ON(obj->pages != NULL); + obj->pages = drm_malloc_ab(page_count, sizeof(struct page *)); + if (obj->pages == NULL) + return -ENOMEM; + + inode = obj->base.filp->f_path.dentry->d_inode; + mapping = inode->i_mapping; + for (i = 0; i < page_count; i++) { + page = read_cache_page_gfp(mapping, i, + GFP_HIGHUSER | + __GFP_COLD | + __GFP_RECLAIMABLE | + gfpmask); + if (IS_ERR(page)) + goto err_pages; + + obj->pages[i] = page; + } + + if (obj->tiling_mode != I915_TILING_NONE) + i915_gem_object_do_bit_17_swizzle(obj); + + return 0; + +err_pages: + while (i--) + page_cache_release(obj->pages[i]); + + drm_free_large(obj->pages); + obj->pages = NULL; + return PTR_ERR(page); +} + static void -i915_gem_object_put_pages(struct drm_gem_object *obj) +i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int page_count = obj->size / PAGE_SIZE; + int page_count = obj->base.size / PAGE_SIZE; int i; - BUG_ON(obj_priv->pages_refcount == 0); - BUG_ON(obj_priv->madv == __I915_MADV_PURGED); - - if (--obj_priv->pages_refcount != 0) - return; + BUG_ON(obj->madv == __I915_MADV_PURGED); - if (obj_priv->tiling_mode != I915_TILING_NONE) + if (obj->tiling_mode != I915_TILING_NONE) i915_gem_object_save_bit_17_swizzle(obj); - if (obj_priv->madv == I915_MADV_DONTNEED) - obj_priv->dirty = 0; + if (obj->madv == I915_MADV_DONTNEED) + obj->dirty = 0; for (i = 0; i < page_count; i++) { - if (obj_priv->dirty) - set_page_dirty(obj_priv->pages[i]); + if (obj->dirty) + set_page_dirty(obj->pages[i]); - if (obj_priv->madv == I915_MADV_WILLNEED) - mark_page_accessed(obj_priv->pages[i]); + if (obj->madv == I915_MADV_WILLNEED) + mark_page_accessed(obj->pages[i]); - page_cache_release(obj_priv->pages[i]); + page_cache_release(obj->pages[i]); } - obj_priv->dirty = 0; - - drm_free_large(obj_priv->pages); - obj_priv->pages = NULL; -} - -static uint32_t -i915_gem_next_request_seqno(struct drm_device *dev, - struct intel_ring_buffer *ring) -{ - drm_i915_private_t *dev_priv = dev->dev_private; + obj->dirty = 0; - ring->outstanding_lazy_request = true; - return dev_priv->next_seqno; + drm_free_large(obj->pages); + obj->pages = NULL; } -static void -i915_gem_object_move_to_active(struct drm_gem_object *obj, - struct intel_ring_buffer *ring) +void +i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *ring, + u32 seqno) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - uint32_t seqno = i915_gem_next_request_seqno(dev, ring); BUG_ON(ring == NULL); - obj_priv->ring = ring; + obj->ring = ring; /* Add a reference if we're newly entering the active list. */ - if (!obj_priv->active) { - drm_gem_object_reference(obj); - obj_priv->active = 1; + if (!obj->active) { + drm_gem_object_reference(&obj->base); + obj->active = 1; } /* Move from whatever list we were on to the tail of execution. */ - list_move_tail(&obj_priv->mm_list, &dev_priv->mm.active_list); - list_move_tail(&obj_priv->ring_list, &ring->active_list); - obj_priv->last_rendering_seqno = seqno; + list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); + list_move_tail(&obj->ring_list, &ring->active_list); + + obj->last_rendering_seqno = seqno; + if (obj->fenced_gpu_access) { + struct drm_i915_fence_reg *reg; + + BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE); + + obj->last_fenced_seqno = seqno; + obj->last_fenced_ring = ring; + + reg = &dev_priv->fence_regs[obj->fence_reg]; + list_move_tail(®->lru_list, &dev_priv->mm.fence_list); + } } static void -i915_gem_object_move_to_flushing(struct drm_gem_object *obj) +i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; + list_del_init(&obj->ring_list); + obj->last_rendering_seqno = 0; +} + +static void +i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) +{ + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - BUG_ON(!obj_priv->active); - list_move_tail(&obj_priv->mm_list, &dev_priv->mm.flushing_list); - list_del_init(&obj_priv->ring_list); - obj_priv->last_rendering_seqno = 0; + BUG_ON(!obj->active); + list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); + + i915_gem_object_move_off_active(obj); +} + +static void +i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) +{ + struct drm_device *dev = obj->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + + if (obj->pin_count != 0) + list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); + else + list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); + + BUG_ON(!list_empty(&obj->gpu_write_list)); + BUG_ON(!obj->active); + obj->ring = NULL; + + i915_gem_object_move_off_active(obj); + obj->fenced_gpu_access = false; + + obj->active = 0; + obj->pending_gpu_write = false; + drm_gem_object_unreference(&obj->base); + + WARN_ON(i915_verify_lists(dev)); } /* Immediately discard the backing storage */ static void -i915_gem_object_truncate(struct drm_gem_object *obj) +i915_gem_object_truncate(struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); struct inode *inode; /* Our goal here is to return as much of the memory as @@ -1600,42 +1654,18 @@ i915_gem_object_truncate(struct drm_gem_object *obj) * backing pages, *now*. Here we mirror the actions taken * when by shmem_delete_inode() to release the backing store. */ - inode = obj->filp->f_path.dentry->d_inode; + inode = obj->base.filp->f_path.dentry->d_inode; truncate_inode_pages(inode->i_mapping, 0); if (inode->i_op->truncate_range) inode->i_op->truncate_range(inode, 0, (loff_t)-1); - obj_priv->madv = __I915_MADV_PURGED; + obj->madv = __I915_MADV_PURGED; } static inline int -i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv) +i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) { - return obj_priv->madv == I915_MADV_DONTNEED; -} - -static void -i915_gem_object_move_to_inactive(struct drm_gem_object *obj) -{ - struct drm_device *dev = obj->dev; - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - - if (obj_priv->pin_count != 0) - list_move_tail(&obj_priv->mm_list, &dev_priv->mm.pinned_list); - else - list_move_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); - list_del_init(&obj_priv->ring_list); - - BUG_ON(!list_empty(&obj_priv->gpu_write_list)); - - obj_priv->last_rendering_seqno = 0; - obj_priv->ring = NULL; - if (obj_priv->active) { - obj_priv->active = 0; - drm_gem_object_unreference(obj); - } - WARN_ON(i915_verify_lists(dev)); + return obj->madv == I915_MADV_DONTNEED; } static void @@ -1643,37 +1673,27 @@ i915_gem_process_flushing_list(struct drm_device *dev, uint32_t flush_domains, struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv, *next; + struct drm_i915_gem_object *obj, *next; - list_for_each_entry_safe(obj_priv, next, + list_for_each_entry_safe(obj, next, &ring->gpu_write_list, gpu_write_list) { - struct drm_gem_object *obj = &obj_priv->base; - - if (obj->write_domain & flush_domains) { - uint32_t old_write_domain = obj->write_domain; + if (obj->base.write_domain & flush_domains) { + uint32_t old_write_domain = obj->base.write_domain; - obj->write_domain = 0; - list_del_init(&obj_priv->gpu_write_list); - i915_gem_object_move_to_active(obj, ring); - - /* update the fence lru list */ - if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { - struct drm_i915_fence_reg *reg = - &dev_priv->fence_regs[obj_priv->fence_reg]; - list_move_tail(®->lru_list, - &dev_priv->mm.fence_list); - } + obj->base.write_domain = 0; + list_del_init(&obj->gpu_write_list); + i915_gem_object_move_to_active(obj, ring, + i915_gem_next_request_seqno(dev, ring)); trace_i915_gem_object_change_domain(obj, - obj->read_domains, + obj->base.read_domains, old_write_domain); } } } -uint32_t +int i915_add_request(struct drm_device *dev, struct drm_file *file, struct drm_i915_gem_request *request, @@ -1683,17 +1703,17 @@ i915_add_request(struct drm_device *dev, struct drm_i915_file_private *file_priv = NULL; uint32_t seqno; int was_empty; + int ret; + + BUG_ON(request == NULL); if (file != NULL) file_priv = file->driver_priv; - if (request == NULL) { - request = kzalloc(sizeof(*request), GFP_KERNEL); - if (request == NULL) - return 0; - } + ret = ring->add_request(ring, &seqno); + if (ret) + return ret; - seqno = ring->add_request(dev, ring, 0); ring->outstanding_lazy_request = false; request->seqno = seqno; @@ -1717,26 +1737,7 @@ i915_add_request(struct drm_device *dev, queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); } - return seqno; -} - -/** - * Command execution barrier - * - * Ensures that all commands in the ring are finished - * before signalling the CPU - */ -static void -i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring) -{ - uint32_t flush_domains = 0; - - /* The sampler always gets flushed on i965 (sigh) */ - if (INTEL_INFO(dev)->gen >= 4) - flush_domains |= I915_GEM_DOMAIN_SAMPLER; - - ring->flush(dev, ring, - I915_GEM_DOMAIN_COMMAND, flush_domains); + return 0; } static inline void @@ -1769,62 +1770,76 @@ static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, } while (!list_empty(&ring->active_list)) { - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; + + obj = list_first_entry(&ring->active_list, + struct drm_i915_gem_object, + ring_list); + + obj->base.write_domain = 0; + list_del_init(&obj->gpu_write_list); + i915_gem_object_move_to_inactive(obj); + } +} + +static void i915_gem_reset_fences(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int i; + + for (i = 0; i < 16; i++) { + struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; + struct drm_i915_gem_object *obj = reg->obj; + + if (!obj) + continue; - obj_priv = list_first_entry(&ring->active_list, - struct drm_i915_gem_object, - ring_list); + if (obj->tiling_mode) + i915_gem_release_mmap(obj); - obj_priv->base.write_domain = 0; - list_del_init(&obj_priv->gpu_write_list); - i915_gem_object_move_to_inactive(&obj_priv->base); + reg->obj->fence_reg = I915_FENCE_REG_NONE; + reg->obj->fenced_gpu_access = false; + reg->obj->last_fenced_seqno = 0; + reg->obj->last_fenced_ring = NULL; + i915_gem_clear_fence_reg(dev, reg); } } void i915_gem_reset(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int i; - i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring); - i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring); - i915_gem_reset_ring_lists(dev_priv, &dev_priv->blt_ring); + for (i = 0; i < I915_NUM_RINGS; i++) + i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]); /* Remove anything from the flushing lists. The GPU cache is likely * to be lost on reset along with the data, so simply move the * lost bo to the inactive list. */ while (!list_empty(&dev_priv->mm.flushing_list)) { - obj_priv = list_first_entry(&dev_priv->mm.flushing_list, - struct drm_i915_gem_object, - mm_list); + obj= list_first_entry(&dev_priv->mm.flushing_list, + struct drm_i915_gem_object, + mm_list); - obj_priv->base.write_domain = 0; - list_del_init(&obj_priv->gpu_write_list); - i915_gem_object_move_to_inactive(&obj_priv->base); + obj->base.write_domain = 0; + list_del_init(&obj->gpu_write_list); + i915_gem_object_move_to_inactive(obj); } /* Move everything out of the GPU domains to ensure we do any * necessary invalidation upon reuse. */ - list_for_each_entry(obj_priv, + list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { - obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS; + obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; } /* The fence registers are invalidated so clear them out */ - for (i = 0; i < 16; i++) { - struct drm_i915_fence_reg *reg; - - reg = &dev_priv->fence_regs[i]; - if (!reg->obj) - continue; - - i915_gem_clear_fence_reg(reg->obj); - } + i915_gem_reset_fences(dev); } /** @@ -1836,6 +1851,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev, { drm_i915_private_t *dev_priv = dev->dev_private; uint32_t seqno; + int i; if (!ring->status_page.page_addr || list_empty(&ring->request_list)) @@ -1843,7 +1859,12 @@ i915_gem_retire_requests_ring(struct drm_device *dev, WARN_ON(i915_verify_lists(dev)); - seqno = ring->get_seqno(dev, ring); + seqno = ring->get_seqno(ring); + + for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) + if (seqno >= ring->sync_seqno[i]) + ring->sync_seqno[i] = 0; + while (!list_empty(&ring->request_list)) { struct drm_i915_gem_request *request; @@ -1865,18 +1886,16 @@ i915_gem_retire_requests_ring(struct drm_device *dev, * by the ringbuffer to the flushing/inactive lists as appropriate. */ while (!list_empty(&ring->active_list)) { - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; - obj_priv = list_first_entry(&ring->active_list, - struct drm_i915_gem_object, - ring_list); + obj= list_first_entry(&ring->active_list, + struct drm_i915_gem_object, + ring_list); - if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno)) + if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) break; - obj = &obj_priv->base; - if (obj->write_domain != 0) + if (obj->base.write_domain != 0) i915_gem_object_move_to_flushing(obj); else i915_gem_object_move_to_inactive(obj); @@ -1884,7 +1903,7 @@ i915_gem_retire_requests_ring(struct drm_device *dev, if (unlikely (dev_priv->trace_irq_seqno && i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) { - ring->user_irq_put(dev, ring); + ring->irq_put(ring); dev_priv->trace_irq_seqno = 0; } @@ -1895,24 +1914,24 @@ void i915_gem_retire_requests(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + int i; if (!list_empty(&dev_priv->mm.deferred_free_list)) { - struct drm_i915_gem_object *obj_priv, *tmp; + struct drm_i915_gem_object *obj, *next; /* We must be careful that during unbind() we do not * accidentally infinitely recurse into retire requests. * Currently: * retire -> free -> unbind -> wait -> retire_ring */ - list_for_each_entry_safe(obj_priv, tmp, + list_for_each_entry_safe(obj, next, &dev_priv->mm.deferred_free_list, mm_list) - i915_gem_free_object_tail(&obj_priv->base); + i915_gem_free_object_tail(obj); } - i915_gem_retire_requests_ring(dev, &dev_priv->render_ring); - i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring); - i915_gem_retire_requests_ring(dev, &dev_priv->blt_ring); + for (i = 0; i < I915_NUM_RINGS; i++) + i915_gem_retire_requests_ring(dev, &dev_priv->ring[i]); } static void @@ -1920,6 +1939,8 @@ i915_gem_retire_work_handler(struct work_struct *work) { drm_i915_private_t *dev_priv; struct drm_device *dev; + bool idle; + int i; dev_priv = container_of(work, drm_i915_private_t, mm.retire_work.work); @@ -1933,11 +1954,31 @@ i915_gem_retire_work_handler(struct work_struct *work) i915_gem_retire_requests(dev); - if (!dev_priv->mm.suspended && - (!list_empty(&dev_priv->render_ring.request_list) || - !list_empty(&dev_priv->bsd_ring.request_list) || - !list_empty(&dev_priv->blt_ring.request_list))) + /* Send a periodic flush down the ring so we don't hold onto GEM + * objects indefinitely. + */ + idle = true; + for (i = 0; i < I915_NUM_RINGS; i++) { + struct intel_ring_buffer *ring = &dev_priv->ring[i]; + + if (!list_empty(&ring->gpu_write_list)) { + struct drm_i915_gem_request *request; + int ret; + + ret = i915_gem_flush_ring(dev, ring, 0, + I915_GEM_GPU_DOMAINS); + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (ret || request == NULL || + i915_add_request(dev, NULL, request, ring)) + kfree(request); + } + + idle &= list_empty(&ring->request_list); + } + + if (!dev_priv->mm.suspended && !idle) queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ); + mutex_unlock(&dev->struct_mutex); } @@ -1954,14 +1995,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, if (atomic_read(&dev_priv->mm.wedged)) return -EAGAIN; - if (ring->outstanding_lazy_request) { - seqno = i915_add_request(dev, NULL, NULL, ring); - if (seqno == 0) + if (seqno == ring->outstanding_lazy_request) { + struct drm_i915_gem_request *request; + + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (request == NULL) return -ENOMEM; + + ret = i915_add_request(dev, NULL, request, ring); + if (ret) { + kfree(request); + return ret; + } + + seqno = request->seqno; } - BUG_ON(seqno == dev_priv->next_seqno); - if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { + if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { if (HAS_PCH_SPLIT(dev)) ier = I915_READ(DEIER) | I915_READ(GTIER); else @@ -1975,21 +2025,23 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, trace_i915_gem_request_wait_begin(dev, seqno); - ring->waiting_gem_seqno = seqno; - ring->user_irq_get(dev, ring); - if (interruptible) - ret = wait_event_interruptible(ring->irq_queue, - i915_seqno_passed( - ring->get_seqno(dev, ring), seqno) - || atomic_read(&dev_priv->mm.wedged)); - else - wait_event(ring->irq_queue, - i915_seqno_passed( - ring->get_seqno(dev, ring), seqno) - || atomic_read(&dev_priv->mm.wedged)); + ring->waiting_seqno = seqno; + if (ring->irq_get(ring)) { + if (interruptible) + ret = wait_event_interruptible(ring->irq_queue, + i915_seqno_passed(ring->get_seqno(ring), seqno) + || atomic_read(&dev_priv->mm.wedged)); + else + wait_event(ring->irq_queue, + i915_seqno_passed(ring->get_seqno(ring), seqno) + || atomic_read(&dev_priv->mm.wedged)); - ring->user_irq_put(dev, ring); - ring->waiting_gem_seqno = 0; + ring->irq_put(ring); + } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring), + seqno) || + atomic_read(&dev_priv->mm.wedged), 3000)) + ret = -EBUSY; + ring->waiting_seqno = 0; trace_i915_gem_request_wait_end(dev, seqno); } @@ -1998,7 +2050,7 @@ i915_do_wait_request(struct drm_device *dev, uint32_t seqno, if (ret && ret != -ERESTARTSYS) DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n", - __func__, ret, seqno, ring->get_seqno(dev, ring), + __func__, ret, seqno, ring->get_seqno(ring), dev_priv->next_seqno); /* Directly dispatch request retiring. While we have the work queue @@ -2023,70 +2075,30 @@ i915_wait_request(struct drm_device *dev, uint32_t seqno, return i915_do_wait_request(dev, seqno, 1, ring); } -static void -i915_gem_flush_ring(struct drm_device *dev, - struct drm_file *file_priv, - struct intel_ring_buffer *ring, - uint32_t invalidate_domains, - uint32_t flush_domains) -{ - ring->flush(dev, ring, invalidate_domains, flush_domains); - i915_gem_process_flushing_list(dev, flush_domains, ring); -} - -static void -i915_gem_flush(struct drm_device *dev, - struct drm_file *file_priv, - uint32_t invalidate_domains, - uint32_t flush_domains, - uint32_t flush_rings) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - - if (flush_domains & I915_GEM_DOMAIN_CPU) - drm_agp_chipset_flush(dev); - - if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) { - if (flush_rings & RING_RENDER) - i915_gem_flush_ring(dev, file_priv, - &dev_priv->render_ring, - invalidate_domains, flush_domains); - if (flush_rings & RING_BSD) - i915_gem_flush_ring(dev, file_priv, - &dev_priv->bsd_ring, - invalidate_domains, flush_domains); - if (flush_rings & RING_BLT) - i915_gem_flush_ring(dev, file_priv, - &dev_priv->blt_ring, - invalidate_domains, flush_domains); - } -} - /** * Ensures that all rendering to the object has completed and the object is * safe to unbind from the GTT or access from the CPU. */ -static int -i915_gem_object_wait_rendering(struct drm_gem_object *obj, +int +i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, bool interruptible) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_device *dev = obj->base.dev; int ret; /* This function only exists to support waiting for existing rendering, * not for emitting required flushes. */ - BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0); + BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0); /* If there is rendering queued on the buffer being evicted, wait for * it. */ - if (obj_priv->active) { + if (obj->active) { ret = i915_do_wait_request(dev, - obj_priv->last_rendering_seqno, + obj->last_rendering_seqno, interruptible, - obj_priv->ring); + obj->ring); if (ret) return ret; } @@ -2098,17 +2110,14 @@ i915_gem_object_wait_rendering(struct drm_gem_object *obj, * Unbinds an object from the GTT aperture. */ int -i915_gem_object_unbind(struct drm_gem_object *obj) +i915_gem_object_unbind(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int ret = 0; - if (obj_priv->gtt_space == NULL) + if (obj->gtt_space == NULL) return 0; - if (obj_priv->pin_count != 0) { + if (obj->pin_count != 0) { DRM_ERROR("Attempting to unbind pinned buffer\n"); return -EINVAL; } @@ -2131,27 +2140,27 @@ i915_gem_object_unbind(struct drm_gem_object *obj) */ if (ret) { i915_gem_clflush_object(obj); - obj->read_domains = obj->write_domain = I915_GEM_DOMAIN_CPU; + obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU; } /* release the fence reg _after_ flushing */ - if (obj_priv->fence_reg != I915_FENCE_REG_NONE) - i915_gem_clear_fence_reg(obj); - - drm_unbind_agp(obj_priv->agp_mem); - drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE); + ret = i915_gem_object_put_fence(obj); + if (ret == -ERESTARTSYS) + return ret; - i915_gem_object_put_pages(obj); - BUG_ON(obj_priv->pages_refcount); + i915_gem_gtt_unbind_object(obj); + i915_gem_object_put_pages_gtt(obj); - i915_gem_info_remove_gtt(dev_priv, obj->size); - list_del_init(&obj_priv->mm_list); + list_del_init(&obj->gtt_list); + list_del_init(&obj->mm_list); + /* Avoid an unnecessary call to unbind on rebind. */ + obj->map_and_fenceable = true; - drm_mm_put_block(obj_priv->gtt_space); - obj_priv->gtt_space = NULL; - obj_priv->gtt_offset = 0; + drm_mm_put_block(obj->gtt_space); + obj->gtt_space = NULL; + obj->gtt_offset = 0; - if (i915_gem_object_is_purgeable(obj_priv)) + if (i915_gem_object_is_purgeable(obj)) i915_gem_object_truncate(obj); trace_i915_gem_object_unbind(obj); @@ -2159,14 +2168,37 @@ i915_gem_object_unbind(struct drm_gem_object *obj) return ret; } +int +i915_gem_flush_ring(struct drm_device *dev, + struct intel_ring_buffer *ring, + uint32_t invalidate_domains, + uint32_t flush_domains) +{ + int ret; + + ret = ring->flush(ring, invalidate_domains, flush_domains); + if (ret) + return ret; + + i915_gem_process_flushing_list(dev, flush_domains, ring); + return 0; +} + static int i915_ring_idle(struct drm_device *dev, struct intel_ring_buffer *ring) { + int ret; + if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) return 0; - i915_gem_flush_ring(dev, NULL, ring, - I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + if (!list_empty(&ring->gpu_write_list)) { + ret = i915_gem_flush_ring(dev, ring, + I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS); + if (ret) + return ret; + } + return i915_wait_request(dev, i915_gem_next_request_seqno(dev, ring), ring); @@ -2177,7 +2209,7 @@ i915_gpu_idle(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; bool lists_empty; - int ret; + int ret, i; lists_empty = (list_empty(&dev_priv->mm.flushing_list) && list_empty(&dev_priv->mm.active_list)); @@ -2185,258 +2217,305 @@ i915_gpu_idle(struct drm_device *dev) return 0; /* Flush everything onto the inactive list. */ - ret = i915_ring_idle(dev, &dev_priv->render_ring); - if (ret) - return ret; - - ret = i915_ring_idle(dev, &dev_priv->bsd_ring); - if (ret) - return ret; - - ret = i915_ring_idle(dev, &dev_priv->blt_ring); - if (ret) - return ret; - - return 0; -} - -static int -i915_gem_object_get_pages(struct drm_gem_object *obj, - gfp_t gfpmask) -{ - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int page_count, i; - struct address_space *mapping; - struct inode *inode; - struct page *page; - - BUG_ON(obj_priv->pages_refcount - == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT); - - if (obj_priv->pages_refcount++ != 0) - return 0; - - /* Get the list of pages out of our struct file. They'll be pinned - * at this point until we release them. - */ - page_count = obj->size / PAGE_SIZE; - BUG_ON(obj_priv->pages != NULL); - obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *)); - if (obj_priv->pages == NULL) { - obj_priv->pages_refcount--; - return -ENOMEM; - } - - inode = obj->filp->f_path.dentry->d_inode; - mapping = inode->i_mapping; - for (i = 0; i < page_count; i++) { - page = read_cache_page_gfp(mapping, i, - GFP_HIGHUSER | - __GFP_COLD | - __GFP_RECLAIMABLE | - gfpmask); - if (IS_ERR(page)) - goto err_pages; - - obj_priv->pages[i] = page; + for (i = 0; i < I915_NUM_RINGS; i++) { + ret = i915_ring_idle(dev, &dev_priv->ring[i]); + if (ret) + return ret; } - if (obj_priv->tiling_mode != I915_TILING_NONE) - i915_gem_object_do_bit_17_swizzle(obj); - return 0; - -err_pages: - while (i--) - page_cache_release(obj_priv->pages[i]); - - drm_free_large(obj_priv->pages); - obj_priv->pages = NULL; - obj_priv->pages_refcount--; - return PTR_ERR(page); } -static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg) +static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined) { - struct drm_gem_object *obj = reg->obj; - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int regnum = obj_priv->fence_reg; + u32 size = obj->gtt_space->size; + int regnum = obj->fence_reg; uint64_t val; - val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & - 0xfffff000) << 32; - val |= obj_priv->gtt_offset & 0xfffff000; - val |= (uint64_t)((obj_priv->stride / 128) - 1) << + val = (uint64_t)((obj->gtt_offset + size - 4096) & + 0xfffff000) << 32; + val |= obj->gtt_offset & 0xfffff000; + val |= (uint64_t)((obj->stride / 128) - 1) << SANDYBRIDGE_FENCE_PITCH_SHIFT; - if (obj_priv->tiling_mode == I915_TILING_Y) + if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I965_FENCE_TILING_Y_SHIFT; val |= I965_FENCE_REG_VALID; - I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val); + if (pipelined) { + int ret = intel_ring_begin(pipelined, 6); + if (ret) + return ret; + + intel_ring_emit(pipelined, MI_NOOP); + intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); + intel_ring_emit(pipelined, (u32)val); + intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); + intel_ring_emit(pipelined, (u32)(val >> 32)); + intel_ring_advance(pipelined); + } else + I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); + + return 0; } -static void i965_write_fence_reg(struct drm_i915_fence_reg *reg) +static int i965_write_fence_reg(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined) { - struct drm_gem_object *obj = reg->obj; - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int regnum = obj_priv->fence_reg; + u32 size = obj->gtt_space->size; + int regnum = obj->fence_reg; uint64_t val; - val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) & + val = (uint64_t)((obj->gtt_offset + size - 4096) & 0xfffff000) << 32; - val |= obj_priv->gtt_offset & 0xfffff000; - val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; - if (obj_priv->tiling_mode == I915_TILING_Y) + val |= obj->gtt_offset & 0xfffff000; + val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; + if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I965_FENCE_TILING_Y_SHIFT; val |= I965_FENCE_REG_VALID; - I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val); + if (pipelined) { + int ret = intel_ring_begin(pipelined, 6); + if (ret) + return ret; + + intel_ring_emit(pipelined, MI_NOOP); + intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); + intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); + intel_ring_emit(pipelined, (u32)val); + intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); + intel_ring_emit(pipelined, (u32)(val >> 32)); + intel_ring_advance(pipelined); + } else + I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); + + return 0; } -static void i915_write_fence_reg(struct drm_i915_fence_reg *reg) +static int i915_write_fence_reg(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined) { - struct drm_gem_object *obj = reg->obj; - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int regnum = obj_priv->fence_reg; + u32 size = obj->gtt_space->size; + u32 fence_reg, val, pitch_val; int tile_width; - uint32_t fence_reg, val; - uint32_t pitch_val; - if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) || - (obj_priv->gtt_offset & (obj->size - 1))) { - WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n", - __func__, obj_priv->gtt_offset, obj->size); - return; - } + if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) || + (size & -size) != size || + (obj->gtt_offset & (size - 1)), + "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", + obj->gtt_offset, obj->map_and_fenceable, size)) + return -EINVAL; - if (obj_priv->tiling_mode == I915_TILING_Y && - HAS_128_BYTE_Y_TILING(dev)) + if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) tile_width = 128; else tile_width = 512; /* Note: pitch better be a power of two tile widths */ - pitch_val = obj_priv->stride / tile_width; + pitch_val = obj->stride / tile_width; pitch_val = ffs(pitch_val) - 1; - if (obj_priv->tiling_mode == I915_TILING_Y && - HAS_128_BYTE_Y_TILING(dev)) - WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL); - else - WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL); - - val = obj_priv->gtt_offset; - if (obj_priv->tiling_mode == I915_TILING_Y) + val = obj->gtt_offset; + if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; - val |= I915_FENCE_SIZE_BITS(obj->size); + val |= I915_FENCE_SIZE_BITS(size); val |= pitch_val << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; - if (regnum < 8) - fence_reg = FENCE_REG_830_0 + (regnum * 4); + fence_reg = obj->fence_reg; + if (fence_reg < 8) + fence_reg = FENCE_REG_830_0 + fence_reg * 4; else - fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4); - I915_WRITE(fence_reg, val); + fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; + + if (pipelined) { + int ret = intel_ring_begin(pipelined, 4); + if (ret) + return ret; + + intel_ring_emit(pipelined, MI_NOOP); + intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(pipelined, fence_reg); + intel_ring_emit(pipelined, val); + intel_ring_advance(pipelined); + } else + I915_WRITE(fence_reg, val); + + return 0; } -static void i830_write_fence_reg(struct drm_i915_fence_reg *reg) +static int i830_write_fence_reg(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined) { - struct drm_gem_object *obj = reg->obj; - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int regnum = obj_priv->fence_reg; + u32 size = obj->gtt_space->size; + int regnum = obj->fence_reg; uint32_t val; uint32_t pitch_val; - uint32_t fence_size_bits; - if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) || - (obj_priv->gtt_offset & (obj->size - 1))) { - WARN(1, "%s: object 0x%08x not 512K or size aligned\n", - __func__, obj_priv->gtt_offset); - return; - } + if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) || + (size & -size) != size || + (obj->gtt_offset & (size - 1)), + "object 0x%08x not 512K or pot-size 0x%08x aligned\n", + obj->gtt_offset, size)) + return -EINVAL; - pitch_val = obj_priv->stride / 128; + pitch_val = obj->stride / 128; pitch_val = ffs(pitch_val) - 1; - WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL); - val = obj_priv->gtt_offset; - if (obj_priv->tiling_mode == I915_TILING_Y) + val = obj->gtt_offset; + if (obj->tiling_mode == I915_TILING_Y) val |= 1 << I830_FENCE_TILING_Y_SHIFT; - fence_size_bits = I830_FENCE_SIZE_BITS(obj->size); - WARN_ON(fence_size_bits & ~0x00000f00); - val |= fence_size_bits; + val |= I830_FENCE_SIZE_BITS(size); val |= pitch_val << I830_FENCE_PITCH_SHIFT; val |= I830_FENCE_REG_VALID; - I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val); + if (pipelined) { + int ret = intel_ring_begin(pipelined, 4); + if (ret) + return ret; + + intel_ring_emit(pipelined, MI_NOOP); + intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); + intel_ring_emit(pipelined, val); + intel_ring_advance(pipelined); + } else + I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); + + return 0; } -static int i915_find_fence_reg(struct drm_device *dev, - bool interruptible) +static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) +{ + return i915_seqno_passed(ring->get_seqno(ring), seqno); +} + +static int +i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined, + bool interruptible) +{ + int ret; + + if (obj->fenced_gpu_access) { + if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { + ret = i915_gem_flush_ring(obj->base.dev, + obj->last_fenced_ring, + 0, obj->base.write_domain); + if (ret) + return ret; + } + + obj->fenced_gpu_access = false; + } + + if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { + if (!ring_passed_seqno(obj->last_fenced_ring, + obj->last_fenced_seqno)) { + ret = i915_do_wait_request(obj->base.dev, + obj->last_fenced_seqno, + interruptible, + obj->last_fenced_ring); + if (ret) + return ret; + } + + obj->last_fenced_seqno = 0; + obj->last_fenced_ring = NULL; + } + + /* Ensure that all CPU reads are completed before installing a fence + * and all writes before removing the fence. + */ + if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) + mb(); + + return 0; +} + +int +i915_gem_object_put_fence(struct drm_i915_gem_object *obj) +{ + int ret; + + if (obj->tiling_mode) + i915_gem_release_mmap(obj); + + ret = i915_gem_object_flush_fence(obj, NULL, true); + if (ret) + return ret; + + if (obj->fence_reg != I915_FENCE_REG_NONE) { + struct drm_i915_private *dev_priv = obj->base.dev->dev_private; + i915_gem_clear_fence_reg(obj->base.dev, + &dev_priv->fence_regs[obj->fence_reg]); + + obj->fence_reg = I915_FENCE_REG_NONE; + } + + return 0; +} + +static struct drm_i915_fence_reg * +i915_find_fence_reg(struct drm_device *dev, + struct intel_ring_buffer *pipelined) { - struct drm_i915_fence_reg *reg = NULL; - struct drm_i915_gem_object *obj_priv = NULL; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_gem_object *obj = NULL; - int i, avail, ret; + struct drm_i915_fence_reg *reg, *first, *avail; + int i; /* First try to find a free reg */ - avail = 0; + avail = NULL; for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { reg = &dev_priv->fence_regs[i]; if (!reg->obj) - return i; + return reg; - obj_priv = to_intel_bo(reg->obj); - if (!obj_priv->pin_count) - avail++; + if (!reg->obj->pin_count) + avail = reg; } - if (avail == 0) - return -ENOSPC; + if (avail == NULL) + return NULL; /* None available, try to steal one or wait for a user to finish */ - i = I915_FENCE_REG_NONE; - list_for_each_entry(reg, &dev_priv->mm.fence_list, - lru_list) { - obj = reg->obj; - obj_priv = to_intel_bo(obj); - - if (obj_priv->pin_count) + avail = first = NULL; + list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { + if (reg->obj->pin_count) continue; - /* found one! */ - i = obj_priv->fence_reg; - break; + if (first == NULL) + first = reg; + + if (!pipelined || + !reg->obj->last_fenced_ring || + reg->obj->last_fenced_ring == pipelined) { + avail = reg; + break; + } } - BUG_ON(i == I915_FENCE_REG_NONE); + if (avail == NULL) + avail = first; - /* We only have a reference on obj from the active list. put_fence_reg - * might drop that one, causing a use-after-free in it. So hold a - * private reference to obj like the other callers of put_fence_reg - * (set_tiling ioctl) do. */ - drm_gem_object_reference(obj); - ret = i915_gem_object_put_fence_reg(obj, interruptible); - drm_gem_object_unreference(obj); - if (ret != 0) - return ret; - - return i; + return avail; } /** - * i915_gem_object_get_fence_reg - set up a fence reg for an object + * i915_gem_object_get_fence - set up a fence reg for an object * @obj: object to map through a fence reg + * @pipelined: ring on which to queue the change, or NULL for CPU access + * @interruptible: must we wait uninterruptibly for the register to retire? * * When mapping objects through the GTT, userspace wants to be able to write * to them without having to worry about swizzling if the object is tiled. @@ -2448,72 +2527,141 @@ static int i915_find_fence_reg(struct drm_device *dev, * and tiling format. */ int -i915_gem_object_get_fence_reg(struct drm_gem_object *obj, - bool interruptible) +i915_gem_object_get_fence(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined, + bool interruptible) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - struct drm_i915_fence_reg *reg = NULL; + struct drm_i915_fence_reg *reg; int ret; - /* Just update our place in the LRU if our fence is getting used. */ - if (obj_priv->fence_reg != I915_FENCE_REG_NONE) { - reg = &dev_priv->fence_regs[obj_priv->fence_reg]; + /* XXX disable pipelining. There are bugs. Shocking. */ + pipelined = NULL; + + /* Just update our place in the LRU if our fence is getting reused. */ + if (obj->fence_reg != I915_FENCE_REG_NONE) { + reg = &dev_priv->fence_regs[obj->fence_reg]; list_move_tail(®->lru_list, &dev_priv->mm.fence_list); + + if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) + pipelined = NULL; + + if (!pipelined) { + if (reg->setup_seqno) { + if (!ring_passed_seqno(obj->last_fenced_ring, + reg->setup_seqno)) { + ret = i915_do_wait_request(obj->base.dev, + reg->setup_seqno, + interruptible, + obj->last_fenced_ring); + if (ret) + return ret; + } + + reg->setup_seqno = 0; + } + } else if (obj->last_fenced_ring && + obj->last_fenced_ring != pipelined) { + ret = i915_gem_object_flush_fence(obj, + pipelined, + interruptible); + if (ret) + return ret; + } else if (obj->tiling_changed) { + if (obj->fenced_gpu_access) { + if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { + ret = i915_gem_flush_ring(obj->base.dev, obj->ring, + 0, obj->base.write_domain); + if (ret) + return ret; + } + + obj->fenced_gpu_access = false; + } + } + + if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) + pipelined = NULL; + BUG_ON(!pipelined && reg->setup_seqno); + + if (obj->tiling_changed) { + if (pipelined) { + reg->setup_seqno = + i915_gem_next_request_seqno(dev, pipelined); + obj->last_fenced_seqno = reg->setup_seqno; + obj->last_fenced_ring = pipelined; + } + goto update; + } + return 0; } - switch (obj_priv->tiling_mode) { - case I915_TILING_NONE: - WARN(1, "allocating a fence for non-tiled object?\n"); - break; - case I915_TILING_X: - if (!obj_priv->stride) - return -EINVAL; - WARN((obj_priv->stride & (512 - 1)), - "object 0x%08x is X tiled but has non-512B pitch\n", - obj_priv->gtt_offset); - break; - case I915_TILING_Y: - if (!obj_priv->stride) - return -EINVAL; - WARN((obj_priv->stride & (128 - 1)), - "object 0x%08x is Y tiled but has non-128B pitch\n", - obj_priv->gtt_offset); - break; - } + reg = i915_find_fence_reg(dev, pipelined); + if (reg == NULL) + return -ENOSPC; - ret = i915_find_fence_reg(dev, interruptible); - if (ret < 0) + ret = i915_gem_object_flush_fence(obj, pipelined, interruptible); + if (ret) return ret; - obj_priv->fence_reg = ret; - reg = &dev_priv->fence_regs[obj_priv->fence_reg]; - list_add_tail(®->lru_list, &dev_priv->mm.fence_list); + if (reg->obj) { + struct drm_i915_gem_object *old = reg->obj; + + drm_gem_object_reference(&old->base); + + if (old->tiling_mode) + i915_gem_release_mmap(old); + + ret = i915_gem_object_flush_fence(old, + pipelined, + interruptible); + if (ret) { + drm_gem_object_unreference(&old->base); + return ret; + } + + if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) + pipelined = NULL; + + old->fence_reg = I915_FENCE_REG_NONE; + old->last_fenced_ring = pipelined; + old->last_fenced_seqno = + pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0; + + drm_gem_object_unreference(&old->base); + } else if (obj->last_fenced_seqno == 0) + pipelined = NULL; reg->obj = obj; + list_move_tail(®->lru_list, &dev_priv->mm.fence_list); + obj->fence_reg = reg - dev_priv->fence_regs; + obj->last_fenced_ring = pipelined; + reg->setup_seqno = + pipelined ? i915_gem_next_request_seqno(dev, pipelined) : 0; + obj->last_fenced_seqno = reg->setup_seqno; + +update: + obj->tiling_changed = false; switch (INTEL_INFO(dev)->gen) { case 6: - sandybridge_write_fence_reg(reg); + ret = sandybridge_write_fence_reg(obj, pipelined); break; case 5: case 4: - i965_write_fence_reg(reg); + ret = i965_write_fence_reg(obj, pipelined); break; case 3: - i915_write_fence_reg(reg); + ret = i915_write_fence_reg(obj, pipelined); break; case 2: - i830_write_fence_reg(reg); + ret = i830_write_fence_reg(obj, pipelined); break; } - trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg, - obj_priv->tiling_mode); - - return 0; + return ret; } /** @@ -2521,154 +2669,125 @@ i915_gem_object_get_fence_reg(struct drm_gem_object *obj, * @obj: object to clear * * Zeroes out the fence register itself and clears out the associated - * data structures in dev_priv and obj_priv. + * data structures in dev_priv and obj. */ static void -i915_gem_clear_fence_reg(struct drm_gem_object *obj) +i915_gem_clear_fence_reg(struct drm_device *dev, + struct drm_i915_fence_reg *reg) { - struct drm_device *dev = obj->dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - struct drm_i915_fence_reg *reg = - &dev_priv->fence_regs[obj_priv->fence_reg]; - uint32_t fence_reg; + uint32_t fence_reg = reg - dev_priv->fence_regs; switch (INTEL_INFO(dev)->gen) { case 6: - I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + - (obj_priv->fence_reg * 8), 0); + I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); break; case 5: case 4: - I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0); + I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); break; case 3: - if (obj_priv->fence_reg >= 8) - fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4; + if (fence_reg >= 8) + fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; else case 2: - fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4; + fence_reg = FENCE_REG_830_0 + fence_reg * 4; I915_WRITE(fence_reg, 0); break; } - reg->obj = NULL; - obj_priv->fence_reg = I915_FENCE_REG_NONE; list_del_init(®->lru_list); -} - -/** - * i915_gem_object_put_fence_reg - waits on outstanding fenced access - * to the buffer to finish, and then resets the fence register. - * @obj: tiled object holding a fence register. - * @bool: whether the wait upon the fence is interruptible - * - * Zeroes out the fence register itself and clears out the associated - * data structures in dev_priv and obj_priv. - */ -int -i915_gem_object_put_fence_reg(struct drm_gem_object *obj, - bool interruptible) -{ - struct drm_device *dev = obj->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - struct drm_i915_fence_reg *reg; - - if (obj_priv->fence_reg == I915_FENCE_REG_NONE) - return 0; - - /* If we've changed tiling, GTT-mappings of the object - * need to re-fault to ensure that the correct fence register - * setup is in place. - */ - i915_gem_release_mmap(obj); - - /* On the i915, GPU access to tiled buffers is via a fence, - * therefore we must wait for any outstanding access to complete - * before clearing the fence. - */ - reg = &dev_priv->fence_regs[obj_priv->fence_reg]; - if (reg->gpu) { - int ret; - - ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret) - return ret; - - ret = i915_gem_object_wait_rendering(obj, interruptible); - if (ret) - return ret; - - reg->gpu = false; - } - - i915_gem_object_flush_gtt_write_domain(obj); - i915_gem_clear_fence_reg(obj); - - return 0; + reg->obj = NULL; + reg->setup_seqno = 0; } /** * Finds free space in the GTT aperture and binds the object there. */ static int -i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) +i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, + unsigned alignment, + bool map_and_fenceable) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); struct drm_mm_node *free_space; - gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN; + gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN; + u32 size, fence_size, fence_alignment, unfenced_alignment; + bool mappable, fenceable; int ret; - if (obj_priv->madv != I915_MADV_WILLNEED) { + if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to bind a purgeable object\n"); return -EINVAL; } + fence_size = i915_gem_get_gtt_size(obj); + fence_alignment = i915_gem_get_gtt_alignment(obj); + unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(obj); + if (alignment == 0) - alignment = i915_gem_get_gtt_alignment(obj); - if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) { + alignment = map_and_fenceable ? fence_alignment : + unfenced_alignment; + if (map_and_fenceable && alignment & (fence_alignment - 1)) { DRM_ERROR("Invalid object alignment requested %u\n", alignment); return -EINVAL; } + size = map_and_fenceable ? fence_size : obj->base.size; + /* If the object is bigger than the entire aperture, reject it early * before evicting everything in a vain attempt to find space. */ - if (obj->size > dev_priv->mm.gtt_total) { + if (obj->base.size > + (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { DRM_ERROR("Attempting to bind an object larger than the aperture\n"); return -E2BIG; } search_free: - free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, - obj->size, alignment, 0); - if (free_space != NULL) - obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size, - alignment); - if (obj_priv->gtt_space == NULL) { + if (map_and_fenceable) + free_space = + drm_mm_search_free_in_range(&dev_priv->mm.gtt_space, + size, alignment, 0, + dev_priv->mm.gtt_mappable_end, + 0); + else + free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, + size, alignment, 0); + + if (free_space != NULL) { + if (map_and_fenceable) + obj->gtt_space = + drm_mm_get_block_range_generic(free_space, + size, alignment, 0, + dev_priv->mm.gtt_mappable_end, + 0); + else + obj->gtt_space = + drm_mm_get_block(free_space, size, alignment); + } + if (obj->gtt_space == NULL) { /* If the gtt is empty and we're still having trouble * fitting our object in, we're out of memory. */ - ret = i915_gem_evict_something(dev, obj->size, alignment); + ret = i915_gem_evict_something(dev, size, alignment, + map_and_fenceable); if (ret) return ret; goto search_free; } - ret = i915_gem_object_get_pages(obj, gfpmask); + ret = i915_gem_object_get_pages_gtt(obj, gfpmask); if (ret) { - drm_mm_put_block(obj_priv->gtt_space); - obj_priv->gtt_space = NULL; + drm_mm_put_block(obj->gtt_space); + obj->gtt_space = NULL; if (ret == -ENOMEM) { - /* first try to clear up some space from the GTT */ - ret = i915_gem_evict_something(dev, obj->size, - alignment); + /* first try to reclaim some memory by clearing the GTT */ + ret = i915_gem_evict_everything(dev, false); if (ret) { /* now try to shrink everyone else */ if (gfpmask) { @@ -2676,7 +2795,7 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) goto search_free; } - return ret; + return -ENOMEM; } goto search_free; @@ -2685,122 +2804,116 @@ i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment) return ret; } - /* Create an AGP memory structure pointing at our pages, and bind it - * into the GTT. - */ - obj_priv->agp_mem = drm_agp_bind_pages(dev, - obj_priv->pages, - obj->size >> PAGE_SHIFT, - obj_priv->gtt_space->start, - obj_priv->agp_type); - if (obj_priv->agp_mem == NULL) { - i915_gem_object_put_pages(obj); - drm_mm_put_block(obj_priv->gtt_space); - obj_priv->gtt_space = NULL; - - ret = i915_gem_evict_something(dev, obj->size, alignment); - if (ret) + ret = i915_gem_gtt_bind_object(obj); + if (ret) { + i915_gem_object_put_pages_gtt(obj); + drm_mm_put_block(obj->gtt_space); + obj->gtt_space = NULL; + + if (i915_gem_evict_everything(dev, false)) return ret; goto search_free; } - /* keep track of bounds object by adding it to the inactive list */ - list_add_tail(&obj_priv->mm_list, &dev_priv->mm.inactive_list); - i915_gem_info_add_gtt(dev_priv, obj->size); + list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); + list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); /* Assert that the object is not currently in any GPU domain. As it * wasn't in the GTT, there shouldn't be any way it could have been in * a GPU cache */ - BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS); - BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS); + BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS); + BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS); + + obj->gtt_offset = obj->gtt_space->start; + + fenceable = + obj->gtt_space->size == fence_size && + (obj->gtt_space->start & (fence_alignment -1)) == 0; - obj_priv->gtt_offset = obj_priv->gtt_space->start; - trace_i915_gem_object_bind(obj, obj_priv->gtt_offset); + mappable = + obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; + obj->map_and_fenceable = mappable && fenceable; + + trace_i915_gem_object_bind(obj, obj->gtt_offset, map_and_fenceable); return 0; } void -i915_gem_clflush_object(struct drm_gem_object *obj) +i915_gem_clflush_object(struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - /* If we don't have a page list set up, then we're not pinned * to GPU, and we can ignore the cache flush because it'll happen * again at bind time. */ - if (obj_priv->pages == NULL) + if (obj->pages == NULL) return; trace_i915_gem_object_clflush(obj); - drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE); + drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); } /** Flushes any GPU write domain for the object if it's dirty. */ static int -i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj) +i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; - uint32_t old_write_domain; + struct drm_device *dev = obj->base.dev; - if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0) + if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) return 0; /* Queue the GPU write cache flushing we need. */ - old_write_domain = obj->write_domain; - i915_gem_flush_ring(dev, NULL, - to_intel_bo(obj)->ring, - 0, obj->write_domain); - BUG_ON(obj->write_domain); - - trace_i915_gem_object_change_domain(obj, - obj->read_domains, - old_write_domain); - - return 0; + return i915_gem_flush_ring(dev, obj->ring, 0, obj->base.write_domain); } /** Flushes the GTT write domain for the object if it's dirty. */ static void -i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj) +i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) { uint32_t old_write_domain; - if (obj->write_domain != I915_GEM_DOMAIN_GTT) + if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) return; - /* No actual flushing is required for the GTT write domain. Writes + /* No actual flushing is required for the GTT write domain. Writes * to it immediately go to main memory as far as we know, so there's * no chipset flush. It also doesn't land in render cache. + * + * However, we do have to enforce the order so that all writes through + * the GTT land before any writes to the device, such as updates to + * the GATT itself. */ - old_write_domain = obj->write_domain; - obj->write_domain = 0; + wmb(); + + i915_gem_release_mmap(obj); + + old_write_domain = obj->base.write_domain; + obj->base.write_domain = 0; trace_i915_gem_object_change_domain(obj, - obj->read_domains, + obj->base.read_domains, old_write_domain); } /** Flushes the CPU write domain for the object if it's dirty. */ static void -i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) +i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; uint32_t old_write_domain; - if (obj->write_domain != I915_GEM_DOMAIN_CPU) + if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) return; i915_gem_clflush_object(obj); - drm_agp_chipset_flush(dev); - old_write_domain = obj->write_domain; - obj->write_domain = 0; + intel_gtt_chipset_flush(); + old_write_domain = obj->base.write_domain; + obj->base.write_domain = 0; trace_i915_gem_object_change_domain(obj, - obj->read_domains, + obj->base.read_domains, old_write_domain); } @@ -2811,37 +2924,39 @@ i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj) * flushes to occur. */ int -i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) +i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); uint32_t old_write_domain, old_read_domains; int ret; /* Not valid to be called on unbound objects. */ - if (obj_priv->gtt_space == NULL) + if (obj->gtt_space == NULL) return -EINVAL; ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret != 0) - return ret; - ret = i915_gem_object_wait_rendering(obj, true); if (ret) return ret; + if (obj->pending_gpu_write || write) { + ret = i915_gem_object_wait_rendering(obj, true); + if (ret) + return ret; + } + i915_gem_object_flush_cpu_write_domain(obj); - old_write_domain = obj->write_domain; - old_read_domains = obj->read_domains; + old_write_domain = obj->base.write_domain; + old_read_domains = obj->base.read_domains; /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0); - obj->read_domains |= I915_GEM_DOMAIN_GTT; + BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0); + obj->base.read_domains |= I915_GEM_DOMAIN_GTT; if (write) { - obj->read_domains = I915_GEM_DOMAIN_GTT; - obj->write_domain = I915_GEM_DOMAIN_GTT; - obj_priv->dirty = 1; + obj->base.read_domains = I915_GEM_DOMAIN_GTT; + obj->base.write_domain = I915_GEM_DOMAIN_GTT; + obj->dirty = 1; } trace_i915_gem_object_change_domain(obj, @@ -2856,23 +2971,23 @@ i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write) * wait, as in modesetting process we're not supposed to be interrupted. */ int -i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, - bool pipelined) +i915_gem_object_set_to_display_plane(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); uint32_t old_read_domains; int ret; /* Not valid to be called on unbound objects. */ - if (obj_priv->gtt_space == NULL) + if (obj->gtt_space == NULL) return -EINVAL; ret = i915_gem_object_flush_gpu_write_domain(obj); if (ret) return ret; + /* Currently, we are always called from an non-interruptible context. */ - if (!pipelined) { + if (pipelined != obj->ring) { ret = i915_gem_object_wait_rendering(obj, false); if (ret) return ret; @@ -2880,12 +2995,12 @@ i915_gem_object_set_to_display_plane(struct drm_gem_object *obj, i915_gem_object_flush_cpu_write_domain(obj); - old_read_domains = obj->read_domains; - obj->read_domains |= I915_GEM_DOMAIN_GTT; + old_read_domains = obj->base.read_domains; + obj->base.read_domains |= I915_GEM_DOMAIN_GTT; trace_i915_gem_object_change_domain(obj, old_read_domains, - obj->write_domain); + obj->base.write_domain); return 0; } @@ -2894,14 +3009,19 @@ int i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj, bool interruptible) { + int ret; + if (!obj->active) return 0; - if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) - i915_gem_flush_ring(obj->base.dev, NULL, obj->ring, - 0, obj->base.write_domain); + if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { + ret = i915_gem_flush_ring(obj->base.dev, obj->ring, + 0, obj->base.write_domain); + if (ret) + return ret; + } - return i915_gem_object_wait_rendering(&obj->base, interruptible); + return i915_gem_object_wait_rendering(obj, interruptible); } /** @@ -2911,14 +3031,15 @@ i915_gem_object_flush_gpu(struct drm_i915_gem_object *obj, * flushes to occur. */ static int -i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) +i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) { uint32_t old_write_domain, old_read_domains; int ret; ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret != 0) + if (ret) return ret; + ret = i915_gem_object_wait_rendering(obj, true); if (ret) return ret; @@ -2930,27 +3051,27 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) */ i915_gem_object_set_to_full_cpu_read_domain(obj); - old_write_domain = obj->write_domain; - old_read_domains = obj->read_domains; + old_write_domain = obj->base.write_domain; + old_read_domains = obj->base.read_domains; /* Flush the CPU cache if it's still invalid. */ - if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) { + if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { i915_gem_clflush_object(obj); - obj->read_domains |= I915_GEM_DOMAIN_CPU; + obj->base.read_domains |= I915_GEM_DOMAIN_CPU; } /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); + BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); /* If we're writing through the CPU, then the GPU read domains will * need to be invalidated at next use. */ if (write) { - obj->read_domains = I915_GEM_DOMAIN_CPU; - obj->write_domain = I915_GEM_DOMAIN_CPU; + obj->base.read_domains = I915_GEM_DOMAIN_CPU; + obj->base.write_domain = I915_GEM_DOMAIN_CPU; } trace_i915_gem_object_change_domain(obj, @@ -2960,184 +3081,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write) return 0; } -/* - * Set the next domain for the specified object. This - * may not actually perform the necessary flushing/invaliding though, - * as that may want to be batched with other set_domain operations - * - * This is (we hope) the only really tricky part of gem. The goal - * is fairly simple -- track which caches hold bits of the object - * and make sure they remain coherent. A few concrete examples may - * help to explain how it works. For shorthand, we use the notation - * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the - * a pair of read and write domain masks. - * - * Case 1: the batch buffer - * - * 1. Allocated - * 2. Written by CPU - * 3. Mapped to GTT - * 4. Read by GPU - * 5. Unmapped from GTT - * 6. Freed - * - * Let's take these a step at a time - * - * 1. Allocated - * Pages allocated from the kernel may still have - * cache contents, so we set them to (CPU, CPU) always. - * 2. Written by CPU (using pwrite) - * The pwrite function calls set_domain (CPU, CPU) and - * this function does nothing (as nothing changes) - * 3. Mapped by GTT - * This function asserts that the object is not - * currently in any GPU-based read or write domains - * 4. Read by GPU - * i915_gem_execbuffer calls set_domain (COMMAND, 0). - * As write_domain is zero, this function adds in the - * current read domains (CPU+COMMAND, 0). - * flush_domains is set to CPU. - * invalidate_domains is set to COMMAND - * clflush is run to get data out of the CPU caches - * then i915_dev_set_domain calls i915_gem_flush to - * emit an MI_FLUSH and drm_agp_chipset_flush - * 5. Unmapped from GTT - * i915_gem_object_unbind calls set_domain (CPU, CPU) - * flush_domains and invalidate_domains end up both zero - * so no flushing/invalidating happens - * 6. Freed - * yay, done - * - * Case 2: The shared render buffer - * - * 1. Allocated - * 2. Mapped to GTT - * 3. Read/written by GPU - * 4. set_domain to (CPU,CPU) - * 5. Read/written by CPU - * 6. Read/written by GPU - * - * 1. Allocated - * Same as last example, (CPU, CPU) - * 2. Mapped to GTT - * Nothing changes (assertions find that it is not in the GPU) - * 3. Read/written by GPU - * execbuffer calls set_domain (RENDER, RENDER) - * flush_domains gets CPU - * invalidate_domains gets GPU - * clflush (obj) - * MI_FLUSH and drm_agp_chipset_flush - * 4. set_domain (CPU, CPU) - * flush_domains gets GPU - * invalidate_domains gets CPU - * wait_rendering (obj) to make sure all drawing is complete. - * This will include an MI_FLUSH to get the data from GPU - * to memory - * clflush (obj) to invalidate the CPU cache - * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) - * 5. Read/written by CPU - * cache lines are loaded and dirtied - * 6. Read written by GPU - * Same as last GPU access - * - * Case 3: The constant buffer - * - * 1. Allocated - * 2. Written by CPU - * 3. Read by GPU - * 4. Updated (written) by CPU again - * 5. Read by GPU - * - * 1. Allocated - * (CPU, CPU) - * 2. Written by CPU - * (CPU, CPU) - * 3. Read by GPU - * (CPU+RENDER, 0) - * flush_domains = CPU - * invalidate_domains = RENDER - * clflush (obj) - * MI_FLUSH - * drm_agp_chipset_flush - * 4. Updated (written) by CPU again - * (CPU, CPU) - * flush_domains = 0 (no previous write domain) - * invalidate_domains = 0 (no new read domains) - * 5. Read by GPU - * (CPU+RENDER, 0) - * flush_domains = CPU - * invalidate_domains = RENDER - * clflush (obj) - * MI_FLUSH - * drm_agp_chipset_flush - */ -static void -i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj, - struct intel_ring_buffer *ring) -{ - struct drm_device *dev = obj->dev; - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - uint32_t invalidate_domains = 0; - uint32_t flush_domains = 0; - uint32_t old_read_domains; - - intel_mark_busy(dev, obj); - - /* - * If the object isn't moving to a new write domain, - * let the object stay in multiple read domains - */ - if (obj->pending_write_domain == 0) - obj->pending_read_domains |= obj->read_domains; - else - obj_priv->dirty = 1; - - /* - * Flush the current write domain if - * the new read domains don't match. Invalidate - * any read domains which differ from the old - * write domain - */ - if (obj->write_domain && - (obj->write_domain != obj->pending_read_domains || - obj_priv->ring != ring)) { - flush_domains |= obj->write_domain; - invalidate_domains |= - obj->pending_read_domains & ~obj->write_domain; - } - /* - * Invalidate any read caches which may have - * stale data. That is, any new read domains. - */ - invalidate_domains |= obj->pending_read_domains & ~obj->read_domains; - if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) - i915_gem_clflush_object(obj); - - old_read_domains = obj->read_domains; - - /* The actual obj->write_domain will be updated with - * pending_write_domain after we emit the accumulated flush for all - * of our domain changes in execbuffers (which clears objects' - * write_domains). So if we have a current write domain that we - * aren't changing, set pending_write_domain to that. - */ - if (flush_domains == 0 && obj->pending_write_domain == 0) - obj->pending_write_domain = obj->write_domain; - obj->read_domains = obj->pending_read_domains; - - dev->invalidate_domains |= invalidate_domains; - dev->flush_domains |= flush_domains; - if (flush_domains & I915_GEM_GPU_DOMAINS) - dev_priv->mm.flush_rings |= obj_priv->ring->id; - if (invalidate_domains & I915_GEM_GPU_DOMAINS) - dev_priv->mm.flush_rings |= ring->id; - - trace_i915_gem_object_change_domain(obj, - old_read_domains, - obj->write_domain); -} - /** * Moves the object from a partially CPU read to a full one. * @@ -3145,30 +3088,28 @@ i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj, * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU). */ static void -i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) +i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - - if (!obj_priv->page_cpu_valid) + if (!obj->page_cpu_valid) return; /* If we're partially in the CPU read domain, finish moving it in. */ - if (obj->read_domains & I915_GEM_DOMAIN_CPU) { + if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) { int i; - for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) { - if (obj_priv->page_cpu_valid[i]) + for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { + if (obj->page_cpu_valid[i]) continue; - drm_clflush_pages(obj_priv->pages + i, 1); + drm_clflush_pages(obj->pages + i, 1); } } /* Free the page_cpu_valid mappings which are now stale, whether * or not we've got I915_GEM_DOMAIN_CPU. */ - kfree(obj_priv->page_cpu_valid); - obj_priv->page_cpu_valid = NULL; + kfree(obj->page_cpu_valid); + obj->page_cpu_valid = NULL; } /** @@ -3184,19 +3125,19 @@ i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj) * flushes to occur. */ static int -i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, +i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); uint32_t old_read_domains; int i, ret; - if (offset == 0 && size == obj->size) + if (offset == 0 && size == obj->base.size) return i915_gem_object_set_to_cpu_domain(obj, 0); ret = i915_gem_object_flush_gpu_write_domain(obj); - if (ret != 0) + if (ret) return ret; + ret = i915_gem_object_wait_rendering(obj, true); if (ret) return ret; @@ -3204,457 +3145,45 @@ i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj, i915_gem_object_flush_gtt_write_domain(obj); /* If we're already fully in the CPU read domain, we're done. */ - if (obj_priv->page_cpu_valid == NULL && - (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0) + if (obj->page_cpu_valid == NULL && + (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) return 0; /* Otherwise, create/clear the per-page CPU read domain flag if we're * newly adding I915_GEM_DOMAIN_CPU */ - if (obj_priv->page_cpu_valid == NULL) { - obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE, - GFP_KERNEL); - if (obj_priv->page_cpu_valid == NULL) + if (obj->page_cpu_valid == NULL) { + obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE, + GFP_KERNEL); + if (obj->page_cpu_valid == NULL) return -ENOMEM; - } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) - memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE); + } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) + memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); /* Flush the cache on any pages that are still invalid from the CPU's * perspective. */ for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) { - if (obj_priv->page_cpu_valid[i]) + if (obj->page_cpu_valid[i]) continue; - drm_clflush_pages(obj_priv->pages + i, 1); + drm_clflush_pages(obj->pages + i, 1); - obj_priv->page_cpu_valid[i] = 1; + obj->page_cpu_valid[i] = 1; } /* It should now be out of any other write domains, and we can update * the domain values for our changes. */ - BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0); + BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0); - old_read_domains = obj->read_domains; - obj->read_domains |= I915_GEM_DOMAIN_CPU; + old_read_domains = obj->base.read_domains; + obj->base.read_domains |= I915_GEM_DOMAIN_CPU; trace_i915_gem_object_change_domain(obj, old_read_domains, - obj->write_domain); - - return 0; -} - -static int -i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, - struct drm_file *file_priv, - struct drm_i915_gem_exec_object2 *entry, - struct drm_i915_gem_relocation_entry *reloc) -{ - struct drm_device *dev = obj->base.dev; - struct drm_gem_object *target_obj; - uint32_t target_offset; - int ret = -EINVAL; - - target_obj = drm_gem_object_lookup(dev, file_priv, - reloc->target_handle); - if (target_obj == NULL) - return -ENOENT; - - target_offset = to_intel_bo(target_obj)->gtt_offset; - -#if WATCH_RELOC - DRM_INFO("%s: obj %p offset %08x target %d " - "read %08x write %08x gtt %08x " - "presumed %08x delta %08x\n", - __func__, - obj, - (int) reloc->offset, - (int) reloc->target_handle, - (int) reloc->read_domains, - (int) reloc->write_domain, - (int) target_offset, - (int) reloc->presumed_offset, - reloc->delta); -#endif - - /* The target buffer should have appeared before us in the - * exec_object list, so it should have a GTT space bound by now. - */ - if (target_offset == 0) { - DRM_ERROR("No GTT space found for object %d\n", - reloc->target_handle); - goto err; - } - - /* Validate that the target is in a valid r/w GPU domain */ - if (reloc->write_domain & (reloc->write_domain - 1)) { - DRM_ERROR("reloc with multiple write domains: " - "obj %p target %d offset %d " - "read %08x write %08x", - obj, reloc->target_handle, - (int) reloc->offset, - reloc->read_domains, - reloc->write_domain); - goto err; - } - if (reloc->write_domain & I915_GEM_DOMAIN_CPU || - reloc->read_domains & I915_GEM_DOMAIN_CPU) { - DRM_ERROR("reloc with read/write CPU domains: " - "obj %p target %d offset %d " - "read %08x write %08x", - obj, reloc->target_handle, - (int) reloc->offset, - reloc->read_domains, - reloc->write_domain); - goto err; - } - if (reloc->write_domain && target_obj->pending_write_domain && - reloc->write_domain != target_obj->pending_write_domain) { - DRM_ERROR("Write domain conflict: " - "obj %p target %d offset %d " - "new %08x old %08x\n", - obj, reloc->target_handle, - (int) reloc->offset, - reloc->write_domain, - target_obj->pending_write_domain); - goto err; - } - - target_obj->pending_read_domains |= reloc->read_domains; - target_obj->pending_write_domain |= reloc->write_domain; - - /* If the relocation already has the right value in it, no - * more work needs to be done. - */ - if (target_offset == reloc->presumed_offset) - goto out; - - /* Check that the relocation address is valid... */ - if (reloc->offset > obj->base.size - 4) { - DRM_ERROR("Relocation beyond object bounds: " - "obj %p target %d offset %d size %d.\n", - obj, reloc->target_handle, - (int) reloc->offset, - (int) obj->base.size); - goto err; - } - if (reloc->offset & 3) { - DRM_ERROR("Relocation not 4-byte aligned: " - "obj %p target %d offset %d.\n", - obj, reloc->target_handle, - (int) reloc->offset); - goto err; - } - - /* and points to somewhere within the target object. */ - if (reloc->delta >= target_obj->size) { - DRM_ERROR("Relocation beyond target object bounds: " - "obj %p target %d delta %d size %d.\n", - obj, reloc->target_handle, - (int) reloc->delta, - (int) target_obj->size); - goto err; - } - - reloc->delta += target_offset; - if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) { - uint32_t page_offset = reloc->offset & ~PAGE_MASK; - char *vaddr; - - vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]); - *(uint32_t *)(vaddr + page_offset) = reloc->delta; - kunmap_atomic(vaddr); - } else { - struct drm_i915_private *dev_priv = dev->dev_private; - uint32_t __iomem *reloc_entry; - void __iomem *reloc_page; - - ret = i915_gem_object_set_to_gtt_domain(&obj->base, 1); - if (ret) - goto err; - - /* Map the page containing the relocation we're going to perform. */ - reloc->offset += obj->gtt_offset; - reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping, - reloc->offset & PAGE_MASK); - reloc_entry = (uint32_t __iomem *) - (reloc_page + (reloc->offset & ~PAGE_MASK)); - iowrite32(reloc->delta, reloc_entry); - io_mapping_unmap_atomic(reloc_page); - } - - /* and update the user's relocation entry */ - reloc->presumed_offset = target_offset; - -out: - ret = 0; -err: - drm_gem_object_unreference(target_obj); - return ret; -} - -static int -i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, - struct drm_file *file_priv, - struct drm_i915_gem_exec_object2 *entry) -{ - struct drm_i915_gem_relocation_entry __user *user_relocs; - int i, ret; - - user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr; - for (i = 0; i < entry->relocation_count; i++) { - struct drm_i915_gem_relocation_entry reloc; - - if (__copy_from_user_inatomic(&reloc, - user_relocs+i, - sizeof(reloc))) - return -EFAULT; - - ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &reloc); - if (ret) - return ret; - - if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset, - &reloc.presumed_offset, - sizeof(reloc.presumed_offset))) - return -EFAULT; - } - - return 0; -} - -static int -i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, - struct drm_file *file_priv, - struct drm_i915_gem_exec_object2 *entry, - struct drm_i915_gem_relocation_entry *relocs) -{ - int i, ret; - - for (i = 0; i < entry->relocation_count; i++) { - ret = i915_gem_execbuffer_relocate_entry(obj, file_priv, entry, &relocs[i]); - if (ret) - return ret; - } - - return 0; -} - -static int -i915_gem_execbuffer_relocate(struct drm_device *dev, - struct drm_file *file, - struct drm_gem_object **object_list, - struct drm_i915_gem_exec_object2 *exec_list, - int count) -{ - int i, ret; - - for (i = 0; i < count; i++) { - struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]); - obj->base.pending_read_domains = 0; - obj->base.pending_write_domain = 0; - ret = i915_gem_execbuffer_relocate_object(obj, file, - &exec_list[i]); - if (ret) - return ret; - } - - return 0; -} - -static int -i915_gem_execbuffer_reserve(struct drm_device *dev, - struct drm_file *file, - struct drm_gem_object **object_list, - struct drm_i915_gem_exec_object2 *exec_list, - int count) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int ret, i, retry; - - /* attempt to pin all of the buffers into the GTT */ - for (retry = 0; retry < 2; retry++) { - ret = 0; - for (i = 0; i < count; i++) { - struct drm_i915_gem_exec_object2 *entry = &exec_list[i]; - struct drm_i915_gem_object *obj= to_intel_bo(object_list[i]); - bool need_fence = - entry->flags & EXEC_OBJECT_NEEDS_FENCE && - obj->tiling_mode != I915_TILING_NONE; - - /* Check fence reg constraints and rebind if necessary */ - if (need_fence && - !i915_gem_object_fence_offset_ok(&obj->base, - obj->tiling_mode)) { - ret = i915_gem_object_unbind(&obj->base); - if (ret) - break; - } - - ret = i915_gem_object_pin(&obj->base, entry->alignment); - if (ret) - break; - - /* - * Pre-965 chips need a fence register set up in order - * to properly handle blits to/from tiled surfaces. - */ - if (need_fence) { - ret = i915_gem_object_get_fence_reg(&obj->base, true); - if (ret) { - i915_gem_object_unpin(&obj->base); - break; - } - - dev_priv->fence_regs[obj->fence_reg].gpu = true; - } - - entry->offset = obj->gtt_offset; - } - - while (i--) - i915_gem_object_unpin(object_list[i]); - - if (ret == 0) - break; - - if (ret != -ENOSPC || retry) - return ret; - - ret = i915_gem_evict_everything(dev); - if (ret) - return ret; - } - - return 0; -} - -static int -i915_gem_execbuffer_relocate_slow(struct drm_device *dev, - struct drm_file *file, - struct drm_gem_object **object_list, - struct drm_i915_gem_exec_object2 *exec_list, - int count) -{ - struct drm_i915_gem_relocation_entry *reloc; - int i, total, ret; - - for (i = 0; i < count; i++) { - struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]); - obj->in_execbuffer = false; - } - - mutex_unlock(&dev->struct_mutex); - - total = 0; - for (i = 0; i < count; i++) - total += exec_list[i].relocation_count; - - reloc = drm_malloc_ab(total, sizeof(*reloc)); - if (reloc == NULL) { - mutex_lock(&dev->struct_mutex); - return -ENOMEM; - } - - total = 0; - for (i = 0; i < count; i++) { - struct drm_i915_gem_relocation_entry __user *user_relocs; - - user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr; - - if (copy_from_user(reloc+total, user_relocs, - exec_list[i].relocation_count * - sizeof(*reloc))) { - ret = -EFAULT; - mutex_lock(&dev->struct_mutex); - goto err; - } - - total += exec_list[i].relocation_count; - } - - ret = i915_mutex_lock_interruptible(dev); - if (ret) { - mutex_lock(&dev->struct_mutex); - goto err; - } - - ret = i915_gem_execbuffer_reserve(dev, file, - object_list, exec_list, - count); - if (ret) - goto err; - - total = 0; - for (i = 0; i < count; i++) { - struct drm_i915_gem_object *obj = to_intel_bo(object_list[i]); - obj->base.pending_read_domains = 0; - obj->base.pending_write_domain = 0; - ret = i915_gem_execbuffer_relocate_object_slow(obj, file, - &exec_list[i], - reloc + total); - if (ret) - goto err; - - total += exec_list[i].relocation_count; - } - - /* Leave the user relocations as are, this is the painfully slow path, - * and we want to avoid the complication of dropping the lock whilst - * having buffers reserved in the aperture and so causing spurious - * ENOSPC for random operations. - */ - -err: - drm_free_large(reloc); - return ret; -} - -static int -i915_gem_execbuffer_move_to_gpu(struct drm_device *dev, - struct drm_file *file, - struct intel_ring_buffer *ring, - struct drm_gem_object **objects, - int count) -{ - struct drm_i915_private *dev_priv = dev->dev_private; - int ret, i; - - /* Zero the global flush/invalidate flags. These - * will be modified as new domains are computed - * for each object - */ - dev->invalidate_domains = 0; - dev->flush_domains = 0; - dev_priv->mm.flush_rings = 0; - for (i = 0; i < count; i++) - i915_gem_object_set_to_gpu_domain(objects[i], ring); - - if (dev->invalidate_domains | dev->flush_domains) { -#if WATCH_EXEC - DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n", - __func__, - dev->invalidate_domains, - dev->flush_domains); -#endif - i915_gem_flush(dev, file, - dev->invalidate_domains, - dev->flush_domains, - dev_priv->mm.flush_rings); - } - - for (i = 0; i < count; i++) { - struct drm_i915_gem_object *obj = to_intel_bo(objects[i]); - /* XXX replace with semaphores */ - if (obj->ring && ring != obj->ring) { - ret = i915_gem_object_wait_rendering(&obj->base, true); - if (ret) - return ret; - } - } + obj->base.write_domain); return 0; } @@ -3694,599 +3223,129 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) return 0; ret = 0; - if (!i915_seqno_passed(ring->get_seqno(dev, ring), seqno)) { + if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { /* And wait for the seqno passing without holding any locks and * causing extra latency for others. This is safe as the irq * generation is designed to be run atomically and so is * lockless. */ - ring->user_irq_get(dev, ring); - ret = wait_event_interruptible(ring->irq_queue, - i915_seqno_passed(ring->get_seqno(dev, ring), seqno) - || atomic_read(&dev_priv->mm.wedged)); - ring->user_irq_put(dev, ring); - - if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) - ret = -EIO; - } - - if (ret == 0) - queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); - - return ret; -} - -static int -i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec, - uint64_t exec_offset) -{ - uint32_t exec_start, exec_len; - - exec_start = (uint32_t) exec_offset + exec->batch_start_offset; - exec_len = (uint32_t) exec->batch_len; - - if ((exec_start | exec_len) & 0x7) - return -EINVAL; - - if (!exec_start) - return -EINVAL; - - return 0; -} - -static int -validate_exec_list(struct drm_i915_gem_exec_object2 *exec, - int count) -{ - int i; - - for (i = 0; i < count; i++) { - char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr; - int length; /* limited by fault_in_pages_readable() */ - - /* First check for malicious input causing overflow */ - if (exec[i].relocation_count > - INT_MAX / sizeof(struct drm_i915_gem_relocation_entry)) - return -EINVAL; - - length = exec[i].relocation_count * - sizeof(struct drm_i915_gem_relocation_entry); - if (!access_ok(VERIFY_READ, ptr, length)) - return -EFAULT; - - /* we may also need to update the presumed offsets */ - if (!access_ok(VERIFY_WRITE, ptr, length)) - return -EFAULT; - - if (fault_in_pages_readable(ptr, length)) - return -EFAULT; - } - - return 0; -} - -static int -i915_gem_do_execbuffer(struct drm_device *dev, void *data, - struct drm_file *file, - struct drm_i915_gem_execbuffer2 *args, - struct drm_i915_gem_exec_object2 *exec_list) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object **object_list = NULL; - struct drm_gem_object *batch_obj; - struct drm_i915_gem_object *obj_priv; - struct drm_clip_rect *cliprects = NULL; - struct drm_i915_gem_request *request = NULL; - int ret, i, flips; - uint64_t exec_offset; - - struct intel_ring_buffer *ring = NULL; - - ret = i915_gem_check_is_wedged(dev); - if (ret) - return ret; - - ret = validate_exec_list(exec_list, args->buffer_count); - if (ret) - return ret; - -#if WATCH_EXEC - DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", - (int) args->buffers_ptr, args->buffer_count, args->batch_len); -#endif - switch (args->flags & I915_EXEC_RING_MASK) { - case I915_EXEC_DEFAULT: - case I915_EXEC_RENDER: - ring = &dev_priv->render_ring; - break; - case I915_EXEC_BSD: - if (!HAS_BSD(dev)) { - DRM_ERROR("execbuf with invalid ring (BSD)\n"); - return -EINVAL; - } - ring = &dev_priv->bsd_ring; - break; - case I915_EXEC_BLT: - if (!HAS_BLT(dev)) { - DRM_ERROR("execbuf with invalid ring (BLT)\n"); - return -EINVAL; - } - ring = &dev_priv->blt_ring; - break; - default: - DRM_ERROR("execbuf with unknown ring: %d\n", - (int)(args->flags & I915_EXEC_RING_MASK)); - return -EINVAL; - } - - if (args->buffer_count < 1) { - DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); - return -EINVAL; - } - object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count); - if (object_list == NULL) { - DRM_ERROR("Failed to allocate object list for %d buffers\n", - args->buffer_count); - ret = -ENOMEM; - goto pre_mutex_err; - } - - if (args->num_cliprects != 0) { - cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects), - GFP_KERNEL); - if (cliprects == NULL) { - ret = -ENOMEM; - goto pre_mutex_err; - } - - ret = copy_from_user(cliprects, - (struct drm_clip_rect __user *) - (uintptr_t) args->cliprects_ptr, - sizeof(*cliprects) * args->num_cliprects); - if (ret != 0) { - DRM_ERROR("copy %d cliprects failed: %d\n", - args->num_cliprects, ret); - ret = -EFAULT; - goto pre_mutex_err; - } - } - - request = kzalloc(sizeof(*request), GFP_KERNEL); - if (request == NULL) { - ret = -ENOMEM; - goto pre_mutex_err; - } - - ret = i915_mutex_lock_interruptible(dev); - if (ret) - goto pre_mutex_err; - - if (dev_priv->mm.suspended) { - mutex_unlock(&dev->struct_mutex); - ret = -EBUSY; - goto pre_mutex_err; - } - - /* Look up object handles */ - for (i = 0; i < args->buffer_count; i++) { - object_list[i] = drm_gem_object_lookup(dev, file, - exec_list[i].handle); - if (object_list[i] == NULL) { - DRM_ERROR("Invalid object handle %d at index %d\n", - exec_list[i].handle, i); - /* prevent error path from reading uninitialized data */ - args->buffer_count = i + 1; - ret = -ENOENT; - goto err; - } - - obj_priv = to_intel_bo(object_list[i]); - if (obj_priv->in_execbuffer) { - DRM_ERROR("Object %p appears more than once in object list\n", - object_list[i]); - /* prevent error path from reading uninitialized data */ - args->buffer_count = i + 1; - ret = -EINVAL; - goto err; - } - obj_priv->in_execbuffer = true; - } - - /* Move the objects en-masse into the GTT, evicting if necessary. */ - ret = i915_gem_execbuffer_reserve(dev, file, - object_list, exec_list, - args->buffer_count); - if (ret) - goto err; - - /* The objects are in their final locations, apply the relocations. */ - ret = i915_gem_execbuffer_relocate(dev, file, - object_list, exec_list, - args->buffer_count); - if (ret) { - if (ret == -EFAULT) { - ret = i915_gem_execbuffer_relocate_slow(dev, file, - object_list, - exec_list, - args->buffer_count); - BUG_ON(!mutex_is_locked(&dev->struct_mutex)); - } - if (ret) - goto err; - } - - /* Set the pending read domains for the batch buffer to COMMAND */ - batch_obj = object_list[args->buffer_count-1]; - if (batch_obj->pending_write_domain) { - DRM_ERROR("Attempting to use self-modifying batch buffer\n"); - ret = -EINVAL; - goto err; - } - batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND; - - /* Sanity check the batch buffer */ - exec_offset = to_intel_bo(batch_obj)->gtt_offset; - ret = i915_gem_check_execbuffer(args, exec_offset); - if (ret != 0) { - DRM_ERROR("execbuf with invalid offset/length\n"); - goto err; - } - - ret = i915_gem_execbuffer_move_to_gpu(dev, file, ring, - object_list, args->buffer_count); - if (ret) - goto err; - - for (i = 0; i < args->buffer_count; i++) { - struct drm_gem_object *obj = object_list[i]; - uint32_t old_write_domain = obj->write_domain; - obj->write_domain = obj->pending_write_domain; - trace_i915_gem_object_change_domain(obj, - obj->read_domains, - old_write_domain); - } - -#if WATCH_COHERENCY - for (i = 0; i < args->buffer_count; i++) { - i915_gem_object_check_coherency(object_list[i], - exec_list[i].handle); - } -#endif - -#if WATCH_EXEC - i915_gem_dump_object(batch_obj, - args->batch_len, - __func__, - ~0); -#endif - - /* Check for any pending flips. As we only maintain a flip queue depth - * of 1, we can simply insert a WAIT for the next display flip prior - * to executing the batch and avoid stalling the CPU. - */ - flips = 0; - for (i = 0; i < args->buffer_count; i++) { - if (object_list[i]->write_domain) - flips |= atomic_read(&to_intel_bo(object_list[i])->pending_flip); - } - if (flips) { - int plane, flip_mask; - - for (plane = 0; flips >> plane; plane++) { - if (((flips >> plane) & 1) == 0) - continue; - - if (plane) - flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; - else - flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; - - intel_ring_begin(dev, ring, 2); - intel_ring_emit(dev, ring, - MI_WAIT_FOR_EVENT | flip_mask); - intel_ring_emit(dev, ring, MI_NOOP); - intel_ring_advance(dev, ring); - } - } - - /* Exec the batchbuffer */ - ret = ring->dispatch_gem_execbuffer(dev, ring, args, - cliprects, exec_offset); - if (ret) { - DRM_ERROR("dispatch failed %d\n", ret); - goto err; - } - - /* - * Ensure that the commands in the batch buffer are - * finished before the interrupt fires - */ - i915_retire_commands(dev, ring); - - for (i = 0; i < args->buffer_count; i++) { - struct drm_gem_object *obj = object_list[i]; - - i915_gem_object_move_to_active(obj, ring); - if (obj->write_domain) - list_move_tail(&to_intel_bo(obj)->gpu_write_list, - &ring->gpu_write_list); - } - - i915_add_request(dev, file, request, ring); - request = NULL; - -err: - for (i = 0; i < args->buffer_count; i++) { - if (object_list[i]) { - obj_priv = to_intel_bo(object_list[i]); - obj_priv->in_execbuffer = false; - } - drm_gem_object_unreference(object_list[i]); - } - - mutex_unlock(&dev->struct_mutex); - -pre_mutex_err: - drm_free_large(object_list); - kfree(cliprects); - kfree(request); - - return ret; -} - -/* - * Legacy execbuffer just creates an exec2 list from the original exec object - * list array and passes it to the real function. - */ -int -i915_gem_execbuffer(struct drm_device *dev, void *data, - struct drm_file *file_priv) -{ - struct drm_i915_gem_execbuffer *args = data; - struct drm_i915_gem_execbuffer2 exec2; - struct drm_i915_gem_exec_object *exec_list = NULL; - struct drm_i915_gem_exec_object2 *exec2_list = NULL; - int ret, i; - -#if WATCH_EXEC - DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", - (int) args->buffers_ptr, args->buffer_count, args->batch_len); -#endif - - if (args->buffer_count < 1) { - DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); - return -EINVAL; - } - - /* Copy in the exec list from userland */ - exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count); - exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count); - if (exec_list == NULL || exec2_list == NULL) { - DRM_ERROR("Failed to allocate exec list for %d buffers\n", - args->buffer_count); - drm_free_large(exec_list); - drm_free_large(exec2_list); - return -ENOMEM; - } - ret = copy_from_user(exec_list, - (struct drm_i915_relocation_entry __user *) - (uintptr_t) args->buffers_ptr, - sizeof(*exec_list) * args->buffer_count); - if (ret != 0) { - DRM_ERROR("copy %d exec entries failed %d\n", - args->buffer_count, ret); - drm_free_large(exec_list); - drm_free_large(exec2_list); - return -EFAULT; - } - - for (i = 0; i < args->buffer_count; i++) { - exec2_list[i].handle = exec_list[i].handle; - exec2_list[i].relocation_count = exec_list[i].relocation_count; - exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; - exec2_list[i].alignment = exec_list[i].alignment; - exec2_list[i].offset = exec_list[i].offset; - if (INTEL_INFO(dev)->gen < 4) - exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; - else - exec2_list[i].flags = 0; - } + if (ring->irq_get(ring)) { + ret = wait_event_interruptible(ring->irq_queue, + i915_seqno_passed(ring->get_seqno(ring), seqno) + || atomic_read(&dev_priv->mm.wedged)); + ring->irq_put(ring); - exec2.buffers_ptr = args->buffers_ptr; - exec2.buffer_count = args->buffer_count; - exec2.batch_start_offset = args->batch_start_offset; - exec2.batch_len = args->batch_len; - exec2.DR1 = args->DR1; - exec2.DR4 = args->DR4; - exec2.num_cliprects = args->num_cliprects; - exec2.cliprects_ptr = args->cliprects_ptr; - exec2.flags = I915_EXEC_RENDER; - - ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list); - if (!ret) { - /* Copy the new buffer offsets back to the user's exec list. */ - for (i = 0; i < args->buffer_count; i++) - exec_list[i].offset = exec2_list[i].offset; - /* ... and back out to userspace */ - ret = copy_to_user((struct drm_i915_relocation_entry __user *) - (uintptr_t) args->buffers_ptr, - exec_list, - sizeof(*exec_list) * args->buffer_count); - if (ret) { - ret = -EFAULT; - DRM_ERROR("failed to copy %d exec entries " - "back to user (%d)\n", - args->buffer_count, ret); + if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) + ret = -EIO; } } - drm_free_large(exec_list); - drm_free_large(exec2_list); - return ret; -} - -int -i915_gem_execbuffer2(struct drm_device *dev, void *data, - struct drm_file *file_priv) -{ - struct drm_i915_gem_execbuffer2 *args = data; - struct drm_i915_gem_exec_object2 *exec2_list = NULL; - int ret; - -#if WATCH_EXEC - DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", - (int) args->buffers_ptr, args->buffer_count, args->batch_len); -#endif - - if (args->buffer_count < 1) { - DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count); - return -EINVAL; - } - - exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count); - if (exec2_list == NULL) { - DRM_ERROR("Failed to allocate exec list for %d buffers\n", - args->buffer_count); - return -ENOMEM; - } - ret = copy_from_user(exec2_list, - (struct drm_i915_relocation_entry __user *) - (uintptr_t) args->buffers_ptr, - sizeof(*exec2_list) * args->buffer_count); - if (ret != 0) { - DRM_ERROR("copy %d exec entries failed %d\n", - args->buffer_count, ret); - drm_free_large(exec2_list); - return -EFAULT; - } - - ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list); - if (!ret) { - /* Copy the new buffer offsets back to the user's exec list. */ - ret = copy_to_user((struct drm_i915_relocation_entry __user *) - (uintptr_t) args->buffers_ptr, - exec2_list, - sizeof(*exec2_list) * args->buffer_count); - if (ret) { - ret = -EFAULT; - DRM_ERROR("failed to copy %d exec entries " - "back to user (%d)\n", - args->buffer_count, ret); - } - } + if (ret == 0) + queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); - drm_free_large(exec2_list); return ret; } int -i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment) +i915_gem_object_pin(struct drm_i915_gem_object *obj, + uint32_t alignment, + bool map_and_fenceable) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int ret; - BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); + BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT); WARN_ON(i915_verify_lists(dev)); - if (obj_priv->gtt_space != NULL) { - if (alignment == 0) - alignment = i915_gem_get_gtt_alignment(obj); - if (obj_priv->gtt_offset & (alignment - 1)) { - WARN(obj_priv->pin_count, - "bo is already pinned with incorrect alignment: offset=%x, req.alignment=%x\n", - obj_priv->gtt_offset, alignment); + if (obj->gtt_space != NULL) { + if ((alignment && obj->gtt_offset & (alignment - 1)) || + (map_and_fenceable && !obj->map_and_fenceable)) { + WARN(obj->pin_count, + "bo is already pinned with incorrect alignment:" + " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," + " obj->map_and_fenceable=%d\n", + obj->gtt_offset, alignment, + map_and_fenceable, + obj->map_and_fenceable); ret = i915_gem_object_unbind(obj); if (ret) return ret; } } - if (obj_priv->gtt_space == NULL) { - ret = i915_gem_object_bind_to_gtt(obj, alignment); + if (obj->gtt_space == NULL) { + ret = i915_gem_object_bind_to_gtt(obj, alignment, + map_and_fenceable); if (ret) return ret; } - obj_priv->pin_count++; - - /* If the object is not active and not pending a flush, - * remove it from the inactive list - */ - if (obj_priv->pin_count == 1) { - i915_gem_info_add_pin(dev_priv, obj->size); - if (!obj_priv->active) - list_move_tail(&obj_priv->mm_list, + if (obj->pin_count++ == 0) { + if (!obj->active) + list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); } + obj->pin_mappable |= map_and_fenceable; WARN_ON(i915_verify_lists(dev)); return 0; } void -i915_gem_object_unpin(struct drm_gem_object *obj) +i915_gem_object_unpin(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); WARN_ON(i915_verify_lists(dev)); - obj_priv->pin_count--; - BUG_ON(obj_priv->pin_count < 0); - BUG_ON(obj_priv->gtt_space == NULL); + BUG_ON(obj->pin_count == 0); + BUG_ON(obj->gtt_space == NULL); - /* If the object is no longer pinned, and is - * neither active nor being flushed, then stick it on - * the inactive list - */ - if (obj_priv->pin_count == 0) { - if (!obj_priv->active) - list_move_tail(&obj_priv->mm_list, + if (--obj->pin_count == 0) { + if (!obj->active) + list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); - i915_gem_info_remove_pin(dev_priv, obj->size); + obj->pin_mappable = false; } WARN_ON(i915_verify_lists(dev)); } int i915_gem_pin_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_pin *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); - if (obj_priv->madv != I915_MADV_WILLNEED) { + if (obj->madv != I915_MADV_WILLNEED) { DRM_ERROR("Attempting to pin a purgeable buffer\n"); ret = -EINVAL; goto out; } - if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) { + if (obj->pin_filp != NULL && obj->pin_filp != file) { DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; } - obj_priv->user_pin_count++; - obj_priv->pin_filp = file_priv; - if (obj_priv->user_pin_count == 1) { - ret = i915_gem_object_pin(obj, args->alignment); + obj->user_pin_count++; + obj->pin_filp = file; + if (obj->user_pin_count == 1) { + ret = i915_gem_object_pin(obj, args->alignment, true); if (ret) goto out; } @@ -4295,9 +3354,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data, * as the X server doesn't manage domains yet */ i915_gem_object_flush_cpu_write_domain(obj); - args->offset = obj_priv->gtt_offset; + args->offset = obj->gtt_offset; out: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -4305,38 +3364,36 @@ unlock: int i915_gem_unpin_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_pin *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); - if (obj_priv->pin_filp != file_priv) { + if (obj->pin_filp != file) { DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", args->handle); ret = -EINVAL; goto out; } - obj_priv->user_pin_count--; - if (obj_priv->user_pin_count == 0) { - obj_priv->pin_filp = NULL; + obj->user_pin_count--; + if (obj->user_pin_count == 0) { + obj->pin_filp = NULL; i915_gem_object_unpin(obj); } out: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -4344,48 +3401,50 @@ unlock: int i915_gem_busy_ioctl(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_busy *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; ret = i915_mutex_lock_interruptible(dev); if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); /* Count all active objects as busy, even if they are currently not used * by the gpu. Users of this interface expect objects to eventually * become non-busy without any further actions, therefore emit any * necessary flushes here. */ - args->busy = obj_priv->active; + args->busy = obj->active; if (args->busy) { /* Unconditionally flush objects, even when the gpu still uses this * object. Userspace calling this function indicates that it wants to * use this buffer rather sooner than later, so issuing the required * flush earlier is beneficial. */ - if (obj->write_domain & I915_GEM_GPU_DOMAINS) { - i915_gem_flush_ring(dev, file_priv, - obj_priv->ring, - 0, obj->write_domain); - } else if (obj_priv->ring->outstanding_lazy_request) { + if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { + ret = i915_gem_flush_ring(dev, obj->ring, + 0, obj->base.write_domain); + } else if (obj->ring->outstanding_lazy_request == + obj->last_rendering_seqno) { + struct drm_i915_gem_request *request; + /* This ring is not being cleared by active usage, * so emit a request to do so. */ - u32 seqno = i915_add_request(dev, - NULL, NULL, - obj_priv->ring); - if (seqno == 0) + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (request) + ret = i915_add_request(dev, + NULL, request, + obj->ring); + else ret = -ENOMEM; } @@ -4394,12 +3453,12 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data, * are actually unmasked, and our working set ends up being * larger than required. */ - i915_gem_retire_requests_ring(dev, obj_priv->ring); + i915_gem_retire_requests_ring(dev, obj->ring); - args->busy = obj_priv->active; + args->busy = obj->active; } - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; @@ -4417,8 +3476,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { struct drm_i915_gem_madvise *args = data; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; switch (args->madv) { @@ -4433,37 +3491,36 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); if (obj == NULL) { ret = -ENOENT; goto unlock; } - obj_priv = to_intel_bo(obj); - if (obj_priv->pin_count) { + if (obj->pin_count) { ret = -EINVAL; goto out; } - if (obj_priv->madv != __I915_MADV_PURGED) - obj_priv->madv = args->madv; + if (obj->madv != __I915_MADV_PURGED) + obj->madv = args->madv; /* if the object is no longer bound, discard its backing storage */ - if (i915_gem_object_is_purgeable(obj_priv) && - obj_priv->gtt_space == NULL) + if (i915_gem_object_is_purgeable(obj) && + obj->gtt_space == NULL) i915_gem_object_truncate(obj); - args->retained = obj_priv->madv != __I915_MADV_PURGED; + args->retained = obj->madv != __I915_MADV_PURGED; out: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); unlock: mutex_unlock(&dev->struct_mutex); return ret; } -struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, - size_t size) +struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev, + size_t size) { struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_object *obj; @@ -4486,11 +3543,15 @@ struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev, obj->base.driver_private = NULL; obj->fence_reg = I915_FENCE_REG_NONE; INIT_LIST_HEAD(&obj->mm_list); + INIT_LIST_HEAD(&obj->gtt_list); INIT_LIST_HEAD(&obj->ring_list); + INIT_LIST_HEAD(&obj->exec_list); INIT_LIST_HEAD(&obj->gpu_write_list); obj->madv = I915_MADV_WILLNEED; + /* Avoid an unnecessary call to unbind on the first bind. */ + obj->map_and_fenceable = true; - return &obj->base; + return obj; } int i915_gem_init_object(struct drm_gem_object *obj) @@ -4500,42 +3561,41 @@ int i915_gem_init_object(struct drm_gem_object *obj) return 0; } -static void i915_gem_free_object_tail(struct drm_gem_object *obj) +static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int ret; ret = i915_gem_object_unbind(obj); if (ret == -ERESTARTSYS) { - list_move(&obj_priv->mm_list, + list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); return; } - if (obj_priv->mmap_offset) + if (obj->base.map_list.map) i915_gem_free_mmap_offset(obj); - drm_gem_object_release(obj); - i915_gem_info_remove_obj(dev_priv, obj->size); + drm_gem_object_release(&obj->base); + i915_gem_info_remove_obj(dev_priv, obj->base.size); - kfree(obj_priv->page_cpu_valid); - kfree(obj_priv->bit_17); - kfree(obj_priv); + kfree(obj->page_cpu_valid); + kfree(obj->bit_17); + kfree(obj); } -void i915_gem_free_object(struct drm_gem_object *obj) +void i915_gem_free_object(struct drm_gem_object *gem_obj) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_i915_gem_object *obj = to_intel_bo(gem_obj); + struct drm_device *dev = obj->base.dev; trace_i915_gem_object_destroy(obj); - while (obj_priv->pin_count > 0) + while (obj->pin_count > 0) i915_gem_object_unpin(obj); - if (obj_priv->phys_obj) + if (obj->phys_obj) i915_gem_detach_phys_object(dev, obj); i915_gem_free_object_tail(obj); @@ -4562,13 +3622,15 @@ i915_gem_idle(struct drm_device *dev) /* Under UMS, be paranoid and evict. */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) { - ret = i915_gem_evict_inactive(dev); + ret = i915_gem_evict_inactive(dev, false); if (ret) { mutex_unlock(&dev->struct_mutex); return ret; } } + i915_gem_reset_fences(dev); + /* Hack! Don't let anybody do execbuf while we don't control the chip. * We need to replace this with a semaphore, or something. * And not confound mm.suspended! @@ -4587,82 +3649,15 @@ i915_gem_idle(struct drm_device *dev) return 0; } -/* - * 965+ support PIPE_CONTROL commands, which provide finer grained control - * over cache flushing. - */ -static int -i915_gem_init_pipe_control(struct drm_device *dev) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; - int ret; - - obj = i915_gem_alloc_object(dev, 4096); - if (obj == NULL) { - DRM_ERROR("Failed to allocate seqno page\n"); - ret = -ENOMEM; - goto err; - } - obj_priv = to_intel_bo(obj); - obj_priv->agp_type = AGP_USER_CACHED_MEMORY; - - ret = i915_gem_object_pin(obj, 4096); - if (ret) - goto err_unref; - - dev_priv->seqno_gfx_addr = obj_priv->gtt_offset; - dev_priv->seqno_page = kmap(obj_priv->pages[0]); - if (dev_priv->seqno_page == NULL) - goto err_unpin; - - dev_priv->seqno_obj = obj; - memset(dev_priv->seqno_page, 0, PAGE_SIZE); - - return 0; - -err_unpin: - i915_gem_object_unpin(obj); -err_unref: - drm_gem_object_unreference(obj); -err: - return ret; -} - - -static void -i915_gem_cleanup_pipe_control(struct drm_device *dev) -{ - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; - - obj = dev_priv->seqno_obj; - obj_priv = to_intel_bo(obj); - kunmap(obj_priv->pages[0]); - i915_gem_object_unpin(obj); - drm_gem_object_unreference(obj); - dev_priv->seqno_obj = NULL; - - dev_priv->seqno_page = NULL; -} - int i915_gem_init_ringbuffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; - if (HAS_PIPE_CONTROL(dev)) { - ret = i915_gem_init_pipe_control(dev); - if (ret) - return ret; - } - ret = intel_init_render_ring_buffer(dev); if (ret) - goto cleanup_pipe_control; + return ret; if (HAS_BSD(dev)) { ret = intel_init_bsd_ring_buffer(dev); @@ -4681,12 +3676,9 @@ i915_gem_init_ringbuffer(struct drm_device *dev) return 0; cleanup_bsd_ring: - intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); + intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); cleanup_render_ring: - intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); -cleanup_pipe_control: - if (HAS_PIPE_CONTROL(dev)) - i915_gem_cleanup_pipe_control(dev); + intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); return ret; } @@ -4694,12 +3686,10 @@ void i915_gem_cleanup_ringbuffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + int i; - intel_cleanup_ring_buffer(dev, &dev_priv->render_ring); - intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring); - intel_cleanup_ring_buffer(dev, &dev_priv->blt_ring); - if (HAS_PIPE_CONTROL(dev)) - i915_gem_cleanup_pipe_control(dev); + for (i = 0; i < I915_NUM_RINGS; i++) + intel_cleanup_ring_buffer(&dev_priv->ring[i]); } int @@ -4707,7 +3697,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { drm_i915_private_t *dev_priv = dev->dev_private; - int ret; + int ret, i; if (drm_core_check_feature(dev, DRIVER_MODESET)) return 0; @@ -4727,14 +3717,12 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data, } BUG_ON(!list_empty(&dev_priv->mm.active_list)); - BUG_ON(!list_empty(&dev_priv->render_ring.active_list)); - BUG_ON(!list_empty(&dev_priv->bsd_ring.active_list)); - BUG_ON(!list_empty(&dev_priv->blt_ring.active_list)); BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); BUG_ON(!list_empty(&dev_priv->mm.inactive_list)); - BUG_ON(!list_empty(&dev_priv->render_ring.request_list)); - BUG_ON(!list_empty(&dev_priv->bsd_ring.request_list)); - BUG_ON(!list_empty(&dev_priv->blt_ring.request_list)); + for (i = 0; i < I915_NUM_RINGS; i++) { + BUG_ON(!list_empty(&dev_priv->ring[i].active_list)); + BUG_ON(!list_empty(&dev_priv->ring[i].request_list)); + } mutex_unlock(&dev->struct_mutex); ret = drm_irq_install(dev); @@ -4796,17 +3784,14 @@ i915_gem_load(struct drm_device *dev) INIT_LIST_HEAD(&dev_priv->mm.pinned_list); INIT_LIST_HEAD(&dev_priv->mm.fence_list); INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); - init_ring_lists(&dev_priv->render_ring); - init_ring_lists(&dev_priv->bsd_ring); - init_ring_lists(&dev_priv->blt_ring); + INIT_LIST_HEAD(&dev_priv->mm.gtt_list); + for (i = 0; i < I915_NUM_RINGS; i++) + init_ring_lists(&dev_priv->ring[i]); for (i = 0; i < 16; i++) INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); INIT_DELAYED_WORK(&dev_priv->mm.retire_work, i915_gem_retire_work_handler); init_completion(&dev_priv->error_completion); - spin_lock(&shrink_list_lock); - list_add(&dev_priv->mm.shrink_list, &shrink_list); - spin_unlock(&shrink_list_lock); /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ if (IS_GEN3(dev)) { @@ -4818,6 +3803,8 @@ i915_gem_load(struct drm_device *dev) } } + dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; + /* Old X drivers will take 0-2 for front, back, depth buffers */ if (!drm_core_check_feature(dev, DRIVER_MODESET)) dev_priv->fence_reg_start = 3; @@ -4849,6 +3836,10 @@ i915_gem_load(struct drm_device *dev) } i915_gem_detect_bit_6_swizzle(dev); init_waitqueue_head(&dev_priv->pending_flip_queue); + + dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink; + dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS; + register_shrinker(&dev_priv->mm.inactive_shrinker); } /* @@ -4918,47 +3909,47 @@ void i915_gem_free_all_phys_object(struct drm_device *dev) } void i915_gem_detach_phys_object(struct drm_device *dev, - struct drm_gem_object *obj) + struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj_priv; + struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; + char *vaddr; int i; - int ret; int page_count; - obj_priv = to_intel_bo(obj); - if (!obj_priv->phys_obj) + if (!obj->phys_obj) return; + vaddr = obj->phys_obj->handle->vaddr; - ret = i915_gem_object_get_pages(obj, 0); - if (ret) - goto out; - - page_count = obj->size / PAGE_SIZE; - + page_count = obj->base.size / PAGE_SIZE; for (i = 0; i < page_count; i++) { - char *dst = kmap_atomic(obj_priv->pages[i]); - char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); - - memcpy(dst, src, PAGE_SIZE); - kunmap_atomic(dst); + struct page *page = read_cache_page_gfp(mapping, i, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (!IS_ERR(page)) { + char *dst = kmap_atomic(page); + memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE); + kunmap_atomic(dst); + + drm_clflush_pages(&page, 1); + + set_page_dirty(page); + mark_page_accessed(page); + page_cache_release(page); + } } - drm_clflush_pages(obj_priv->pages, page_count); - drm_agp_chipset_flush(dev); + intel_gtt_chipset_flush(); - i915_gem_object_put_pages(obj); -out: - obj_priv->phys_obj->cur_obj = NULL; - obj_priv->phys_obj = NULL; + obj->phys_obj->cur_obj = NULL; + obj->phys_obj = NULL; } int i915_gem_attach_phys_object(struct drm_device *dev, - struct drm_gem_object *obj, + struct drm_i915_gem_object *obj, int id, int align) { + struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; int ret = 0; int page_count; int i; @@ -4966,10 +3957,8 @@ i915_gem_attach_phys_object(struct drm_device *dev, if (id > I915_MAX_PHYS_OBJECT) return -EINVAL; - obj_priv = to_intel_bo(obj); - - if (obj_priv->phys_obj) { - if (obj_priv->phys_obj->id == id) + if (obj->phys_obj) { + if (obj->phys_obj->id == id) return 0; i915_gem_detach_phys_object(dev, obj); } @@ -4977,51 +3966,50 @@ i915_gem_attach_phys_object(struct drm_device *dev, /* create a new object */ if (!dev_priv->mm.phys_objs[id - 1]) { ret = i915_gem_init_phys_object(dev, id, - obj->size, align); + obj->base.size, align); if (ret) { - DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size); - goto out; + DRM_ERROR("failed to init phys object %d size: %zu\n", + id, obj->base.size); + return ret; } } /* bind to the object */ - obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1]; - obj_priv->phys_obj->cur_obj = obj; - - ret = i915_gem_object_get_pages(obj, 0); - if (ret) { - DRM_ERROR("failed to get page list\n"); - goto out; - } + obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; + obj->phys_obj->cur_obj = obj; - page_count = obj->size / PAGE_SIZE; + page_count = obj->base.size / PAGE_SIZE; for (i = 0; i < page_count; i++) { - char *src = kmap_atomic(obj_priv->pages[i]); - char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE); + struct page *page; + char *dst, *src; + page = read_cache_page_gfp(mapping, i, + GFP_HIGHUSER | __GFP_RECLAIMABLE); + if (IS_ERR(page)) + return PTR_ERR(page); + + src = kmap_atomic(page); + dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE); memcpy(dst, src, PAGE_SIZE); kunmap_atomic(src); - } - i915_gem_object_put_pages(obj); + mark_page_accessed(page); + page_cache_release(page); + } return 0; -out: - return ret; } static int -i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, +i915_gem_phys_pwrite(struct drm_device *dev, + struct drm_i915_gem_object *obj, struct drm_i915_gem_pwrite *args, struct drm_file *file_priv) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - void *vaddr = obj_priv->phys_obj->handle->vaddr + args->offset; + void *vaddr = obj->phys_obj->handle->vaddr + args->offset; char __user *user_data = (char __user *) (uintptr_t) args->data_ptr; - DRM_DEBUG_DRIVER("vaddr %p, %lld\n", vaddr, args->size); - if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) { unsigned long unwritten; @@ -5036,7 +4024,7 @@ i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj, return -EFAULT; } - drm_agp_chipset_flush(dev); + intel_gtt_chipset_flush(); return 0; } @@ -5074,144 +4062,68 @@ i915_gpu_is_active(struct drm_device *dev) } static int -i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask) -{ - drm_i915_private_t *dev_priv, *next_dev; - struct drm_i915_gem_object *obj_priv, *next_obj; - int cnt = 0; - int would_deadlock = 1; +i915_gem_inactive_shrink(struct shrinker *shrinker, + int nr_to_scan, + gfp_t gfp_mask) +{ + struct drm_i915_private *dev_priv = + container_of(shrinker, + struct drm_i915_private, + mm.inactive_shrinker); + struct drm_device *dev = dev_priv->dev; + struct drm_i915_gem_object *obj, *next; + int cnt; + + if (!mutex_trylock(&dev->struct_mutex)) + return 0; /* "fast-path" to count number of available objects */ if (nr_to_scan == 0) { - spin_lock(&shrink_list_lock); - list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (mutex_trylock(&dev->struct_mutex)) { - list_for_each_entry(obj_priv, - &dev_priv->mm.inactive_list, - mm_list) - cnt++; - mutex_unlock(&dev->struct_mutex); - } - } - spin_unlock(&shrink_list_lock); - - return (cnt / 100) * sysctl_vfs_cache_pressure; + cnt = 0; + list_for_each_entry(obj, + &dev_priv->mm.inactive_list, + mm_list) + cnt++; + mutex_unlock(&dev->struct_mutex); + return cnt / 100 * sysctl_vfs_cache_pressure; } - spin_lock(&shrink_list_lock); - rescan: /* first scan for clean buffers */ - list_for_each_entry_safe(dev_priv, next_dev, - &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (! mutex_trylock(&dev->struct_mutex)) - continue; - - spin_unlock(&shrink_list_lock); - i915_gem_retire_requests(dev); + i915_gem_retire_requests(dev); - list_for_each_entry_safe(obj_priv, next_obj, - &dev_priv->mm.inactive_list, - mm_list) { - if (i915_gem_object_is_purgeable(obj_priv)) { - i915_gem_object_unbind(&obj_priv->base); - if (--nr_to_scan <= 0) - break; - } + list_for_each_entry_safe(obj, next, + &dev_priv->mm.inactive_list, + mm_list) { + if (i915_gem_object_is_purgeable(obj)) { + if (i915_gem_object_unbind(obj) == 0 && + --nr_to_scan == 0) + break; } - - spin_lock(&shrink_list_lock); - mutex_unlock(&dev->struct_mutex); - - would_deadlock = 0; - - if (nr_to_scan <= 0) - break; } /* second pass, evict/count anything still on the inactive list */ - list_for_each_entry_safe(dev_priv, next_dev, - &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (! mutex_trylock(&dev->struct_mutex)) - continue; - - spin_unlock(&shrink_list_lock); - - list_for_each_entry_safe(obj_priv, next_obj, - &dev_priv->mm.inactive_list, - mm_list) { - if (nr_to_scan > 0) { - i915_gem_object_unbind(&obj_priv->base); - nr_to_scan--; - } else - cnt++; - } - - spin_lock(&shrink_list_lock); - mutex_unlock(&dev->struct_mutex); - - would_deadlock = 0; + cnt = 0; + list_for_each_entry_safe(obj, next, + &dev_priv->mm.inactive_list, + mm_list) { + if (nr_to_scan && + i915_gem_object_unbind(obj) == 0) + nr_to_scan--; + else + cnt++; } - if (nr_to_scan) { - int active = 0; - + if (nr_to_scan && i915_gpu_is_active(dev)) { /* * We are desperate for pages, so as a last resort, wait * for the GPU to finish and discard whatever we can. * This has a dramatic impact to reduce the number of * OOM-killer events whilst running the GPU aggressively. */ - list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) { - struct drm_device *dev = dev_priv->dev; - - if (!mutex_trylock(&dev->struct_mutex)) - continue; - - spin_unlock(&shrink_list_lock); - - if (i915_gpu_is_active(dev)) { - i915_gpu_idle(dev); - active++; - } - - spin_lock(&shrink_list_lock); - mutex_unlock(&dev->struct_mutex); - } - - if (active) + if (i915_gpu_idle(dev) == 0) goto rescan; } - - spin_unlock(&shrink_list_lock); - - if (would_deadlock) - return -1; - else if (cnt > 0) - return (cnt / 100) * sysctl_vfs_cache_pressure; - else - return 0; -} - -static struct shrinker shrinker = { - .shrink = i915_gem_shrink, - .seeks = DEFAULT_SEEKS, -}; - -__init void -i915_gem_shrinker_init(void) -{ - register_shrinker(&shrinker); -} - -__exit void -i915_gem_shrinker_exit(void) -{ - unregister_shrinker(&shrinker); + mutex_unlock(&dev->struct_mutex); + return cnt / 100 * sysctl_vfs_cache_pressure; } diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c index 48644b840a8d..29d014c48ca2 100644 --- a/drivers/gpu/drm/i915/i915_gem_debug.c +++ b/drivers/gpu/drm/i915/i915_gem_debug.c @@ -152,13 +152,12 @@ i915_gem_dump_page(struct page *page, uint32_t start, uint32_t end, } void -i915_gem_dump_object(struct drm_gem_object *obj, int len, +i915_gem_dump_object(struct drm_i915_gem_object *obj, int len, const char *where, uint32_t mark) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int page; - DRM_INFO("%s: object at offset %08x\n", where, obj_priv->gtt_offset); + DRM_INFO("%s: object at offset %08x\n", where, obj->gtt_offset); for (page = 0; page < (len + PAGE_SIZE-1) / PAGE_SIZE; page++) { int page_len, chunk, chunk_len; @@ -170,9 +169,9 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len, chunk_len = page_len - chunk; if (chunk_len > 128) chunk_len = 128; - i915_gem_dump_page(obj_priv->pages[page], + i915_gem_dump_page(obj->pages[page], chunk, chunk + chunk_len, - obj_priv->gtt_offset + + obj->gtt_offset + page * PAGE_SIZE, mark); } @@ -182,21 +181,19 @@ i915_gem_dump_object(struct drm_gem_object *obj, int len, #if WATCH_COHERENCY void -i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle) +i915_gem_object_check_coherency(struct drm_i915_gem_object *obj, int handle) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_device *dev = obj->base.dev; int page; uint32_t *gtt_mapping; uint32_t *backing_map = NULL; int bad_count = 0; DRM_INFO("%s: checking coherency of object %p@0x%08x (%d, %zdkb):\n", - __func__, obj, obj_priv->gtt_offset, handle, + __func__, obj, obj->gtt_offset, handle, obj->size / 1024); - gtt_mapping = ioremap(dev->agp->base + obj_priv->gtt_offset, - obj->size); + gtt_mapping = ioremap(dev->agp->base + obj->gtt_offset, obj->base.size); if (gtt_mapping == NULL) { DRM_ERROR("failed to map GTT space\n"); return; @@ -205,7 +202,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle) for (page = 0; page < obj->size / PAGE_SIZE; page++) { int i; - backing_map = kmap_atomic(obj_priv->pages[page], KM_USER0); + backing_map = kmap_atomic(obj->pages[page], KM_USER0); if (backing_map == NULL) { DRM_ERROR("failed to map backing page\n"); @@ -220,7 +217,7 @@ i915_gem_object_check_coherency(struct drm_gem_object *obj, int handle) if (cpuval != gttval) { DRM_INFO("incoherent CPU vs GPU at 0x%08x: " "0x%08x vs 0x%08x\n", - (int)(obj_priv->gtt_offset + + (int)(obj->gtt_offset + page * PAGE_SIZE + i * 4), cpuval, gttval); if (bad_count++ >= 8) { diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c index d8ae7d1d0cc6..3d39005540aa 100644 --- a/drivers/gpu/drm/i915/i915_gem_evict.c +++ b/drivers/gpu/drm/i915/i915_gem_evict.c @@ -32,28 +32,36 @@ #include "i915_drm.h" static bool -mark_free(struct drm_i915_gem_object *obj_priv, - struct list_head *unwind) +mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind) { - list_add(&obj_priv->evict_list, unwind); - drm_gem_object_reference(&obj_priv->base); - return drm_mm_scan_add_block(obj_priv->gtt_space); + list_add(&obj->exec_list, unwind); + drm_gem_object_reference(&obj->base); + return drm_mm_scan_add_block(obj->gtt_space); } int -i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignment) +i915_gem_evict_something(struct drm_device *dev, int min_size, + unsigned alignment, bool mappable) { drm_i915_private_t *dev_priv = dev->dev_private; struct list_head eviction_list, unwind_list; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret = 0; i915_gem_retire_requests(dev); /* Re-check for free space after retiring requests */ - if (drm_mm_search_free(&dev_priv->mm.gtt_space, - min_size, alignment, 0)) - return 0; + if (mappable) { + if (drm_mm_search_free_in_range(&dev_priv->mm.gtt_space, + min_size, alignment, 0, + dev_priv->mm.gtt_mappable_end, + 0)) + return 0; + } else { + if (drm_mm_search_free(&dev_priv->mm.gtt_space, + min_size, alignment, 0)) + return 0; + } /* * The goal is to evict objects and amalgamate space in LRU order. @@ -79,45 +87,56 @@ i915_gem_evict_something(struct drm_device *dev, int min_size, unsigned alignmen */ INIT_LIST_HEAD(&unwind_list); - drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment); + if (mappable) + drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, min_size, + alignment, 0, + dev_priv->mm.gtt_mappable_end); + else + drm_mm_init_scan(&dev_priv->mm.gtt_space, min_size, alignment); /* First see if there is a large enough contiguous idle region... */ - list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) { - if (mark_free(obj_priv, &unwind_list)) + list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { + if (mark_free(obj, &unwind_list)) goto found; } /* Now merge in the soon-to-be-expired objects... */ - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { + list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { /* Does the object require an outstanding flush? */ - if (obj_priv->base.write_domain || obj_priv->pin_count) + if (obj->base.write_domain || obj->pin_count) continue; - if (mark_free(obj_priv, &unwind_list)) + if (mark_free(obj, &unwind_list)) goto found; } /* Finally add anything with a pending flush (in order of retirement) */ - list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) { - if (obj_priv->pin_count) + list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) { + if (obj->pin_count) continue; - if (mark_free(obj_priv, &unwind_list)) + if (mark_free(obj, &unwind_list)) goto found; } - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { - if (! obj_priv->base.write_domain || obj_priv->pin_count) + list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { + if (! obj->base.write_domain || obj->pin_count) continue; - if (mark_free(obj_priv, &unwind_list)) + if (mark_free(obj, &unwind_list)) goto found; } /* Nothing found, clean up and bail out! */ - list_for_each_entry(obj_priv, &unwind_list, evict_list) { - ret = drm_mm_scan_remove_block(obj_priv->gtt_space); + while (!list_empty(&unwind_list)) { + obj = list_first_entry(&unwind_list, + struct drm_i915_gem_object, + exec_list); + + ret = drm_mm_scan_remove_block(obj->gtt_space); BUG_ON(ret); - drm_gem_object_unreference(&obj_priv->base); + + list_del_init(&obj->exec_list); + drm_gem_object_unreference(&obj->base); } /* We expect the caller to unpin, evict all and try again, or give up. @@ -131,33 +150,34 @@ found: * temporary list. */ INIT_LIST_HEAD(&eviction_list); while (!list_empty(&unwind_list)) { - obj_priv = list_first_entry(&unwind_list, - struct drm_i915_gem_object, - evict_list); - if (drm_mm_scan_remove_block(obj_priv->gtt_space)) { - list_move(&obj_priv->evict_list, &eviction_list); + obj = list_first_entry(&unwind_list, + struct drm_i915_gem_object, + exec_list); + if (drm_mm_scan_remove_block(obj->gtt_space)) { + list_move(&obj->exec_list, &eviction_list); continue; } - list_del(&obj_priv->evict_list); - drm_gem_object_unreference(&obj_priv->base); + list_del_init(&obj->exec_list); + drm_gem_object_unreference(&obj->base); } /* Unbinding will emit any required flushes */ while (!list_empty(&eviction_list)) { - obj_priv = list_first_entry(&eviction_list, - struct drm_i915_gem_object, - evict_list); + obj = list_first_entry(&eviction_list, + struct drm_i915_gem_object, + exec_list); if (ret == 0) - ret = i915_gem_object_unbind(&obj_priv->base); - list_del(&obj_priv->evict_list); - drm_gem_object_unreference(&obj_priv->base); + ret = i915_gem_object_unbind(obj); + + list_del_init(&obj->exec_list); + drm_gem_object_unreference(&obj->base); } return ret; } int -i915_gem_evict_everything(struct drm_device *dev) +i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only) { drm_i915_private_t *dev_priv = dev->dev_private; int ret; @@ -176,36 +196,22 @@ i915_gem_evict_everything(struct drm_device *dev) BUG_ON(!list_empty(&dev_priv->mm.flushing_list)); - ret = i915_gem_evict_inactive(dev); - if (ret) - return ret; - - lists_empty = (list_empty(&dev_priv->mm.inactive_list) && - list_empty(&dev_priv->mm.flushing_list) && - list_empty(&dev_priv->mm.active_list)); - BUG_ON(!lists_empty); - - return 0; + return i915_gem_evict_inactive(dev, purgeable_only); } /** Unbinds all inactive objects. */ int -i915_gem_evict_inactive(struct drm_device *dev) +i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only) { drm_i915_private_t *dev_priv = dev->dev_private; - - while (!list_empty(&dev_priv->mm.inactive_list)) { - struct drm_gem_object *obj; - int ret; - - obj = &list_first_entry(&dev_priv->mm.inactive_list, - struct drm_i915_gem_object, - mm_list)->base; - - ret = i915_gem_object_unbind(obj); - if (ret != 0) { - DRM_ERROR("Error unbinding object: %d\n", ret); - return ret; + struct drm_i915_gem_object *obj, *next; + + list_for_each_entry_safe(obj, next, + &dev_priv->mm.inactive_list, mm_list) { + if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) { + int ret = i915_gem_object_unbind(obj); + if (ret) + return ret; } } diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c new file mode 100644 index 000000000000..d2f445e825f2 --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -0,0 +1,1377 @@ +/* + * Copyright © 2008,2010 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + * Authors: + * Eric Anholt <eric@anholt.net> + * Chris Wilson <chris@chris-wilson.co.uk> + * + */ + +#include "drmP.h" +#include "drm.h" +#include "i915_drm.h" +#include "i915_drv.h" +#include "i915_trace.h" +#include "intel_drv.h" + +struct change_domains { + uint32_t invalidate_domains; + uint32_t flush_domains; + uint32_t flush_rings; +}; + +/* + * Set the next domain for the specified object. This + * may not actually perform the necessary flushing/invaliding though, + * as that may want to be batched with other set_domain operations + * + * This is (we hope) the only really tricky part of gem. The goal + * is fairly simple -- track which caches hold bits of the object + * and make sure they remain coherent. A few concrete examples may + * help to explain how it works. For shorthand, we use the notation + * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the + * a pair of read and write domain masks. + * + * Case 1: the batch buffer + * + * 1. Allocated + * 2. Written by CPU + * 3. Mapped to GTT + * 4. Read by GPU + * 5. Unmapped from GTT + * 6. Freed + * + * Let's take these a step at a time + * + * 1. Allocated + * Pages allocated from the kernel may still have + * cache contents, so we set them to (CPU, CPU) always. + * 2. Written by CPU (using pwrite) + * The pwrite function calls set_domain (CPU, CPU) and + * this function does nothing (as nothing changes) + * 3. Mapped by GTT + * This function asserts that the object is not + * currently in any GPU-based read or write domains + * 4. Read by GPU + * i915_gem_execbuffer calls set_domain (COMMAND, 0). + * As write_domain is zero, this function adds in the + * current read domains (CPU+COMMAND, 0). + * flush_domains is set to CPU. + * invalidate_domains is set to COMMAND + * clflush is run to get data out of the CPU caches + * then i915_dev_set_domain calls i915_gem_flush to + * emit an MI_FLUSH and drm_agp_chipset_flush + * 5. Unmapped from GTT + * i915_gem_object_unbind calls set_domain (CPU, CPU) + * flush_domains and invalidate_domains end up both zero + * so no flushing/invalidating happens + * 6. Freed + * yay, done + * + * Case 2: The shared render buffer + * + * 1. Allocated + * 2. Mapped to GTT + * 3. Read/written by GPU + * 4. set_domain to (CPU,CPU) + * 5. Read/written by CPU + * 6. Read/written by GPU + * + * 1. Allocated + * Same as last example, (CPU, CPU) + * 2. Mapped to GTT + * Nothing changes (assertions find that it is not in the GPU) + * 3. Read/written by GPU + * execbuffer calls set_domain (RENDER, RENDER) + * flush_domains gets CPU + * invalidate_domains gets GPU + * clflush (obj) + * MI_FLUSH and drm_agp_chipset_flush + * 4. set_domain (CPU, CPU) + * flush_domains gets GPU + * invalidate_domains gets CPU + * wait_rendering (obj) to make sure all drawing is complete. + * This will include an MI_FLUSH to get the data from GPU + * to memory + * clflush (obj) to invalidate the CPU cache + * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?) + * 5. Read/written by CPU + * cache lines are loaded and dirtied + * 6. Read written by GPU + * Same as last GPU access + * + * Case 3: The constant buffer + * + * 1. Allocated + * 2. Written by CPU + * 3. Read by GPU + * 4. Updated (written) by CPU again + * 5. Read by GPU + * + * 1. Allocated + * (CPU, CPU) + * 2. Written by CPU + * (CPU, CPU) + * 3. Read by GPU + * (CPU+RENDER, 0) + * flush_domains = CPU + * invalidate_domains = RENDER + * clflush (obj) + * MI_FLUSH + * drm_agp_chipset_flush + * 4. Updated (written) by CPU again + * (CPU, CPU) + * flush_domains = 0 (no previous write domain) + * invalidate_domains = 0 (no new read domains) + * 5. Read by GPU + * (CPU+RENDER, 0) + * flush_domains = CPU + * invalidate_domains = RENDER + * clflush (obj) + * MI_FLUSH + * drm_agp_chipset_flush + */ +static void +i915_gem_object_set_to_gpu_domain(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *ring, + struct change_domains *cd) +{ + uint32_t invalidate_domains = 0, flush_domains = 0; + + /* + * If the object isn't moving to a new write domain, + * let the object stay in multiple read domains + */ + if (obj->base.pending_write_domain == 0) + obj->base.pending_read_domains |= obj->base.read_domains; + + /* + * Flush the current write domain if + * the new read domains don't match. Invalidate + * any read domains which differ from the old + * write domain + */ + if (obj->base.write_domain && + (((obj->base.write_domain != obj->base.pending_read_domains || + obj->ring != ring)) || + (obj->fenced_gpu_access && !obj->pending_fenced_gpu_access))) { + flush_domains |= obj->base.write_domain; + invalidate_domains |= + obj->base.pending_read_domains & ~obj->base.write_domain; + } + /* + * Invalidate any read caches which may have + * stale data. That is, any new read domains. + */ + invalidate_domains |= obj->base.pending_read_domains & ~obj->base.read_domains; + if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) + i915_gem_clflush_object(obj); + + /* blow away mappings if mapped through GTT */ + if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_GTT) + i915_gem_release_mmap(obj); + + /* The actual obj->write_domain will be updated with + * pending_write_domain after we emit the accumulated flush for all + * of our domain changes in execbuffers (which clears objects' + * write_domains). So if we have a current write domain that we + * aren't changing, set pending_write_domain to that. + */ + if (flush_domains == 0 && obj->base.pending_write_domain == 0) + obj->base.pending_write_domain = obj->base.write_domain; + + cd->invalidate_domains |= invalidate_domains; + cd->flush_domains |= flush_domains; + if (flush_domains & I915_GEM_GPU_DOMAINS) + cd->flush_rings |= obj->ring->id; + if (invalidate_domains & I915_GEM_GPU_DOMAINS) + cd->flush_rings |= ring->id; +} + +struct eb_objects { + int and; + struct hlist_head buckets[0]; +}; + +static struct eb_objects * +eb_create(int size) +{ + struct eb_objects *eb; + int count = PAGE_SIZE / sizeof(struct hlist_head) / 2; + while (count > size) + count >>= 1; + eb = kzalloc(count*sizeof(struct hlist_head) + + sizeof(struct eb_objects), + GFP_KERNEL); + if (eb == NULL) + return eb; + + eb->and = count - 1; + return eb; +} + +static void +eb_reset(struct eb_objects *eb) +{ + memset(eb->buckets, 0, (eb->and+1)*sizeof(struct hlist_head)); +} + +static void +eb_add_object(struct eb_objects *eb, struct drm_i915_gem_object *obj) +{ + hlist_add_head(&obj->exec_node, + &eb->buckets[obj->exec_handle & eb->and]); +} + +static struct drm_i915_gem_object * +eb_get_object(struct eb_objects *eb, unsigned long handle) +{ + struct hlist_head *head; + struct hlist_node *node; + struct drm_i915_gem_object *obj; + + head = &eb->buckets[handle & eb->and]; + hlist_for_each(node, head) { + obj = hlist_entry(node, struct drm_i915_gem_object, exec_node); + if (obj->exec_handle == handle) + return obj; + } + + return NULL; +} + +static void +eb_destroy(struct eb_objects *eb) +{ + kfree(eb); +} + +static int +i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj, + struct eb_objects *eb, + struct drm_i915_gem_relocation_entry *reloc) +{ + struct drm_device *dev = obj->base.dev; + struct drm_gem_object *target_obj; + uint32_t target_offset; + int ret = -EINVAL; + + /* we've already hold a reference to all valid objects */ + target_obj = &eb_get_object(eb, reloc->target_handle)->base; + if (unlikely(target_obj == NULL)) + return -ENOENT; + + target_offset = to_intel_bo(target_obj)->gtt_offset; + +#if WATCH_RELOC + DRM_INFO("%s: obj %p offset %08x target %d " + "read %08x write %08x gtt %08x " + "presumed %08x delta %08x\n", + __func__, + obj, + (int) reloc->offset, + (int) reloc->target_handle, + (int) reloc->read_domains, + (int) reloc->write_domain, + (int) target_offset, + (int) reloc->presumed_offset, + reloc->delta); +#endif + + /* The target buffer should have appeared before us in the + * exec_object list, so it should have a GTT space bound by now. + */ + if (unlikely(target_offset == 0)) { + DRM_ERROR("No GTT space found for object %d\n", + reloc->target_handle); + return ret; + } + + /* Validate that the target is in a valid r/w GPU domain */ + if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { + DRM_ERROR("reloc with multiple write domains: " + "obj %p target %d offset %d " + "read %08x write %08x", + obj, reloc->target_handle, + (int) reloc->offset, + reloc->read_domains, + reloc->write_domain); + return ret; + } + if (unlikely((reloc->write_domain | reloc->read_domains) & I915_GEM_DOMAIN_CPU)) { + DRM_ERROR("reloc with read/write CPU domains: " + "obj %p target %d offset %d " + "read %08x write %08x", + obj, reloc->target_handle, + (int) reloc->offset, + reloc->read_domains, + reloc->write_domain); + return ret; + } + if (unlikely(reloc->write_domain && target_obj->pending_write_domain && + reloc->write_domain != target_obj->pending_write_domain)) { + DRM_ERROR("Write domain conflict: " + "obj %p target %d offset %d " + "new %08x old %08x\n", + obj, reloc->target_handle, + (int) reloc->offset, + reloc->write_domain, + target_obj->pending_write_domain); + return ret; + } + + target_obj->pending_read_domains |= reloc->read_domains; + target_obj->pending_write_domain |= reloc->write_domain; + + /* If the relocation already has the right value in it, no + * more work needs to be done. + */ + if (target_offset == reloc->presumed_offset) + return 0; + + /* Check that the relocation address is valid... */ + if (unlikely(reloc->offset > obj->base.size - 4)) { + DRM_ERROR("Relocation beyond object bounds: " + "obj %p target %d offset %d size %d.\n", + obj, reloc->target_handle, + (int) reloc->offset, + (int) obj->base.size); + return ret; + } + if (unlikely(reloc->offset & 3)) { + DRM_ERROR("Relocation not 4-byte aligned: " + "obj %p target %d offset %d.\n", + obj, reloc->target_handle, + (int) reloc->offset); + return ret; + } + + /* and points to somewhere within the target object. */ + if (unlikely(reloc->delta >= target_obj->size)) { + DRM_ERROR("Relocation beyond target object bounds: " + "obj %p target %d delta %d size %d.\n", + obj, reloc->target_handle, + (int) reloc->delta, + (int) target_obj->size); + return ret; + } + + reloc->delta += target_offset; + if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) { + uint32_t page_offset = reloc->offset & ~PAGE_MASK; + char *vaddr; + + vaddr = kmap_atomic(obj->pages[reloc->offset >> PAGE_SHIFT]); + *(uint32_t *)(vaddr + page_offset) = reloc->delta; + kunmap_atomic(vaddr); + } else { + struct drm_i915_private *dev_priv = dev->dev_private; + uint32_t __iomem *reloc_entry; + void __iomem *reloc_page; + + ret = i915_gem_object_set_to_gtt_domain(obj, 1); + if (ret) + return ret; + + /* Map the page containing the relocation we're going to perform. */ + reloc->offset += obj->gtt_offset; + reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping, + reloc->offset & PAGE_MASK); + reloc_entry = (uint32_t __iomem *) + (reloc_page + (reloc->offset & ~PAGE_MASK)); + iowrite32(reloc->delta, reloc_entry); + io_mapping_unmap_atomic(reloc_page); + } + + /* and update the user's relocation entry */ + reloc->presumed_offset = target_offset; + + return 0; +} + +static int +i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj, + struct eb_objects *eb) +{ + struct drm_i915_gem_relocation_entry __user *user_relocs; + struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; + int i, ret; + + user_relocs = (void __user *)(uintptr_t)entry->relocs_ptr; + for (i = 0; i < entry->relocation_count; i++) { + struct drm_i915_gem_relocation_entry reloc; + + if (__copy_from_user_inatomic(&reloc, + user_relocs+i, + sizeof(reloc))) + return -EFAULT; + + ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc); + if (ret) + return ret; + + if (__copy_to_user_inatomic(&user_relocs[i].presumed_offset, + &reloc.presumed_offset, + sizeof(reloc.presumed_offset))) + return -EFAULT; + } + + return 0; +} + +static int +i915_gem_execbuffer_relocate_object_slow(struct drm_i915_gem_object *obj, + struct eb_objects *eb, + struct drm_i915_gem_relocation_entry *relocs) +{ + const struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; + int i, ret; + + for (i = 0; i < entry->relocation_count; i++) { + ret = i915_gem_execbuffer_relocate_entry(obj, eb, &relocs[i]); + if (ret) + return ret; + } + + return 0; +} + +static int +i915_gem_execbuffer_relocate(struct drm_device *dev, + struct eb_objects *eb, + struct list_head *objects) +{ + struct drm_i915_gem_object *obj; + int ret; + + list_for_each_entry(obj, objects, exec_list) { + ret = i915_gem_execbuffer_relocate_object(obj, eb); + if (ret) + return ret; + } + + return 0; +} + +static int +i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring, + struct drm_file *file, + struct list_head *objects) +{ + struct drm_i915_gem_object *obj; + int ret, retry; + bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4; + struct list_head ordered_objects; + + INIT_LIST_HEAD(&ordered_objects); + while (!list_empty(objects)) { + struct drm_i915_gem_exec_object2 *entry; + bool need_fence, need_mappable; + + obj = list_first_entry(objects, + struct drm_i915_gem_object, + exec_list); + entry = obj->exec_entry; + + need_fence = + has_fenced_gpu_access && + entry->flags & EXEC_OBJECT_NEEDS_FENCE && + obj->tiling_mode != I915_TILING_NONE; + need_mappable = + entry->relocation_count ? true : need_fence; + + if (need_mappable) + list_move(&obj->exec_list, &ordered_objects); + else + list_move_tail(&obj->exec_list, &ordered_objects); + + obj->base.pending_read_domains = 0; + obj->base.pending_write_domain = 0; + } + list_splice(&ordered_objects, objects); + + /* Attempt to pin all of the buffers into the GTT. + * This is done in 3 phases: + * + * 1a. Unbind all objects that do not match the GTT constraints for + * the execbuffer (fenceable, mappable, alignment etc). + * 1b. Increment pin count for already bound objects. + * 2. Bind new objects. + * 3. Decrement pin count. + * + * This avoid unnecessary unbinding of later objects in order to makr + * room for the earlier objects *unless* we need to defragment. + */ + retry = 0; + do { + ret = 0; + + /* Unbind any ill-fitting objects or pin. */ + list_for_each_entry(obj, objects, exec_list) { + struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; + bool need_fence, need_mappable; + if (!obj->gtt_space) + continue; + + need_fence = + has_fenced_gpu_access && + entry->flags & EXEC_OBJECT_NEEDS_FENCE && + obj->tiling_mode != I915_TILING_NONE; + need_mappable = + entry->relocation_count ? true : need_fence; + + if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) || + (need_mappable && !obj->map_and_fenceable)) + ret = i915_gem_object_unbind(obj); + else + ret = i915_gem_object_pin(obj, + entry->alignment, + need_mappable); + if (ret) + goto err; + + entry++; + } + + /* Bind fresh objects */ + list_for_each_entry(obj, objects, exec_list) { + struct drm_i915_gem_exec_object2 *entry = obj->exec_entry; + bool need_fence; + + need_fence = + has_fenced_gpu_access && + entry->flags & EXEC_OBJECT_NEEDS_FENCE && + obj->tiling_mode != I915_TILING_NONE; + + if (!obj->gtt_space) { + bool need_mappable = + entry->relocation_count ? true : need_fence; + + ret = i915_gem_object_pin(obj, + entry->alignment, + need_mappable); + if (ret) + break; + } + + if (has_fenced_gpu_access) { + if (need_fence) { + ret = i915_gem_object_get_fence(obj, ring, 1); + if (ret) + break; + } else if (entry->flags & EXEC_OBJECT_NEEDS_FENCE && + obj->tiling_mode == I915_TILING_NONE) { + /* XXX pipelined! */ + ret = i915_gem_object_put_fence(obj); + if (ret) + break; + } + obj->pending_fenced_gpu_access = need_fence; + } + + entry->offset = obj->gtt_offset; + } + + /* Decrement pin count for bound objects */ + list_for_each_entry(obj, objects, exec_list) { + if (obj->gtt_space) + i915_gem_object_unpin(obj); + } + + if (ret != -ENOSPC || retry > 1) + return ret; + + /* First attempt, just clear anything that is purgeable. + * Second attempt, clear the entire GTT. + */ + ret = i915_gem_evict_everything(ring->dev, retry == 0); + if (ret) + return ret; + + retry++; + } while (1); + +err: + obj = list_entry(obj->exec_list.prev, + struct drm_i915_gem_object, + exec_list); + while (objects != &obj->exec_list) { + if (obj->gtt_space) + i915_gem_object_unpin(obj); + + obj = list_entry(obj->exec_list.prev, + struct drm_i915_gem_object, + exec_list); + } + + return ret; +} + +static int +i915_gem_execbuffer_relocate_slow(struct drm_device *dev, + struct drm_file *file, + struct intel_ring_buffer *ring, + struct list_head *objects, + struct eb_objects *eb, + struct drm_i915_gem_exec_object2 *exec, + int count) +{ + struct drm_i915_gem_relocation_entry *reloc; + struct drm_i915_gem_object *obj; + int *reloc_offset; + int i, total, ret; + + /* We may process another execbuffer during the unlock... */ + while (!list_empty(objects)) { + obj = list_first_entry(objects, + struct drm_i915_gem_object, + exec_list); + list_del_init(&obj->exec_list); + drm_gem_object_unreference(&obj->base); + } + + mutex_unlock(&dev->struct_mutex); + + total = 0; + for (i = 0; i < count; i++) + total += exec[i].relocation_count; + + reloc_offset = drm_malloc_ab(count, sizeof(*reloc_offset)); + reloc = drm_malloc_ab(total, sizeof(*reloc)); + if (reloc == NULL || reloc_offset == NULL) { + drm_free_large(reloc); + drm_free_large(reloc_offset); + mutex_lock(&dev->struct_mutex); + return -ENOMEM; + } + + total = 0; + for (i = 0; i < count; i++) { + struct drm_i915_gem_relocation_entry __user *user_relocs; + + user_relocs = (void __user *)(uintptr_t)exec[i].relocs_ptr; + + if (copy_from_user(reloc+total, user_relocs, + exec[i].relocation_count * sizeof(*reloc))) { + ret = -EFAULT; + mutex_lock(&dev->struct_mutex); + goto err; + } + + reloc_offset[i] = total; + total += exec[i].relocation_count; + } + + ret = i915_mutex_lock_interruptible(dev); + if (ret) { + mutex_lock(&dev->struct_mutex); + goto err; + } + + /* reacquire the objects */ + eb_reset(eb); + for (i = 0; i < count; i++) { + struct drm_i915_gem_object *obj; + + obj = to_intel_bo(drm_gem_object_lookup(dev, file, + exec[i].handle)); + if (obj == NULL) { + DRM_ERROR("Invalid object handle %d at index %d\n", + exec[i].handle, i); + ret = -ENOENT; + goto err; + } + + list_add_tail(&obj->exec_list, objects); + obj->exec_handle = exec[i].handle; + obj->exec_entry = &exec[i]; + eb_add_object(eb, obj); + } + + ret = i915_gem_execbuffer_reserve(ring, file, objects); + if (ret) + goto err; + + list_for_each_entry(obj, objects, exec_list) { + int offset = obj->exec_entry - exec; + ret = i915_gem_execbuffer_relocate_object_slow(obj, eb, + reloc + reloc_offset[offset]); + if (ret) + goto err; + } + + /* Leave the user relocations as are, this is the painfully slow path, + * and we want to avoid the complication of dropping the lock whilst + * having buffers reserved in the aperture and so causing spurious + * ENOSPC for random operations. + */ + +err: + drm_free_large(reloc); + drm_free_large(reloc_offset); + return ret; +} + +static int +i915_gem_execbuffer_flush(struct drm_device *dev, + uint32_t invalidate_domains, + uint32_t flush_domains, + uint32_t flush_rings) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + int i, ret; + + if (flush_domains & I915_GEM_DOMAIN_CPU) + intel_gtt_chipset_flush(); + + if (flush_domains & I915_GEM_DOMAIN_GTT) + wmb(); + + if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) { + for (i = 0; i < I915_NUM_RINGS; i++) + if (flush_rings & (1 << i)) { + ret = i915_gem_flush_ring(dev, + &dev_priv->ring[i], + invalidate_domains, + flush_domains); + if (ret) + return ret; + } + } + + return 0; +} + +static int +i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj, + struct intel_ring_buffer *to) +{ + struct intel_ring_buffer *from = obj->ring; + u32 seqno; + int ret, idx; + + if (from == NULL || to == from) + return 0; + + /* XXX gpu semaphores are currently causing hard hangs on SNB mobile */ + if (INTEL_INFO(obj->base.dev)->gen < 6 || IS_MOBILE(obj->base.dev)) + return i915_gem_object_wait_rendering(obj, true); + + idx = intel_ring_sync_index(from, to); + + seqno = obj->last_rendering_seqno; + if (seqno <= from->sync_seqno[idx]) + return 0; + + if (seqno == from->outstanding_lazy_request) { + struct drm_i915_gem_request *request; + + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (request == NULL) + return -ENOMEM; + + ret = i915_add_request(obj->base.dev, NULL, request, from); + if (ret) { + kfree(request); + return ret; + } + + seqno = request->seqno; + } + + from->sync_seqno[idx] = seqno; + return intel_ring_sync(to, from, seqno - 1); +} + +static int +i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring, + struct list_head *objects) +{ + struct drm_i915_gem_object *obj; + struct change_domains cd; + int ret; + + cd.invalidate_domains = 0; + cd.flush_domains = 0; + cd.flush_rings = 0; + list_for_each_entry(obj, objects, exec_list) + i915_gem_object_set_to_gpu_domain(obj, ring, &cd); + + if (cd.invalidate_domains | cd.flush_domains) { +#if WATCH_EXEC + DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n", + __func__, + cd.invalidate_domains, + cd.flush_domains); +#endif + ret = i915_gem_execbuffer_flush(ring->dev, + cd.invalidate_domains, + cd.flush_domains, + cd.flush_rings); + if (ret) + return ret; + } + + list_for_each_entry(obj, objects, exec_list) { + ret = i915_gem_execbuffer_sync_rings(obj, ring); + if (ret) + return ret; + } + + return 0; +} + +static bool +i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) +{ + return ((exec->batch_start_offset | exec->batch_len) & 0x7) == 0; +} + +static int +validate_exec_list(struct drm_i915_gem_exec_object2 *exec, + int count) +{ + int i; + + for (i = 0; i < count; i++) { + char __user *ptr = (char __user *)(uintptr_t)exec[i].relocs_ptr; + int length; /* limited by fault_in_pages_readable() */ + + /* First check for malicious input causing overflow */ + if (exec[i].relocation_count > + INT_MAX / sizeof(struct drm_i915_gem_relocation_entry)) + return -EINVAL; + + length = exec[i].relocation_count * + sizeof(struct drm_i915_gem_relocation_entry); + if (!access_ok(VERIFY_READ, ptr, length)) + return -EFAULT; + + /* we may also need to update the presumed offsets */ + if (!access_ok(VERIFY_WRITE, ptr, length)) + return -EFAULT; + + if (fault_in_pages_readable(ptr, length)) + return -EFAULT; + } + + return 0; +} + +static int +i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, + struct list_head *objects) +{ + struct drm_i915_gem_object *obj; + int flips; + + /* Check for any pending flips. As we only maintain a flip queue depth + * of 1, we can simply insert a WAIT for the next display flip prior + * to executing the batch and avoid stalling the CPU. + */ + flips = 0; + list_for_each_entry(obj, objects, exec_list) { + if (obj->base.write_domain) + flips |= atomic_read(&obj->pending_flip); + } + if (flips) { + int plane, flip_mask, ret; + + for (plane = 0; flips >> plane; plane++) { + if (((flips >> plane) & 1) == 0) + continue; + + if (plane) + flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; + else + flip_mask = MI_WAIT_FOR_PLANE_A_FLIP; + + ret = intel_ring_begin(ring, 2); + if (ret) + return ret; + + intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); + } + } + + return 0; +} + +static void +i915_gem_execbuffer_move_to_active(struct list_head *objects, + struct intel_ring_buffer *ring, + u32 seqno) +{ + struct drm_i915_gem_object *obj; + + list_for_each_entry(obj, objects, exec_list) { + obj->base.read_domains = obj->base.pending_read_domains; + obj->base.write_domain = obj->base.pending_write_domain; + obj->fenced_gpu_access = obj->pending_fenced_gpu_access; + + i915_gem_object_move_to_active(obj, ring, seqno); + if (obj->base.write_domain) { + obj->dirty = 1; + obj->pending_gpu_write = true; + list_move_tail(&obj->gpu_write_list, + &ring->gpu_write_list); + intel_mark_busy(ring->dev, obj); + } + + trace_i915_gem_object_change_domain(obj, + obj->base.read_domains, + obj->base.write_domain); + } +} + +static void +i915_gem_execbuffer_retire_commands(struct drm_device *dev, + struct drm_file *file, + struct intel_ring_buffer *ring) +{ + struct drm_i915_gem_request *request; + u32 invalidate; + + /* + * Ensure that the commands in the batch buffer are + * finished before the interrupt fires. + * + * The sampler always gets flushed on i965 (sigh). + */ + invalidate = I915_GEM_DOMAIN_COMMAND; + if (INTEL_INFO(dev)->gen >= 4) + invalidate |= I915_GEM_DOMAIN_SAMPLER; + if (ring->flush(ring, invalidate, 0)) { + i915_gem_next_request_seqno(dev, ring); + return; + } + + /* Add a breadcrumb for the completion of the batch buffer */ + request = kzalloc(sizeof(*request), GFP_KERNEL); + if (request == NULL || i915_add_request(dev, file, request, ring)) { + i915_gem_next_request_seqno(dev, ring); + kfree(request); + } +} + +static int +i915_gem_do_execbuffer(struct drm_device *dev, void *data, + struct drm_file *file, + struct drm_i915_gem_execbuffer2 *args, + struct drm_i915_gem_exec_object2 *exec) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct list_head objects; + struct eb_objects *eb; + struct drm_i915_gem_object *batch_obj; + struct drm_clip_rect *cliprects = NULL; + struct intel_ring_buffer *ring; + u32 exec_start, exec_len; + u32 seqno; + int ret, mode, i; + + if (!i915_gem_check_execbuffer(args)) { + DRM_ERROR("execbuf with invalid offset/length\n"); + return -EINVAL; + } + + ret = validate_exec_list(exec, args->buffer_count); + if (ret) + return ret; + +#if WATCH_EXEC + DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", + (int) args->buffers_ptr, args->buffer_count, args->batch_len); +#endif + switch (args->flags & I915_EXEC_RING_MASK) { + case I915_EXEC_DEFAULT: + case I915_EXEC_RENDER: + ring = &dev_priv->ring[RCS]; + break; + case I915_EXEC_BSD: + if (!HAS_BSD(dev)) { + DRM_ERROR("execbuf with invalid ring (BSD)\n"); + return -EINVAL; + } + ring = &dev_priv->ring[VCS]; + break; + case I915_EXEC_BLT: + if (!HAS_BLT(dev)) { + DRM_ERROR("execbuf with invalid ring (BLT)\n"); + return -EINVAL; + } + ring = &dev_priv->ring[BCS]; + break; + default: + DRM_ERROR("execbuf with unknown ring: %d\n", + (int)(args->flags & I915_EXEC_RING_MASK)); + return -EINVAL; + } + + mode = args->flags & I915_EXEC_CONSTANTS_MASK; + switch (mode) { + case I915_EXEC_CONSTANTS_REL_GENERAL: + case I915_EXEC_CONSTANTS_ABSOLUTE: + case I915_EXEC_CONSTANTS_REL_SURFACE: + if (ring == &dev_priv->ring[RCS] && + mode != dev_priv->relative_constants_mode) { + if (INTEL_INFO(dev)->gen < 4) + return -EINVAL; + + if (INTEL_INFO(dev)->gen > 5 && + mode == I915_EXEC_CONSTANTS_REL_SURFACE) + return -EINVAL; + + ret = intel_ring_begin(ring, 4); + if (ret) + return ret; + + intel_ring_emit(ring, MI_NOOP); + intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1)); + intel_ring_emit(ring, INSTPM); + intel_ring_emit(ring, + I915_EXEC_CONSTANTS_MASK << 16 | mode); + intel_ring_advance(ring); + + dev_priv->relative_constants_mode = mode; + } + break; + default: + DRM_ERROR("execbuf with unknown constants: %d\n", mode); + return -EINVAL; + } + + if (args->buffer_count < 1) { + DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); + return -EINVAL; + } + + if (args->num_cliprects != 0) { + if (ring != &dev_priv->ring[RCS]) { + DRM_ERROR("clip rectangles are only valid with the render ring\n"); + return -EINVAL; + } + + cliprects = kmalloc(args->num_cliprects * sizeof(*cliprects), + GFP_KERNEL); + if (cliprects == NULL) { + ret = -ENOMEM; + goto pre_mutex_err; + } + + if (copy_from_user(cliprects, + (struct drm_clip_rect __user *)(uintptr_t) + args->cliprects_ptr, + sizeof(*cliprects)*args->num_cliprects)) { + ret = -EFAULT; + goto pre_mutex_err; + } + } + + ret = i915_mutex_lock_interruptible(dev); + if (ret) + goto pre_mutex_err; + + if (dev_priv->mm.suspended) { + mutex_unlock(&dev->struct_mutex); + ret = -EBUSY; + goto pre_mutex_err; + } + + eb = eb_create(args->buffer_count); + if (eb == NULL) { + mutex_unlock(&dev->struct_mutex); + ret = -ENOMEM; + goto pre_mutex_err; + } + + /* Look up object handles */ + INIT_LIST_HEAD(&objects); + for (i = 0; i < args->buffer_count; i++) { + struct drm_i915_gem_object *obj; + + obj = to_intel_bo(drm_gem_object_lookup(dev, file, + exec[i].handle)); + if (obj == NULL) { + DRM_ERROR("Invalid object handle %d at index %d\n", + exec[i].handle, i); + /* prevent error path from reading uninitialized data */ + ret = -ENOENT; + goto err; + } + + if (!list_empty(&obj->exec_list)) { + DRM_ERROR("Object %p [handle %d, index %d] appears more than once in object list\n", + obj, exec[i].handle, i); + ret = -EINVAL; + goto err; + } + + list_add_tail(&obj->exec_list, &objects); + obj->exec_handle = exec[i].handle; + obj->exec_entry = &exec[i]; + eb_add_object(eb, obj); + } + + /* take note of the batch buffer before we might reorder the lists */ + batch_obj = list_entry(objects.prev, + struct drm_i915_gem_object, + exec_list); + + /* Move the objects en-masse into the GTT, evicting if necessary. */ + ret = i915_gem_execbuffer_reserve(ring, file, &objects); + if (ret) + goto err; + + /* The objects are in their final locations, apply the relocations. */ + ret = i915_gem_execbuffer_relocate(dev, eb, &objects); + if (ret) { + if (ret == -EFAULT) { + ret = i915_gem_execbuffer_relocate_slow(dev, file, ring, + &objects, eb, + exec, + args->buffer_count); + BUG_ON(!mutex_is_locked(&dev->struct_mutex)); + } + if (ret) + goto err; + } + + /* Set the pending read domains for the batch buffer to COMMAND */ + if (batch_obj->base.pending_write_domain) { + DRM_ERROR("Attempting to use self-modifying batch buffer\n"); + ret = -EINVAL; + goto err; + } + batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND; + + ret = i915_gem_execbuffer_move_to_gpu(ring, &objects); + if (ret) + goto err; + + ret = i915_gem_execbuffer_wait_for_flips(ring, &objects); + if (ret) + goto err; + + seqno = i915_gem_next_request_seqno(dev, ring); + for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++) { + if (seqno < ring->sync_seqno[i]) { + /* The GPU can not handle its semaphore value wrapping, + * so every billion or so execbuffers, we need to stall + * the GPU in order to reset the counters. + */ + ret = i915_gpu_idle(dev); + if (ret) + goto err; + + BUG_ON(ring->sync_seqno[i]); + } + } + + exec_start = batch_obj->gtt_offset + args->batch_start_offset; + exec_len = args->batch_len; + if (cliprects) { + for (i = 0; i < args->num_cliprects; i++) { + ret = i915_emit_box(dev, &cliprects[i], + args->DR1, args->DR4); + if (ret) + goto err; + + ret = ring->dispatch_execbuffer(ring, + exec_start, exec_len); + if (ret) + goto err; + } + } else { + ret = ring->dispatch_execbuffer(ring, exec_start, exec_len); + if (ret) + goto err; + } + + i915_gem_execbuffer_move_to_active(&objects, ring, seqno); + i915_gem_execbuffer_retire_commands(dev, file, ring); + +err: + eb_destroy(eb); + while (!list_empty(&objects)) { + struct drm_i915_gem_object *obj; + + obj = list_first_entry(&objects, + struct drm_i915_gem_object, + exec_list); + list_del_init(&obj->exec_list); + drm_gem_object_unreference(&obj->base); + } + + mutex_unlock(&dev->struct_mutex); + +pre_mutex_err: + kfree(cliprects); + return ret; +} + +/* + * Legacy execbuffer just creates an exec2 list from the original exec object + * list array and passes it to the real function. + */ +int +i915_gem_execbuffer(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_i915_gem_execbuffer *args = data; + struct drm_i915_gem_execbuffer2 exec2; + struct drm_i915_gem_exec_object *exec_list = NULL; + struct drm_i915_gem_exec_object2 *exec2_list = NULL; + int ret, i; + +#if WATCH_EXEC + DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", + (int) args->buffers_ptr, args->buffer_count, args->batch_len); +#endif + + if (args->buffer_count < 1) { + DRM_ERROR("execbuf with %d buffers\n", args->buffer_count); + return -EINVAL; + } + + /* Copy in the exec list from userland */ + exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count); + exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count); + if (exec_list == NULL || exec2_list == NULL) { + DRM_ERROR("Failed to allocate exec list for %d buffers\n", + args->buffer_count); + drm_free_large(exec_list); + drm_free_large(exec2_list); + return -ENOMEM; + } + ret = copy_from_user(exec_list, + (struct drm_i915_relocation_entry __user *) + (uintptr_t) args->buffers_ptr, + sizeof(*exec_list) * args->buffer_count); + if (ret != 0) { + DRM_ERROR("copy %d exec entries failed %d\n", + args->buffer_count, ret); + drm_free_large(exec_list); + drm_free_large(exec2_list); + return -EFAULT; + } + + for (i = 0; i < args->buffer_count; i++) { + exec2_list[i].handle = exec_list[i].handle; + exec2_list[i].relocation_count = exec_list[i].relocation_count; + exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; + exec2_list[i].alignment = exec_list[i].alignment; + exec2_list[i].offset = exec_list[i].offset; + if (INTEL_INFO(dev)->gen < 4) + exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; + else + exec2_list[i].flags = 0; + } + + exec2.buffers_ptr = args->buffers_ptr; + exec2.buffer_count = args->buffer_count; + exec2.batch_start_offset = args->batch_start_offset; + exec2.batch_len = args->batch_len; + exec2.DR1 = args->DR1; + exec2.DR4 = args->DR4; + exec2.num_cliprects = args->num_cliprects; + exec2.cliprects_ptr = args->cliprects_ptr; + exec2.flags = I915_EXEC_RENDER; + + ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list); + if (!ret) { + /* Copy the new buffer offsets back to the user's exec list. */ + for (i = 0; i < args->buffer_count; i++) + exec_list[i].offset = exec2_list[i].offset; + /* ... and back out to userspace */ + ret = copy_to_user((struct drm_i915_relocation_entry __user *) + (uintptr_t) args->buffers_ptr, + exec_list, + sizeof(*exec_list) * args->buffer_count); + if (ret) { + ret = -EFAULT; + DRM_ERROR("failed to copy %d exec entries " + "back to user (%d)\n", + args->buffer_count, ret); + } + } + + drm_free_large(exec_list); + drm_free_large(exec2_list); + return ret; +} + +int +i915_gem_execbuffer2(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct drm_i915_gem_execbuffer2 *args = data; + struct drm_i915_gem_exec_object2 *exec2_list = NULL; + int ret; + +#if WATCH_EXEC + DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n", + (int) args->buffers_ptr, args->buffer_count, args->batch_len); +#endif + + if (args->buffer_count < 1) { + DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count); + return -EINVAL; + } + + exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count); + if (exec2_list == NULL) { + DRM_ERROR("Failed to allocate exec list for %d buffers\n", + args->buffer_count); + return -ENOMEM; + } + ret = copy_from_user(exec2_list, + (struct drm_i915_relocation_entry __user *) + (uintptr_t) args->buffers_ptr, + sizeof(*exec2_list) * args->buffer_count); + if (ret != 0) { + DRM_ERROR("copy %d exec entries failed %d\n", + args->buffer_count, ret); + drm_free_large(exec2_list); + return -EFAULT; + } + + ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list); + if (!ret) { + /* Copy the new buffer offsets back to the user's exec list. */ + ret = copy_to_user((struct drm_i915_relocation_entry __user *) + (uintptr_t) args->buffers_ptr, + exec2_list, + sizeof(*exec2_list) * args->buffer_count); + if (ret) { + ret = -EFAULT; + DRM_ERROR("failed to copy %d exec entries " + "back to user (%d)\n", + args->buffer_count, ret); + } + } + + drm_free_large(exec2_list); + return ret; +} diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c new file mode 100644 index 000000000000..b0abdc64aa9f --- /dev/null +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -0,0 +1,99 @@ +/* + * Copyright © 2010 Daniel Vetter + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + * + */ + +#include "drmP.h" +#include "drm.h" +#include "i915_drm.h" +#include "i915_drv.h" +#include "i915_trace.h" +#include "intel_drv.h" + +void i915_gem_restore_gtt_mappings(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_i915_gem_object *obj; + + /* First fill our portion of the GTT with scratch pages */ + intel_gtt_clear_range(dev_priv->mm.gtt_start / PAGE_SIZE, + (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); + + list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) { + i915_gem_clflush_object(obj); + + if (dev_priv->mm.gtt->needs_dmar) { + BUG_ON(!obj->sg_list); + + intel_gtt_insert_sg_entries(obj->sg_list, + obj->num_sg, + obj->gtt_space->start + >> PAGE_SHIFT, + obj->agp_type); + } else + intel_gtt_insert_pages(obj->gtt_space->start + >> PAGE_SHIFT, + obj->base.size >> PAGE_SHIFT, + obj->pages, + obj->agp_type); + } + + intel_gtt_chipset_flush(); +} + +int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj) +{ + struct drm_device *dev = obj->base.dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + if (dev_priv->mm.gtt->needs_dmar) { + ret = intel_gtt_map_memory(obj->pages, + obj->base.size >> PAGE_SHIFT, + &obj->sg_list, + &obj->num_sg); + if (ret != 0) + return ret; + + intel_gtt_insert_sg_entries(obj->sg_list, + obj->num_sg, + obj->gtt_space->start >> PAGE_SHIFT, + obj->agp_type); + } else + intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, + obj->base.size >> PAGE_SHIFT, + obj->pages, + obj->agp_type); + + return 0; +} + +void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) +{ + intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT, + obj->base.size >> PAGE_SHIFT); + + if (obj->sg_list) { + intel_gtt_unmap_memory(obj->sg_list, obj->num_sg); + obj->sg_list = NULL; + } +} diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c index af352de70be1..79a04fde69b5 100644 --- a/drivers/gpu/drm/i915/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/i915_gem_tiling.c @@ -181,10 +181,10 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev) } /* Check pitch constriants for all chips & tiling formats */ -bool +static bool i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) { - int tile_width; + int tile_width, tile_height; /* Linear is always fine */ if (tiling_mode == I915_TILING_NONE) @@ -215,6 +215,20 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) } } + if (IS_GEN2(dev) || + (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))) + tile_height = 32; + else + tile_height = 8; + /* i8xx is strange: It has 2 interleaved rows of tiles, so needs an even + * number of tile rows. */ + if (IS_GEN2(dev)) + tile_height *= 2; + + /* Size needs to be aligned to a full tile row */ + if (size & (tile_height * stride - 1)) + return false; + /* 965+ just needs multiples of tile width */ if (INTEL_INFO(dev)->gen >= 4) { if (stride & (tile_width - 1)) @@ -232,32 +246,44 @@ i915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) return true; } -bool -i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode) +/* Is the current GTT allocation valid for the change in tiling? */ +static bool +i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode) { - struct drm_device *dev = obj->dev; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - - if (obj_priv->gtt_space == NULL) - return true; + u32 size; if (tiling_mode == I915_TILING_NONE) return true; - if (INTEL_INFO(dev)->gen >= 4) + if (INTEL_INFO(obj->base.dev)->gen >= 4) return true; - if (obj_priv->gtt_offset & (obj->size - 1)) - return false; - - if (IS_GEN3(dev)) { - if (obj_priv->gtt_offset & ~I915_FENCE_START_MASK) + if (INTEL_INFO(obj->base.dev)->gen == 3) { + if (obj->gtt_offset & ~I915_FENCE_START_MASK) return false; } else { - if (obj_priv->gtt_offset & ~I830_FENCE_START_MASK) + if (obj->gtt_offset & ~I830_FENCE_START_MASK) return false; } + /* + * Previous chips need to be aligned to the size of the smallest + * fence register that can contain the object. + */ + if (INTEL_INFO(obj->base.dev)->gen == 3) + size = 1024*1024; + else + size = 512*1024; + + while (size < obj->base.size) + size <<= 1; + + if (obj->gtt_space->size != size) + return false; + + if (obj->gtt_offset & (size - 1)) + return false; + return true; } @@ -267,30 +293,29 @@ i915_gem_object_fence_offset_ok(struct drm_gem_object *obj, int tiling_mode) */ int i915_gem_set_tiling(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_set_tiling *args = data; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; ret = i915_gem_check_is_wedged(dev); if (ret) return ret; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) return -ENOENT; - obj_priv = to_intel_bo(obj); - if (!i915_tiling_ok(dev, args->stride, obj->size, args->tiling_mode)) { - drm_gem_object_unreference_unlocked(obj); + if (!i915_tiling_ok(dev, + args->stride, obj->base.size, args->tiling_mode)) { + drm_gem_object_unreference_unlocked(&obj->base); return -EINVAL; } - if (obj_priv->pin_count) { - drm_gem_object_unreference_unlocked(obj); + if (obj->pin_count) { + drm_gem_object_unreference_unlocked(&obj->base); return -EBUSY; } @@ -324,34 +349,28 @@ i915_gem_set_tiling(struct drm_device *dev, void *data, } mutex_lock(&dev->struct_mutex); - if (args->tiling_mode != obj_priv->tiling_mode || - args->stride != obj_priv->stride) { + if (args->tiling_mode != obj->tiling_mode || + args->stride != obj->stride) { /* We need to rebind the object if its current allocation * no longer meets the alignment restrictions for its new * tiling mode. Otherwise we can just leave it alone, but * need to ensure that any fence register is cleared. */ - if (!i915_gem_object_fence_offset_ok(obj, args->tiling_mode)) - ret = i915_gem_object_unbind(obj); - else if (obj_priv->fence_reg != I915_FENCE_REG_NONE) - ret = i915_gem_object_put_fence_reg(obj, true); - else - i915_gem_release_mmap(obj); + i915_gem_release_mmap(obj); - if (ret != 0) { - args->tiling_mode = obj_priv->tiling_mode; - args->stride = obj_priv->stride; - goto err; - } + obj->map_and_fenceable = + obj->gtt_space == NULL || + (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end && + i915_gem_object_fence_ok(obj, args->tiling_mode)); - obj_priv->tiling_mode = args->tiling_mode; - obj_priv->stride = args->stride; + obj->tiling_changed = true; + obj->tiling_mode = args->tiling_mode; + obj->stride = args->stride; } -err: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); - return ret; + return 0; } /** @@ -359,22 +378,20 @@ err: */ int i915_gem_get_tiling(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { struct drm_i915_gem_get_tiling *args = data; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; - obj = drm_gem_object_lookup(dev, file_priv, args->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); if (obj == NULL) return -ENOENT; - obj_priv = to_intel_bo(obj); mutex_lock(&dev->struct_mutex); - args->tiling_mode = obj_priv->tiling_mode; - switch (obj_priv->tiling_mode) { + args->tiling_mode = obj->tiling_mode; + switch (obj->tiling_mode) { case I915_TILING_X: args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; break; @@ -394,7 +411,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data, if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); return 0; @@ -424,46 +441,44 @@ i915_gem_swizzle_page(struct page *page) } void -i915_gem_object_do_bit_17_swizzle(struct drm_gem_object *obj) +i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int page_count = obj->size >> PAGE_SHIFT; + int page_count = obj->base.size >> PAGE_SHIFT; int i; if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) return; - if (obj_priv->bit_17 == NULL) + if (obj->bit_17 == NULL) return; for (i = 0; i < page_count; i++) { - char new_bit_17 = page_to_phys(obj_priv->pages[i]) >> 17; + char new_bit_17 = page_to_phys(obj->pages[i]) >> 17; if ((new_bit_17 & 0x1) != - (test_bit(i, obj_priv->bit_17) != 0)) { - i915_gem_swizzle_page(obj_priv->pages[i]); - set_page_dirty(obj_priv->pages[i]); + (test_bit(i, obj->bit_17) != 0)) { + i915_gem_swizzle_page(obj->pages[i]); + set_page_dirty(obj->pages[i]); } } } void -i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj) +i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) { - struct drm_device *dev = obj->dev; + struct drm_device *dev = obj->base.dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); - int page_count = obj->size >> PAGE_SHIFT; + int page_count = obj->base.size >> PAGE_SHIFT; int i; if (dev_priv->mm.bit_6_swizzle_x != I915_BIT_6_SWIZZLE_9_10_17) return; - if (obj_priv->bit_17 == NULL) { - obj_priv->bit_17 = kmalloc(BITS_TO_LONGS(page_count) * + if (obj->bit_17 == NULL) { + obj->bit_17 = kmalloc(BITS_TO_LONGS(page_count) * sizeof(long), GFP_KERNEL); - if (obj_priv->bit_17 == NULL) { + if (obj->bit_17 == NULL) { DRM_ERROR("Failed to allocate memory for bit 17 " "record\n"); return; @@ -471,9 +486,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_gem_object *obj) } for (i = 0; i < page_count; i++) { - if (page_to_phys(obj_priv->pages[i]) & (1 << 17)) - __set_bit(i, obj_priv->bit_17); + if (page_to_phys(obj->pages[i]) & (1 << 17)) + __set_bit(i, obj->bit_17); else - __clear_bit(i, obj_priv->bit_17); + __clear_bit(i, obj->bit_17); } } diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c index 729fd0c91d7b..8a9e08bf1cf7 100644 --- a/drivers/gpu/drm/i915/i915_irq.c +++ b/drivers/gpu/drm/i915/i915_irq.c @@ -64,64 +64,24 @@ #define DRM_I915_VBLANK_PIPE_ALL (DRM_I915_VBLANK_PIPE_A | \ DRM_I915_VBLANK_PIPE_B) -void -ironlake_enable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask) -{ - if ((dev_priv->gt_irq_mask_reg & mask) != 0) { - dev_priv->gt_irq_mask_reg &= ~mask; - I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); - (void) I915_READ(GTIMR); - } -} - -void -ironlake_disable_graphics_irq(drm_i915_private_t *dev_priv, u32 mask) -{ - if ((dev_priv->gt_irq_mask_reg & mask) != mask) { - dev_priv->gt_irq_mask_reg |= mask; - I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); - (void) I915_READ(GTIMR); - } -} - /* For display hotplug interrupt */ static void ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask) { - if ((dev_priv->irq_mask_reg & mask) != 0) { - dev_priv->irq_mask_reg &= ~mask; - I915_WRITE(DEIMR, dev_priv->irq_mask_reg); - (void) I915_READ(DEIMR); + if ((dev_priv->irq_mask & mask) != 0) { + dev_priv->irq_mask &= ~mask; + I915_WRITE(DEIMR, dev_priv->irq_mask); + POSTING_READ(DEIMR); } } static inline void ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask) { - if ((dev_priv->irq_mask_reg & mask) != mask) { - dev_priv->irq_mask_reg |= mask; - I915_WRITE(DEIMR, dev_priv->irq_mask_reg); - (void) I915_READ(DEIMR); - } -} - -void -i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask) -{ - if ((dev_priv->irq_mask_reg & mask) != 0) { - dev_priv->irq_mask_reg &= ~mask; - I915_WRITE(IMR, dev_priv->irq_mask_reg); - (void) I915_READ(IMR); - } -} - -void -i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask) -{ - if ((dev_priv->irq_mask_reg & mask) != mask) { - dev_priv->irq_mask_reg |= mask; - I915_WRITE(IMR, dev_priv->irq_mask_reg); - (void) I915_READ(IMR); + if ((dev_priv->irq_mask & mask) != mask) { + dev_priv->irq_mask |= mask; + I915_WRITE(DEIMR, dev_priv->irq_mask); + POSTING_READ(DEIMR); } } @@ -144,7 +104,7 @@ i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) dev_priv->pipestat[pipe] |= mask; /* Enable the interrupt, clear any pending status */ I915_WRITE(reg, dev_priv->pipestat[pipe] | (mask >> 16)); - (void) I915_READ(reg); + POSTING_READ(reg); } } @@ -156,16 +116,19 @@ i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask) dev_priv->pipestat[pipe] &= ~mask; I915_WRITE(reg, dev_priv->pipestat[pipe]); - (void) I915_READ(reg); + POSTING_READ(reg); } } /** * intel_enable_asle - enable ASLE interrupt for OpRegion */ -void intel_enable_asle (struct drm_device *dev) +void intel_enable_asle(struct drm_device *dev) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + drm_i915_private_t *dev_priv = dev->dev_private; + unsigned long irqflags; + + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (HAS_PCH_SPLIT(dev)) ironlake_enable_display_irq(dev_priv, DE_GSE); @@ -176,6 +139,8 @@ void intel_enable_asle (struct drm_device *dev) i915_enable_pipestat(dev_priv, 0, PIPE_LEGACY_BLC_EVENT_ENABLE); } + + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } /** @@ -243,6 +208,103 @@ u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe) return I915_READ(reg); } +int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe, + int *vpos, int *hpos) +{ + drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + u32 vbl = 0, position = 0; + int vbl_start, vbl_end, htotal, vtotal; + bool in_vbl = true; + int ret = 0; + + if (!i915_pipe_enabled(dev, pipe)) { + DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled " + "pipe %d\n", pipe); + return 0; + } + + /* Get vtotal. */ + vtotal = 1 + ((I915_READ(VTOTAL(pipe)) >> 16) & 0x1fff); + + if (INTEL_INFO(dev)->gen >= 4) { + /* No obvious pixelcount register. Only query vertical + * scanout position from Display scan line register. + */ + position = I915_READ(PIPEDSL(pipe)); + + /* Decode into vertical scanout position. Don't have + * horizontal scanout position. + */ + *vpos = position & 0x1fff; + *hpos = 0; + } else { + /* Have access to pixelcount since start of frame. + * We can split this into vertical and horizontal + * scanout position. + */ + position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT; + + htotal = 1 + ((I915_READ(HTOTAL(pipe)) >> 16) & 0x1fff); + *vpos = position / htotal; + *hpos = position - (*vpos * htotal); + } + + /* Query vblank area. */ + vbl = I915_READ(VBLANK(pipe)); + + /* Test position against vblank region. */ + vbl_start = vbl & 0x1fff; + vbl_end = (vbl >> 16) & 0x1fff; + + if ((*vpos < vbl_start) || (*vpos > vbl_end)) + in_vbl = false; + + /* Inside "upper part" of vblank area? Apply corrective offset: */ + if (in_vbl && (*vpos >= vbl_start)) + *vpos = *vpos - vtotal; + + /* Readouts valid? */ + if (vbl > 0) + ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE; + + /* In vblank? */ + if (in_vbl) + ret |= DRM_SCANOUTPOS_INVBL; + + return ret; +} + +int i915_get_vblank_timestamp(struct drm_device *dev, int pipe, + int *max_error, + struct timeval *vblank_time, + unsigned flags) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + struct drm_crtc *crtc; + + if (pipe < 0 || pipe >= dev_priv->num_pipe) { + DRM_ERROR("Invalid crtc %d\n", pipe); + return -EINVAL; + } + + /* Get drm_crtc to timestamp: */ + crtc = intel_get_crtc_for_pipe(dev, pipe); + if (crtc == NULL) { + DRM_ERROR("Invalid crtc %d\n", pipe); + return -EINVAL; + } + + if (!crtc->enabled) { + DRM_DEBUG_KMS("crtc %d is disabled\n", pipe); + return -EBUSY; + } + + /* Helper routine in DRM core does all the work: */ + return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error, + vblank_time, flags, + crtc); +} + /* * Handle hotplug events outside the interrupt handler proper. */ @@ -254,6 +316,8 @@ static void i915_hotplug_work_func(struct work_struct *work) struct drm_mode_config *mode_config = &dev->mode_config; struct intel_encoder *encoder; + DRM_DEBUG_KMS("running encoder hotplug functions\n"); + list_for_each_entry(encoder, &mode_config->encoder_list, base.head) if (encoder->hot_plug) encoder->hot_plug(encoder); @@ -297,20 +361,109 @@ static void notify_ring(struct drm_device *dev, struct intel_ring_buffer *ring) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 seqno = ring->get_seqno(dev, ring); - ring->irq_gem_seqno = seqno; + u32 seqno; + + if (ring->obj == NULL) + return; + + seqno = ring->get_seqno(ring); trace_i915_gem_request_complete(dev, seqno); + + ring->irq_seqno = seqno; wake_up_all(&ring->irq_queue); + dev_priv->hangcheck_count = 0; mod_timer(&dev_priv->hangcheck_timer, jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); } +static void gen6_pm_irq_handler(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + u8 new_delay = dev_priv->cur_delay; + u32 pm_iir; + + pm_iir = I915_READ(GEN6_PMIIR); + if (!pm_iir) + return; + + if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { + if (dev_priv->cur_delay != dev_priv->max_delay) + new_delay = dev_priv->cur_delay + 1; + if (new_delay > dev_priv->max_delay) + new_delay = dev_priv->max_delay; + } else if (pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT)) { + if (dev_priv->cur_delay != dev_priv->min_delay) + new_delay = dev_priv->cur_delay - 1; + if (new_delay < dev_priv->min_delay) { + new_delay = dev_priv->min_delay; + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + I915_READ(GEN6_RP_INTERRUPT_LIMITS) | + ((new_delay << 16) & 0x3f0000)); + } else { + /* Make sure we continue to get down interrupts + * until we hit the minimum frequency */ + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + I915_READ(GEN6_RP_INTERRUPT_LIMITS) & ~0x3f0000); + } + + } + + gen6_set_rps(dev, new_delay); + dev_priv->cur_delay = new_delay; + + I915_WRITE(GEN6_PMIIR, pm_iir); +} + +static void pch_irq_handler(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; + u32 pch_iir; + + pch_iir = I915_READ(SDEIIR); + + if (pch_iir & SDE_AUDIO_POWER_MASK) + DRM_DEBUG_DRIVER("PCH audio power change on port %d\n", + (pch_iir & SDE_AUDIO_POWER_MASK) >> + SDE_AUDIO_POWER_SHIFT); + + if (pch_iir & SDE_GMBUS) + DRM_DEBUG_DRIVER("PCH GMBUS interrupt\n"); + + if (pch_iir & SDE_AUDIO_HDCP_MASK) + DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n"); + + if (pch_iir & SDE_AUDIO_TRANS_MASK) + DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n"); + + if (pch_iir & SDE_POISON) + DRM_ERROR("PCH poison interrupt\n"); + + if (pch_iir & SDE_FDI_MASK) { + u32 fdia, fdib; + + fdia = I915_READ(FDI_RXA_IIR); + fdib = I915_READ(FDI_RXB_IIR); + DRM_DEBUG_DRIVER("PCH FDI RX interrupt; FDI RXA IIR: 0x%08x, FDI RXB IIR: 0x%08x\n", fdia, fdib); + } + + if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE)) + DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n"); + + if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR)) + DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n"); + + if (pch_iir & SDE_TRANSB_FIFO_UNDER) + DRM_DEBUG_DRIVER("PCH transcoder B underrun interrupt\n"); + if (pch_iir & SDE_TRANSA_FIFO_UNDER) + DRM_DEBUG_DRIVER("PCH transcoder A underrun interrupt\n"); +} + static irqreturn_t ironlake_irq_handler(struct drm_device *dev) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; int ret = IRQ_NONE; - u32 de_iir, gt_iir, de_ier, pch_iir; + u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir; u32 hotplug_mask; struct drm_i915_master_private *master_priv; u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT; @@ -321,13 +474,15 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev) /* disable master interrupt before clearing iir */ de_ier = I915_READ(DEIER); I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL); - (void)I915_READ(DEIER); + POSTING_READ(DEIER); de_iir = I915_READ(DEIIR); gt_iir = I915_READ(GTIIR); pch_iir = I915_READ(SDEIIR); + pm_iir = I915_READ(GEN6_PMIIR); - if (de_iir == 0 && gt_iir == 0 && pch_iir == 0) + if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && + (!IS_GEN6(dev) || pm_iir == 0)) goto done; if (HAS_PCH_CPT(dev)) @@ -344,12 +499,12 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev) READ_BREADCRUMB(dev_priv); } - if (gt_iir & GT_PIPE_NOTIFY) - notify_ring(dev, &dev_priv->render_ring); + if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY)) + notify_ring(dev, &dev_priv->ring[RCS]); if (gt_iir & bsd_usr_interrupt) - notify_ring(dev, &dev_priv->bsd_ring); - if (HAS_BLT(dev) && gt_iir & GT_BLT_USER_INTERRUPT) - notify_ring(dev, &dev_priv->blt_ring); + notify_ring(dev, &dev_priv->ring[VCS]); + if (gt_iir & GT_BLT_USER_INTERRUPT) + notify_ring(dev, &dev_priv->ring[BCS]); if (de_iir & DE_GSE) intel_opregion_gse_intr(dev); @@ -371,14 +526,20 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev) drm_handle_vblank(dev, 1); /* check event from PCH */ - if ((de_iir & DE_PCH_EVENT) && (pch_iir & hotplug_mask)) - queue_work(dev_priv->wq, &dev_priv->hotplug_work); + if (de_iir & DE_PCH_EVENT) { + if (pch_iir & hotplug_mask) + queue_work(dev_priv->wq, &dev_priv->hotplug_work); + pch_irq_handler(dev); + } if (de_iir & DE_PCU_EVENT) { I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS)); i915_handle_rps_change(dev); } + if (IS_GEN6(dev)) + gen6_pm_irq_handler(dev); + /* should clear PCH hotplug event before clear CPU irq */ I915_WRITE(SDEIIR, pch_iir); I915_WRITE(GTIIR, gt_iir); @@ -386,7 +547,7 @@ static irqreturn_t ironlake_irq_handler(struct drm_device *dev) done: I915_WRITE(DEIER, de_ier); - (void)I915_READ(DEIER); + POSTING_READ(DEIER); return ret; } @@ -422,29 +583,23 @@ static void i915_error_work_func(struct work_struct *work) #ifdef CONFIG_DEBUG_FS static struct drm_i915_error_object * -i915_error_object_create(struct drm_device *dev, - struct drm_gem_object *src) +i915_error_object_create(struct drm_i915_private *dev_priv, + struct drm_i915_gem_object *src) { - drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_error_object *dst; - struct drm_i915_gem_object *src_priv; int page, page_count; u32 reloc_offset; - if (src == NULL) + if (src == NULL || src->pages == NULL) return NULL; - src_priv = to_intel_bo(src); - if (src_priv->pages == NULL) - return NULL; - - page_count = src->size / PAGE_SIZE; + page_count = src->base.size / PAGE_SIZE; dst = kmalloc(sizeof(*dst) + page_count * sizeof (u32 *), GFP_ATOMIC); if (dst == NULL) return NULL; - reloc_offset = src_priv->gtt_offset; + reloc_offset = src->gtt_offset; for (page = 0; page < page_count; page++) { unsigned long flags; void __iomem *s; @@ -466,7 +621,7 @@ i915_error_object_create(struct drm_device *dev, reloc_offset += PAGE_SIZE; } dst->page_count = page_count; - dst->gtt_offset = src_priv->gtt_offset; + dst->gtt_offset = src->gtt_offset; return dst; @@ -503,53 +658,98 @@ i915_error_state_free(struct drm_device *dev, kfree(error); } -static u32 -i915_get_bbaddr(struct drm_device *dev, u32 *ring) +static u32 capture_bo_list(struct drm_i915_error_buffer *err, + int count, + struct list_head *head) { - u32 cmd; + struct drm_i915_gem_object *obj; + int i = 0; + + list_for_each_entry(obj, head, mm_list) { + err->size = obj->base.size; + err->name = obj->base.name; + err->seqno = obj->last_rendering_seqno; + err->gtt_offset = obj->gtt_offset; + err->read_domains = obj->base.read_domains; + err->write_domain = obj->base.write_domain; + err->fence_reg = obj->fence_reg; + err->pinned = 0; + if (obj->pin_count > 0) + err->pinned = 1; + if (obj->user_pin_count > 0) + err->pinned = -1; + err->tiling = obj->tiling_mode; + err->dirty = obj->dirty; + err->purgeable = obj->madv != I915_MADV_WILLNEED; + err->ring = obj->ring ? obj->ring->id : 0; + err->agp_type = obj->agp_type == AGP_USER_CACHED_MEMORY; + + if (++i == count) + break; - if (IS_I830(dev) || IS_845G(dev)) - cmd = MI_BATCH_BUFFER; - else if (INTEL_INFO(dev)->gen >= 4) - cmd = (MI_BATCH_BUFFER_START | (2 << 6) | - MI_BATCH_NON_SECURE_I965); - else - cmd = (MI_BATCH_BUFFER_START | (2 << 6)); + err++; + } - return ring[0] == cmd ? ring[1] : 0; + return i; } -static u32 -i915_ringbuffer_last_batch(struct drm_device *dev) +static void i915_gem_record_fences(struct drm_device *dev, + struct drm_i915_error_state *error) { struct drm_i915_private *dev_priv = dev->dev_private; - u32 head, bbaddr; - u32 *ring; - - /* Locate the current position in the ringbuffer and walk back - * to find the most recently dispatched batch buffer. - */ - bbaddr = 0; - head = I915_READ(PRB0_HEAD) & HEAD_ADDR; - ring = (u32 *)(dev_priv->render_ring.virtual_start + head); + int i; + + /* Fences */ + switch (INTEL_INFO(dev)->gen) { + case 6: + for (i = 0; i < 16; i++) + error->fence[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8)); + break; + case 5: + case 4: + for (i = 0; i < 16; i++) + error->fence[i] = I915_READ64(FENCE_REG_965_0 + (i * 8)); + break; + case 3: + if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) + for (i = 0; i < 8; i++) + error->fence[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4)); + case 2: + for (i = 0; i < 8; i++) + error->fence[i] = I915_READ(FENCE_REG_830_0 + (i * 4)); + break; - while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) { - bbaddr = i915_get_bbaddr(dev, ring); - if (bbaddr) - break; } +} - if (bbaddr == 0) { - ring = (u32 *)(dev_priv->render_ring.virtual_start - + dev_priv->render_ring.size); - while (--ring >= (u32 *)dev_priv->render_ring.virtual_start) { - bbaddr = i915_get_bbaddr(dev, ring); - if (bbaddr) - break; - } +static struct drm_i915_error_object * +i915_error_first_batchbuffer(struct drm_i915_private *dev_priv, + struct intel_ring_buffer *ring) +{ + struct drm_i915_gem_object *obj; + u32 seqno; + + if (!ring->get_seqno) + return NULL; + + seqno = ring->get_seqno(ring); + list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { + if (obj->ring != ring) + continue; + + if (i915_seqno_passed(seqno, obj->last_rendering_seqno)) + continue; + + if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0) + continue; + + /* We need to copy these to an anonymous buffer as the simplest + * method to avoid being overwritten by userspace. + */ + return i915_error_object_create(dev_priv, obj); } - return bbaddr; + return NULL; } /** @@ -564,12 +764,10 @@ i915_ringbuffer_last_batch(struct drm_device *dev) static void i915_capture_error_state(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; struct drm_i915_error_state *error; - struct drm_gem_object *batchbuffer[2]; unsigned long flags; - u32 bbaddr; - int count; + int i; spin_lock_irqsave(&dev_priv->error_lock, flags); error = dev_priv->first_error; @@ -585,20 +783,33 @@ static void i915_capture_error_state(struct drm_device *dev) DRM_DEBUG_DRIVER("generating error event\n"); - error->seqno = - dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring); + error->seqno = dev_priv->ring[RCS].get_seqno(&dev_priv->ring[RCS]); error->eir = I915_READ(EIR); error->pgtbl_er = I915_READ(PGTBL_ER); error->pipeastat = I915_READ(PIPEASTAT); error->pipebstat = I915_READ(PIPEBSTAT); error->instpm = I915_READ(INSTPM); - if (INTEL_INFO(dev)->gen < 4) { - error->ipeir = I915_READ(IPEIR); - error->ipehr = I915_READ(IPEHR); - error->instdone = I915_READ(INSTDONE); - error->acthd = I915_READ(ACTHD); - error->bbaddr = 0; - } else { + error->error = 0; + if (INTEL_INFO(dev)->gen >= 6) { + error->error = I915_READ(ERROR_GEN6); + + error->bcs_acthd = I915_READ(BCS_ACTHD); + error->bcs_ipehr = I915_READ(BCS_IPEHR); + error->bcs_ipeir = I915_READ(BCS_IPEIR); + error->bcs_instdone = I915_READ(BCS_INSTDONE); + error->bcs_seqno = 0; + if (dev_priv->ring[BCS].get_seqno) + error->bcs_seqno = dev_priv->ring[BCS].get_seqno(&dev_priv->ring[BCS]); + + error->vcs_acthd = I915_READ(VCS_ACTHD); + error->vcs_ipehr = I915_READ(VCS_IPEHR); + error->vcs_ipeir = I915_READ(VCS_IPEIR); + error->vcs_instdone = I915_READ(VCS_INSTDONE); + error->vcs_seqno = 0; + if (dev_priv->ring[VCS].get_seqno) + error->vcs_seqno = dev_priv->ring[VCS].get_seqno(&dev_priv->ring[VCS]); + } + if (INTEL_INFO(dev)->gen >= 4) { error->ipeir = I915_READ(IPEIR_I965); error->ipehr = I915_READ(IPEHR_I965); error->instdone = I915_READ(INSTDONE_I965); @@ -606,118 +817,63 @@ static void i915_capture_error_state(struct drm_device *dev) error->instdone1 = I915_READ(INSTDONE1); error->acthd = I915_READ(ACTHD_I965); error->bbaddr = I915_READ64(BB_ADDR); + } else { + error->ipeir = I915_READ(IPEIR); + error->ipehr = I915_READ(IPEHR); + error->instdone = I915_READ(INSTDONE); + error->acthd = I915_READ(ACTHD); + error->bbaddr = 0; } + i915_gem_record_fences(dev, error); - bbaddr = i915_ringbuffer_last_batch(dev); - - /* Grab the current batchbuffer, most likely to have crashed. */ - batchbuffer[0] = NULL; - batchbuffer[1] = NULL; - count = 0; - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { - struct drm_gem_object *obj = &obj_priv->base; - - if (batchbuffer[0] == NULL && - bbaddr >= obj_priv->gtt_offset && - bbaddr < obj_priv->gtt_offset + obj->size) - batchbuffer[0] = obj; - - if (batchbuffer[1] == NULL && - error->acthd >= obj_priv->gtt_offset && - error->acthd < obj_priv->gtt_offset + obj->size) - batchbuffer[1] = obj; - - count++; - } - /* Scan the other lists for completeness for those bizarre errors. */ - if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) { - list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, mm_list) { - struct drm_gem_object *obj = &obj_priv->base; - - if (batchbuffer[0] == NULL && - bbaddr >= obj_priv->gtt_offset && - bbaddr < obj_priv->gtt_offset + obj->size) - batchbuffer[0] = obj; - - if (batchbuffer[1] == NULL && - error->acthd >= obj_priv->gtt_offset && - error->acthd < obj_priv->gtt_offset + obj->size) - batchbuffer[1] = obj; - - if (batchbuffer[0] && batchbuffer[1]) - break; - } - } - if (batchbuffer[0] == NULL || batchbuffer[1] == NULL) { - list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, mm_list) { - struct drm_gem_object *obj = &obj_priv->base; - - if (batchbuffer[0] == NULL && - bbaddr >= obj_priv->gtt_offset && - bbaddr < obj_priv->gtt_offset + obj->size) - batchbuffer[0] = obj; - - if (batchbuffer[1] == NULL && - error->acthd >= obj_priv->gtt_offset && - error->acthd < obj_priv->gtt_offset + obj->size) - batchbuffer[1] = obj; - - if (batchbuffer[0] && batchbuffer[1]) - break; - } - } - - /* We need to copy these to an anonymous buffer as the simplest - * method to avoid being overwritten by userspace. - */ - error->batchbuffer[0] = i915_error_object_create(dev, batchbuffer[0]); - if (batchbuffer[1] != batchbuffer[0]) - error->batchbuffer[1] = i915_error_object_create(dev, batchbuffer[1]); - else - error->batchbuffer[1] = NULL; + /* Record the active batchbuffers */ + for (i = 0; i < I915_NUM_RINGS; i++) + error->batchbuffer[i] = + i915_error_first_batchbuffer(dev_priv, + &dev_priv->ring[i]); /* Record the ringbuffer */ - error->ringbuffer = i915_error_object_create(dev, - dev_priv->render_ring.gem_object); + error->ringbuffer = i915_error_object_create(dev_priv, + dev_priv->ring[RCS].obj); - /* Record buffers on the active list. */ + /* Record buffers on the active and pinned lists. */ error->active_bo = NULL; - error->active_bo_count = 0; + error->pinned_bo = NULL; - if (count) - error->active_bo = kmalloc(sizeof(*error->active_bo)*count, - GFP_ATOMIC); + i = 0; + list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) + i++; + error->active_bo_count = i; + list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) + i++; + error->pinned_bo_count = i - error->active_bo_count; - if (error->active_bo) { - int i = 0; - list_for_each_entry(obj_priv, &dev_priv->mm.active_list, mm_list) { - struct drm_gem_object *obj = &obj_priv->base; - - error->active_bo[i].size = obj->size; - error->active_bo[i].name = obj->name; - error->active_bo[i].seqno = obj_priv->last_rendering_seqno; - error->active_bo[i].gtt_offset = obj_priv->gtt_offset; - error->active_bo[i].read_domains = obj->read_domains; - error->active_bo[i].write_domain = obj->write_domain; - error->active_bo[i].fence_reg = obj_priv->fence_reg; - error->active_bo[i].pinned = 0; - if (obj_priv->pin_count > 0) - error->active_bo[i].pinned = 1; - if (obj_priv->user_pin_count > 0) - error->active_bo[i].pinned = -1; - error->active_bo[i].tiling = obj_priv->tiling_mode; - error->active_bo[i].dirty = obj_priv->dirty; - error->active_bo[i].purgeable = obj_priv->madv != I915_MADV_WILLNEED; - - if (++i == count) - break; - } - error->active_bo_count = i; + error->active_bo = NULL; + error->pinned_bo = NULL; + if (i) { + error->active_bo = kmalloc(sizeof(*error->active_bo)*i, + GFP_ATOMIC); + if (error->active_bo) + error->pinned_bo = + error->active_bo + error->active_bo_count; } + if (error->active_bo) + error->active_bo_count = + capture_bo_list(error->active_bo, + error->active_bo_count, + &dev_priv->mm.active_list); + + if (error->pinned_bo) + error->pinned_bo_count = + capture_bo_list(error->pinned_bo, + error->pinned_bo_count, + &dev_priv->mm.pinned_list); + do_gettimeofday(&error->time); error->overlay = intel_overlay_capture_error_state(dev); + error->display = intel_display_capture_error_state(dev); spin_lock_irqsave(&dev_priv->error_lock, flags); if (dev_priv->first_error == NULL) { @@ -775,7 +931,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) printk(KERN_ERR " ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); I915_WRITE(IPEIR_I965, ipeir); - (void)I915_READ(IPEIR_I965); + POSTING_READ(IPEIR_I965); } if (eir & GM45_ERROR_PAGE_TABLE) { u32 pgtbl_err = I915_READ(PGTBL_ER); @@ -783,7 +939,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) printk(KERN_ERR " PGTBL_ER: 0x%08x\n", pgtbl_err); I915_WRITE(PGTBL_ER, pgtbl_err); - (void)I915_READ(PGTBL_ER); + POSTING_READ(PGTBL_ER); } } @@ -794,7 +950,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) printk(KERN_ERR " PGTBL_ER: 0x%08x\n", pgtbl_err); I915_WRITE(PGTBL_ER, pgtbl_err); - (void)I915_READ(PGTBL_ER); + POSTING_READ(PGTBL_ER); } } @@ -825,7 +981,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) printk(KERN_ERR " ACTHD: 0x%08x\n", I915_READ(ACTHD)); I915_WRITE(IPEIR, ipeir); - (void)I915_READ(IPEIR); + POSTING_READ(IPEIR); } else { u32 ipeir = I915_READ(IPEIR_I965); @@ -842,12 +998,12 @@ static void i915_report_and_clear_eir(struct drm_device *dev) printk(KERN_ERR " ACTHD: 0x%08x\n", I915_READ(ACTHD_I965)); I915_WRITE(IPEIR_I965, ipeir); - (void)I915_READ(IPEIR_I965); + POSTING_READ(IPEIR_I965); } } I915_WRITE(EIR, eir); - (void)I915_READ(EIR); + POSTING_READ(EIR); eir = I915_READ(EIR); if (eir) { /* @@ -870,7 +1026,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev) * so userspace knows something bad happened (should trigger collection * of a ring dump etc.). */ -static void i915_handle_error(struct drm_device *dev, bool wedged) +void i915_handle_error(struct drm_device *dev, bool wedged) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -884,11 +1040,11 @@ static void i915_handle_error(struct drm_device *dev, bool wedged) /* * Wakeup waiting processes so they don't hang */ - wake_up_all(&dev_priv->render_ring.irq_queue); + wake_up_all(&dev_priv->ring[RCS].irq_queue); if (HAS_BSD(dev)) - wake_up_all(&dev_priv->bsd_ring.irq_queue); + wake_up_all(&dev_priv->ring[VCS].irq_queue); if (HAS_BLT(dev)) - wake_up_all(&dev_priv->blt_ring.irq_queue); + wake_up_all(&dev_priv->ring[BCS].irq_queue); } queue_work(dev_priv->wq, &dev_priv->error_work); @@ -899,7 +1055,7 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe) drm_i915_private_t *dev_priv = dev->dev_private; struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; struct intel_unpin_work *work; unsigned long flags; bool stall_detected; @@ -918,13 +1074,13 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe) } /* Potential stall - if we see that the flip has happened, assume a missed interrupt */ - obj_priv = to_intel_bo(work->pending_flip_obj); + obj = work->pending_flip_obj; if (INTEL_INFO(dev)->gen >= 4) { int dspsurf = intel_crtc->plane == 0 ? DSPASURF : DSPBSURF; - stall_detected = I915_READ(dspsurf) == obj_priv->gtt_offset; + stall_detected = I915_READ(dspsurf) == obj->gtt_offset; } else { int dspaddr = intel_crtc->plane == 0 ? DSPAADDR : DSPBADDR; - stall_detected = I915_READ(dspaddr) == (obj_priv->gtt_offset + + stall_detected = I915_READ(dspaddr) == (obj->gtt_offset + crtc->y * crtc->fb->pitch + crtc->x * crtc->fb->bits_per_pixel/8); } @@ -970,7 +1126,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) * It doesn't set the bit in iir again, but it still produces * interrupts (for non-MSI). */ - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); pipea_stats = I915_READ(PIPEASTAT); pipeb_stats = I915_READ(PIPEBSTAT); @@ -993,7 +1149,7 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) I915_WRITE(PIPEBSTAT, pipeb_stats); irq_received = 1; } - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); if (!irq_received) break; @@ -1026,9 +1182,9 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) } if (iir & I915_USER_INTERRUPT) - notify_ring(dev, &dev_priv->render_ring); - if (HAS_BSD(dev) && (iir & I915_BSD_USER_INTERRUPT)) - notify_ring(dev, &dev_priv->bsd_ring); + notify_ring(dev, &dev_priv->ring[RCS]); + if (iir & I915_BSD_USER_INTERRUPT) + notify_ring(dev, &dev_priv->ring[VCS]); if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) { intel_prepare_page_flip(dev, 0); @@ -1042,18 +1198,18 @@ irqreturn_t i915_driver_irq_handler(DRM_IRQ_ARGS) intel_finish_page_flip_plane(dev, 1); } - if (pipea_stats & vblank_status) { + if (pipea_stats & vblank_status && + drm_handle_vblank(dev, 0)) { vblank++; - drm_handle_vblank(dev, 0); if (!dev_priv->flip_pending_is_done) { i915_pageflip_stall_check(dev, 0); intel_finish_page_flip(dev, 0); } } - if (pipeb_stats & vblank_status) { + if (pipeb_stats & vblank_status && + drm_handle_vblank(dev, 1)) { vblank++; - drm_handle_vblank(dev, 1); if (!dev_priv->flip_pending_is_done) { i915_pageflip_stall_check(dev, 1); intel_finish_page_flip(dev, 1); @@ -1101,12 +1257,13 @@ static int i915_emit_irq(struct drm_device * dev) if (master_priv->sarea_priv) master_priv->sarea_priv->last_enqueue = dev_priv->counter; - BEGIN_LP_RING(4); - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(dev_priv->counter); - OUT_RING(MI_USER_INTERRUPT); - ADVANCE_LP_RING(); + if (BEGIN_LP_RING(4) == 0) { + OUT_RING(MI_STORE_DWORD_INDEX); + OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + OUT_RING(dev_priv->counter); + OUT_RING(MI_USER_INTERRUPT); + ADVANCE_LP_RING(); + } return dev_priv->counter; } @@ -1114,12 +1271,11 @@ static int i915_emit_irq(struct drm_device * dev) void i915_trace_irq_get(struct drm_device *dev, u32 seqno) { drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - struct intel_ring_buffer *render_ring = &dev_priv->render_ring; - - if (dev_priv->trace_irq_seqno == 0) - render_ring->user_irq_get(dev, render_ring); + struct intel_ring_buffer *ring = LP_RING(dev_priv); - dev_priv->trace_irq_seqno = seqno; + if (dev_priv->trace_irq_seqno == 0 && + ring->irq_get(ring)) + dev_priv->trace_irq_seqno = seqno; } static int i915_wait_irq(struct drm_device * dev, int irq_nr) @@ -1127,7 +1283,7 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr) drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv; int ret = 0; - struct intel_ring_buffer *render_ring = &dev_priv->render_ring; + struct intel_ring_buffer *ring = LP_RING(dev_priv); DRM_DEBUG_DRIVER("irq_nr=%d breadcrumb=%d\n", irq_nr, READ_BREADCRUMB(dev_priv)); @@ -1141,10 +1297,12 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr) if (master_priv->sarea_priv) master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT; - render_ring->user_irq_get(dev, render_ring); - DRM_WAIT_ON(ret, dev_priv->render_ring.irq_queue, 3 * DRM_HZ, - READ_BREADCRUMB(dev_priv) >= irq_nr); - render_ring->user_irq_put(dev, render_ring); + if (ring->irq_get(ring)) { + DRM_WAIT_ON(ret, ring->irq_queue, 3 * DRM_HZ, + READ_BREADCRUMB(dev_priv) >= irq_nr); + ring->irq_put(ring); + } else if (wait_for(READ_BREADCRUMB(dev_priv) >= irq_nr, 3000)) + ret = -EBUSY; if (ret == -EBUSY) { DRM_ERROR("EBUSY -- rec: %d emitted: %d\n", @@ -1163,7 +1321,7 @@ int i915_irq_emit(struct drm_device *dev, void *data, drm_i915_irq_emit_t *emit = data; int result; - if (!dev_priv || !dev_priv->render_ring.virtual_start) { + if (!dev_priv || !LP_RING(dev_priv)->virtual_start) { DRM_ERROR("called with no initialization\n"); return -EINVAL; } @@ -1209,9 +1367,9 @@ int i915_enable_vblank(struct drm_device *dev, int pipe) if (!i915_pipe_enabled(dev, pipe)) return -EINVAL; - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (HAS_PCH_SPLIT(dev)) - ironlake_enable_display_irq(dev_priv, (pipe == 0) ? + ironlake_enable_display_irq(dev_priv, (pipe == 0) ? DE_PIPEA_VBLANK: DE_PIPEB_VBLANK); else if (INTEL_INFO(dev)->gen >= 4) i915_enable_pipestat(dev_priv, pipe, @@ -1219,7 +1377,7 @@ int i915_enable_vblank(struct drm_device *dev, int pipe) else i915_enable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_ENABLE); - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); return 0; } @@ -1231,15 +1389,15 @@ void i915_disable_vblank(struct drm_device *dev, int pipe) drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; unsigned long irqflags; - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); if (HAS_PCH_SPLIT(dev)) - ironlake_disable_display_irq(dev_priv, (pipe == 0) ? + ironlake_disable_display_irq(dev_priv, (pipe == 0) ? DE_PIPEA_VBLANK: DE_PIPEB_VBLANK); else i915_disable_pipestat(dev_priv, pipe, PIPE_VBLANK_INTERRUPT_ENABLE | PIPE_START_VBLANK_INTERRUPT_ENABLE); - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); } void i915_enable_interrupt (struct drm_device *dev) @@ -1306,12 +1464,50 @@ int i915_vblank_swap(struct drm_device *dev, void *data, return -EINVAL; } -static struct drm_i915_gem_request * -i915_get_tail_request(struct drm_device *dev) +static u32 +ring_last_seqno(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; - return list_entry(dev_priv->render_ring.request_list.prev, - struct drm_i915_gem_request, list); + return list_entry(ring->request_list.prev, + struct drm_i915_gem_request, list)->seqno; +} + +static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err) +{ + if (list_empty(&ring->request_list) || + i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) { + /* Issue a wake-up to catch stuck h/w. */ + if (ring->waiting_seqno && waitqueue_active(&ring->irq_queue)) { + DRM_ERROR("Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n", + ring->name, + ring->waiting_seqno, + ring->get_seqno(ring)); + wake_up_all(&ring->irq_queue); + *err = true; + } + return true; + } + return false; +} + +static bool kick_ring(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + u32 tmp = I915_READ_CTL(ring); + if (tmp & RING_WAIT) { + DRM_ERROR("Kicking stuck wait on %s\n", + ring->name); + I915_WRITE_CTL(ring, tmp); + return true; + } + if (IS_GEN6(dev) && + (tmp & RING_WAIT_SEMAPHORE)) { + DRM_ERROR("Kicking stuck semaphore on %s\n", + ring->name); + I915_WRITE_CTL(ring, tmp); + return true; + } + return false; } /** @@ -1325,6 +1521,17 @@ void i915_hangcheck_elapsed(unsigned long data) struct drm_device *dev = (struct drm_device *)data; drm_i915_private_t *dev_priv = dev->dev_private; uint32_t acthd, instdone, instdone1; + bool err = false; + + /* If all work is done then ACTHD clearly hasn't advanced. */ + if (i915_hangcheck_ring_idle(&dev_priv->ring[RCS], &err) && + i915_hangcheck_ring_idle(&dev_priv->ring[VCS], &err) && + i915_hangcheck_ring_idle(&dev_priv->ring[BCS], &err)) { + dev_priv->hangcheck_count = 0; + if (err) + goto repeat; + return; + } if (INTEL_INFO(dev)->gen < 4) { acthd = I915_READ(ACTHD); @@ -1336,38 +1543,6 @@ void i915_hangcheck_elapsed(unsigned long data) instdone1 = I915_READ(INSTDONE1); } - /* If all work is done then ACTHD clearly hasn't advanced. */ - if (list_empty(&dev_priv->render_ring.request_list) || - i915_seqno_passed(dev_priv->render_ring.get_seqno(dev, &dev_priv->render_ring), - i915_get_tail_request(dev)->seqno)) { - bool missed_wakeup = false; - - dev_priv->hangcheck_count = 0; - - /* Issue a wake-up to catch stuck h/w. */ - if (dev_priv->render_ring.waiting_gem_seqno && - waitqueue_active(&dev_priv->render_ring.irq_queue)) { - wake_up_all(&dev_priv->render_ring.irq_queue); - missed_wakeup = true; - } - - if (dev_priv->bsd_ring.waiting_gem_seqno && - waitqueue_active(&dev_priv->bsd_ring.irq_queue)) { - wake_up_all(&dev_priv->bsd_ring.irq_queue); - missed_wakeup = true; - } - - if (dev_priv->blt_ring.waiting_gem_seqno && - waitqueue_active(&dev_priv->blt_ring.irq_queue)) { - wake_up_all(&dev_priv->blt_ring.irq_queue); - missed_wakeup = true; - } - - if (missed_wakeup) - DRM_ERROR("Hangcheck timer elapsed... GPU idle, missed IRQ.\n"); - return; - } - if (dev_priv->last_acthd == acthd && dev_priv->last_instdone == instdone && dev_priv->last_instdone1 == instdone1) { @@ -1380,12 +1555,17 @@ void i915_hangcheck_elapsed(unsigned long data) * and break the hang. This should work on * all but the second generation chipsets. */ - u32 tmp = I915_READ(PRB0_CTL); - if (tmp & RING_WAIT) { - I915_WRITE(PRB0_CTL, tmp); - POSTING_READ(PRB0_CTL); - goto out; - } + + if (kick_ring(&dev_priv->ring[RCS])) + goto repeat; + + if (HAS_BSD(dev) && + kick_ring(&dev_priv->ring[VCS])) + goto repeat; + + if (HAS_BLT(dev) && + kick_ring(&dev_priv->ring[BCS])) + goto repeat; } i915_handle_error(dev, true); @@ -1399,7 +1579,7 @@ void i915_hangcheck_elapsed(unsigned long data) dev_priv->last_instdone1 = instdone1; } -out: +repeat: /* Reset timer case chip hangs without another request being added */ mod_timer(&dev_priv->hangcheck_timer, jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD)); @@ -1417,17 +1597,17 @@ static void ironlake_irq_preinstall(struct drm_device *dev) I915_WRITE(DEIMR, 0xffffffff); I915_WRITE(DEIER, 0x0); - (void) I915_READ(DEIER); + POSTING_READ(DEIER); /* and GT */ I915_WRITE(GTIMR, 0xffffffff); I915_WRITE(GTIER, 0x0); - (void) I915_READ(GTIER); + POSTING_READ(GTIER); /* south display irq */ I915_WRITE(SDEIMR, 0xffffffff); I915_WRITE(SDEIER, 0x0); - (void) I915_READ(SDEIER); + POSTING_READ(SDEIER); } static int ironlake_irq_postinstall(struct drm_device *dev) @@ -1436,38 +1616,34 @@ static int ironlake_irq_postinstall(struct drm_device *dev) /* enable kind of interrupts always enabled */ u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT | DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE; - u32 render_mask = GT_PIPE_NOTIFY | GT_BSD_USER_INTERRUPT; + u32 render_irqs; u32 hotplug_mask; - dev_priv->irq_mask_reg = ~display_mask; - dev_priv->de_irq_enable_reg = display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK; + dev_priv->irq_mask = ~display_mask; /* should always can generate irq */ I915_WRITE(DEIIR, I915_READ(DEIIR)); - I915_WRITE(DEIMR, dev_priv->irq_mask_reg); - I915_WRITE(DEIER, dev_priv->de_irq_enable_reg); - (void) I915_READ(DEIER); - - if (IS_GEN6(dev)) { - render_mask = - GT_PIPE_NOTIFY | - GT_GEN6_BSD_USER_INTERRUPT | - GT_BLT_USER_INTERRUPT; - } + I915_WRITE(DEIMR, dev_priv->irq_mask); + I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK | DE_PIPEB_VBLANK); + POSTING_READ(DEIER); - dev_priv->gt_irq_mask_reg = ~render_mask; - dev_priv->gt_irq_enable_reg = render_mask; + dev_priv->gt_irq_mask = ~0; I915_WRITE(GTIIR, I915_READ(GTIIR)); - I915_WRITE(GTIMR, dev_priv->gt_irq_mask_reg); - if (IS_GEN6(dev)) { - I915_WRITE(GEN6_RENDER_IMR, ~GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT); - I915_WRITE(GEN6_BSD_IMR, ~GEN6_BSD_IMR_USER_INTERRUPT); - I915_WRITE(GEN6_BLITTER_IMR, ~GEN6_BLITTER_USER_INTERRUPT); - } + I915_WRITE(GTIMR, dev_priv->gt_irq_mask); - I915_WRITE(GTIER, dev_priv->gt_irq_enable_reg); - (void) I915_READ(GTIER); + if (IS_GEN6(dev)) + render_irqs = + GT_USER_INTERRUPT | + GT_GEN6_BSD_USER_INTERRUPT | + GT_BLT_USER_INTERRUPT; + else + render_irqs = + GT_USER_INTERRUPT | + GT_PIPE_NOTIFY | + GT_BSD_USER_INTERRUPT; + I915_WRITE(GTIER, render_irqs); + POSTING_READ(GTIER); if (HAS_PCH_CPT(dev)) { hotplug_mask = SDE_CRT_HOTPLUG_CPT | SDE_PORTB_HOTPLUG_CPT | @@ -1475,15 +1651,15 @@ static int ironlake_irq_postinstall(struct drm_device *dev) } else { hotplug_mask = SDE_CRT_HOTPLUG | SDE_PORTB_HOTPLUG | SDE_PORTC_HOTPLUG | SDE_PORTD_HOTPLUG; + hotplug_mask |= SDE_AUX_MASK; } - dev_priv->pch_irq_mask_reg = ~hotplug_mask; - dev_priv->pch_irq_enable_reg = hotplug_mask; + dev_priv->pch_irq_mask = ~hotplug_mask; I915_WRITE(SDEIIR, I915_READ(SDEIIR)); - I915_WRITE(SDEIMR, dev_priv->pch_irq_mask_reg); - I915_WRITE(SDEIER, dev_priv->pch_irq_enable_reg); - (void) I915_READ(SDEIER); + I915_WRITE(SDEIMR, dev_priv->pch_irq_mask); + I915_WRITE(SDEIER, hotplug_mask); + POSTING_READ(SDEIER); if (IS_IRONLAKE_M(dev)) { /* Clear & enable PCU event interrupts */ @@ -1519,7 +1695,7 @@ void i915_driver_irq_preinstall(struct drm_device * dev) I915_WRITE(PIPEBSTAT, 0); I915_WRITE(IMR, 0xffffffff); I915_WRITE(IER, 0x0); - (void) I915_READ(IER); + POSTING_READ(IER); } /* @@ -1532,11 +1708,11 @@ int i915_driver_irq_postinstall(struct drm_device *dev) u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR; u32 error_mask; - DRM_INIT_WAITQUEUE(&dev_priv->render_ring.irq_queue); + DRM_INIT_WAITQUEUE(&dev_priv->ring[RCS].irq_queue); if (HAS_BSD(dev)) - DRM_INIT_WAITQUEUE(&dev_priv->bsd_ring.irq_queue); + DRM_INIT_WAITQUEUE(&dev_priv->ring[VCS].irq_queue); if (HAS_BLT(dev)) - DRM_INIT_WAITQUEUE(&dev_priv->blt_ring.irq_queue); + DRM_INIT_WAITQUEUE(&dev_priv->ring[BCS].irq_queue); dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B; @@ -1544,7 +1720,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev) return ironlake_irq_postinstall(dev); /* Unmask the interrupts that we always want on. */ - dev_priv->irq_mask_reg = ~I915_INTERRUPT_ENABLE_FIX; + dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX; dev_priv->pipestat[0] = 0; dev_priv->pipestat[1] = 0; @@ -1553,7 +1729,7 @@ int i915_driver_irq_postinstall(struct drm_device *dev) /* Enable in IER... */ enable_mask |= I915_DISPLAY_PORT_INTERRUPT; /* and unmask in IMR */ - dev_priv->irq_mask_reg &= ~I915_DISPLAY_PORT_INTERRUPT; + dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT; } /* @@ -1571,9 +1747,9 @@ int i915_driver_irq_postinstall(struct drm_device *dev) } I915_WRITE(EMR, error_mask); - I915_WRITE(IMR, dev_priv->irq_mask_reg); + I915_WRITE(IMR, dev_priv->irq_mask); I915_WRITE(IER, enable_mask); - (void) I915_READ(IER); + POSTING_READ(IER); if (I915_HAS_HOTPLUG(dev)) { u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN); diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index cb8f43429279..729d4233b763 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -78,6 +78,12 @@ #define GRDOM_RENDER (1<<2) #define GRDOM_MEDIA (3<<2) +#define GEN6_GDRST 0x941c +#define GEN6_GRDOM_FULL (1 << 0) +#define GEN6_GRDOM_RENDER (1 << 1) +#define GEN6_GRDOM_MEDIA (1 << 2) +#define GEN6_GRDOM_BLT (1 << 3) + /* VGA stuff */ #define VGA_ST01_MDA 0x3ba @@ -139,6 +145,8 @@ #define MI_END_SCENE (1 << 4) /* flush binner and incr scene count */ #define MI_INVALIDATE_ISP (1 << 5) /* invalidate indirect state pointers */ #define MI_BATCH_BUFFER_END MI_INSTR(0x0a, 0) +#define MI_SUSPEND_FLUSH MI_INSTR(0x0b, 0) +#define MI_SUSPEND_FLUSH_EN (1<<0) #define MI_REPORT_HEAD MI_INSTR(0x07, 0) #define MI_OVERLAY_FLIP MI_INSTR(0x11,0) #define MI_OVERLAY_CONTINUE (0x0<<21) @@ -153,17 +161,31 @@ #define MI_MM_SPACE_PHYSICAL (0<<8) #define MI_SAVE_EXT_STATE_EN (1<<3) #define MI_RESTORE_EXT_STATE_EN (1<<2) +#define MI_FORCE_RESTORE (1<<1) #define MI_RESTORE_INHIBIT (1<<0) #define MI_STORE_DWORD_IMM MI_INSTR(0x20, 1) #define MI_MEM_VIRTUAL (1 << 22) /* 965+ only */ #define MI_STORE_DWORD_INDEX MI_INSTR(0x21, 1) #define MI_STORE_DWORD_INDEX_SHIFT 2 -#define MI_LOAD_REGISTER_IMM MI_INSTR(0x22, 1) -#define MI_FLUSH_DW MI_INSTR(0x26, 2) /* for GEN6 */ +/* Official intel docs are somewhat sloppy concerning MI_LOAD_REGISTER_IMM: + * - Always issue a MI_NOOP _before_ the MI_LOAD_REGISTER_IMM - otherwise hw + * simply ignores the register load under certain conditions. + * - One can actually load arbitrary many arbitrary registers: Simply issue x + * address/value pairs. Don't overdue it, though, x <= 2^4 must hold! + */ +#define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*x-1) +#define MI_FLUSH_DW MI_INSTR(0x26, 1) /* for GEN6 */ +#define MI_INVALIDATE_TLB (1<<18) +#define MI_INVALIDATE_BSD (1<<7) #define MI_BATCH_BUFFER MI_INSTR(0x30, 1) #define MI_BATCH_NON_SECURE (1) #define MI_BATCH_NON_SECURE_I965 (1<<8) #define MI_BATCH_BUFFER_START MI_INSTR(0x31, 0) +#define MI_SEMAPHORE_MBOX MI_INSTR(0x16, 1) /* gen6+ */ +#define MI_SEMAPHORE_GLOBAL_GTT (1<<22) +#define MI_SEMAPHORE_UPDATE (1<<21) +#define MI_SEMAPHORE_COMPARE (1<<20) +#define MI_SEMAPHORE_REGISTER (1<<18) /* * 3D instructions used by the kernel */ @@ -256,10 +278,6 @@ * Instruction and interrupt control regs */ #define PGTBL_ER 0x02024 -#define PRB0_TAIL 0x02030 -#define PRB0_HEAD 0x02034 -#define PRB0_START 0x02038 -#define PRB0_CTL 0x0203c #define RENDER_RING_BASE 0x02000 #define BSD_RING_BASE 0x04000 #define GEN6_BSD_RING_BASE 0x12000 @@ -268,9 +286,14 @@ #define RING_HEAD(base) ((base)+0x34) #define RING_START(base) ((base)+0x38) #define RING_CTL(base) ((base)+0x3c) +#define RING_SYNC_0(base) ((base)+0x40) +#define RING_SYNC_1(base) ((base)+0x44) +#define RING_MAX_IDLE(base) ((base)+0x54) #define RING_HWS_PGA(base) ((base)+0x80) #define RING_HWS_PGA_GEN6(base) ((base)+0x2080) #define RING_ACTHD(base) ((base)+0x74) +#define RING_NOPID(base) ((base)+0x94) +#define RING_IMR(base) ((base)+0xa8) #define TAIL_ADDR 0x001FFFF8 #define HEAD_WRAP_COUNT 0xFFE00000 #define HEAD_WRAP_ONE 0x00200000 @@ -285,10 +308,17 @@ #define RING_INVALID 0x00000000 #define RING_WAIT_I8XX (1<<0) /* gen2, PRBx_HEAD */ #define RING_WAIT (1<<11) /* gen3+, PRBx_CTL */ +#define RING_WAIT_SEMAPHORE (1<<10) /* gen6+ */ +#if 0 +#define PRB0_TAIL 0x02030 +#define PRB0_HEAD 0x02034 +#define PRB0_START 0x02038 +#define PRB0_CTL 0x0203c #define PRB1_TAIL 0x02040 /* 915+ only */ #define PRB1_HEAD 0x02044 /* 915+ only */ #define PRB1_START 0x02048 /* 915+ only */ #define PRB1_CTL 0x0204c /* 915+ only */ +#endif #define IPEIR_I965 0x02064 #define IPEHR_I965 0x02068 #define INSTDONE_I965 0x0206c @@ -305,11 +335,42 @@ #define INSTDONE 0x02090 #define NOPID 0x02094 #define HWSTAM 0x02098 +#define VCS_INSTDONE 0x1206C +#define VCS_IPEIR 0x12064 +#define VCS_IPEHR 0x12068 +#define VCS_ACTHD 0x12074 +#define BCS_INSTDONE 0x2206C +#define BCS_IPEIR 0x22064 +#define BCS_IPEHR 0x22068 +#define BCS_ACTHD 0x22074 + +#define ERROR_GEN6 0x040a0 + +/* GM45+ chicken bits -- debug workaround bits that may be required + * for various sorts of correct behavior. The top 16 bits of each are + * the enables for writing to the corresponding low bit. + */ +#define _3D_CHICKEN 0x02084 +#define _3D_CHICKEN2 0x0208c +/* Disables pipelining of read flushes past the SF-WIZ interface. + * Required on all Ironlake steppings according to the B-Spec, but the + * particular danger of not doing so is not specified. + */ +# define _3D_CHICKEN2_WM_READ_PIPELINED (1 << 14) +#define _3D_CHICKEN3 0x02090 #define MI_MODE 0x0209c # define VS_TIMER_DISPATCH (1 << 6) # define MI_FLUSH_ENABLE (1 << 11) +#define GFX_MODE 0x02520 +#define GFX_RUN_LIST_ENABLE (1<<15) +#define GFX_TLB_INVALIDATE_ALWAYS (1<<13) +#define GFX_SURFACE_FAULT_ENABLE (1<<12) +#define GFX_REPLAY_MODE (1<<11) +#define GFX_PSMI_GRANULARITY (1<<10) +#define GFX_PPGTT_ENABLE (1<<9) + #define SCPD0 0x0209c /* 915+ only */ #define IER 0x020a0 #define IIR 0x020a4 @@ -454,6 +515,10 @@ #define GEN6_BLITTER_SYNC_STATUS (1 << 24) #define GEN6_BLITTER_USER_INTERRUPT (1 << 22) +#define GEN6_BLITTER_ECOSKPD 0x221d0 +#define GEN6_BLITTER_LOCK_SHIFT 16 +#define GEN6_BLITTER_FBC_NOTIFY (1<<3) + #define GEN6_BSD_SLEEP_PSMI_CONTROL 0x12050 #define GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK (1 << 16) #define GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE (1 << 0) @@ -461,7 +526,7 @@ #define GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR (1 << 3) #define GEN6_BSD_IMR 0x120a8 -#define GEN6_BSD_IMR_USER_INTERRUPT (1 << 12) +#define GEN6_BSD_USER_INTERRUPT (1 << 12) #define GEN6_BSD_RNCID 0x12198 @@ -541,6 +606,18 @@ #define ILK_DISPLAY_CHICKEN1 0x42000 #define ILK_FBCQ_DIS (1<<22) +#define ILK_PABSTRETCH_DIS (1<<21) + + +/* + * Framebuffer compression for Sandybridge + * + * The following two registers are of type GTTMMADR + */ +#define SNB_DPFC_CTL_SA 0x100100 +#define SNB_CPU_FENCE_ENABLE (1<<29) +#define DPFC_CPU_FENCE_OFFSET 0x100104 + /* * GPIO regs @@ -900,6 +977,8 @@ */ #define MCHBAR_MIRROR_BASE 0x10000 +#define MCHBAR_MIRROR_BASE_SNB 0x140000 + /** 915-945 and GM965 MCH register controlling DRAM channel access */ #define DCC 0x10200 #define DCC_ADDRESSING_MODE_SINGLE_CHANNEL (0 << 0) @@ -1061,9 +1140,50 @@ #define RCBMINAVG 0x111a0 #define RCUPEI 0x111b0 #define RCDNEI 0x111b4 -#define MCHBAR_RENDER_STANDBY 0x111b8 -#define RCX_SW_EXIT (1<<23) -#define RSX_STATUS_MASK 0x00700000 +#define RSTDBYCTL 0x111b8 +#define RS1EN (1<<31) +#define RS2EN (1<<30) +#define RS3EN (1<<29) +#define D3RS3EN (1<<28) /* Display D3 imlies RS3 */ +#define SWPROMORSX (1<<27) /* RSx promotion timers ignored */ +#define RCWAKERW (1<<26) /* Resetwarn from PCH causes wakeup */ +#define DPRSLPVREN (1<<25) /* Fast voltage ramp enable */ +#define GFXTGHYST (1<<24) /* Hysteresis to allow trunk gating */ +#define RCX_SW_EXIT (1<<23) /* Leave RSx and prevent re-entry */ +#define RSX_STATUS_MASK (7<<20) +#define RSX_STATUS_ON (0<<20) +#define RSX_STATUS_RC1 (1<<20) +#define RSX_STATUS_RC1E (2<<20) +#define RSX_STATUS_RS1 (3<<20) +#define RSX_STATUS_RS2 (4<<20) /* aka rc6 */ +#define RSX_STATUS_RSVD (5<<20) /* deep rc6 unsupported on ilk */ +#define RSX_STATUS_RS3 (6<<20) /* rs3 unsupported on ilk */ +#define RSX_STATUS_RSVD2 (7<<20) +#define UWRCRSXE (1<<19) /* wake counter limit prevents rsx */ +#define RSCRP (1<<18) /* rs requests control on rs1/2 reqs */ +#define JRSC (1<<17) /* rsx coupled to cpu c-state */ +#define RS2INC0 (1<<16) /* allow rs2 in cpu c0 */ +#define RS1CONTSAV_MASK (3<<14) +#define RS1CONTSAV_NO_RS1 (0<<14) /* rs1 doesn't save/restore context */ +#define RS1CONTSAV_RSVD (1<<14) +#define RS1CONTSAV_SAVE_RS1 (2<<14) /* rs1 saves context */ +#define RS1CONTSAV_FULL_RS1 (3<<14) /* rs1 saves and restores context */ +#define NORMSLEXLAT_MASK (3<<12) +#define SLOW_RS123 (0<<12) +#define SLOW_RS23 (1<<12) +#define SLOW_RS3 (2<<12) +#define NORMAL_RS123 (3<<12) +#define RCMODE_TIMEOUT (1<<11) /* 0 is eval interval method */ +#define IMPROMOEN (1<<10) /* promo is immediate or delayed until next idle interval (only for timeout method above) */ +#define RCENTSYNC (1<<9) /* rs coupled to cpu c-state (3/6/7) */ +#define STATELOCK (1<<7) /* locked to rs_cstate if 0 */ +#define RS_CSTATE_MASK (3<<4) +#define RS_CSTATE_C367_RS1 (0<<4) +#define RS_CSTATE_C36_RS1_C7_RS2 (1<<4) +#define RS_CSTATE_RSVD (2<<4) +#define RS_CSTATE_C367_RS2 (3<<4) +#define REDSAVES (1<<3) /* no context save if was idle during rs0 */ +#define REDRESTORES (1<<2) /* no restore if was idle during rs0 */ #define VIDCTL 0x111c0 #define VIDSTS 0x111c8 #define VIDSTART 0x111cc /* 8 bits */ @@ -1119,6 +1239,10 @@ #define DDRMPLL1 0X12c20 #define PEG_BAND_GAP_DATA 0x14d68 +#define GEN6_GT_PERF_STATUS 0x145948 +#define GEN6_RP_STATE_LIMITS 0x145994 +#define GEN6_RP_STATE_CAP 0x145998 + /* * Logical Context regs */ @@ -1168,7 +1292,6 @@ #define VTOTAL(pipe) _PIPE(pipe, VTOTAL_A, VTOTAL_B) #define VBLANK(pipe) _PIPE(pipe, VBLANK_A, VBLANK_B) #define VSYNC(pipe) _PIPE(pipe, VSYNC_A, VSYNC_B) -#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC) #define BCLRPAT(pipe) _PIPE(pipe, BCLRPAT_A, BCLRPAT_B) /* VGA port control */ @@ -1430,17 +1553,7 @@ /* Backlight control */ #define BLC_PWM_CTL 0x61254 -#define BACKLIGHT_MODULATION_FREQ_SHIFT (17) #define BLC_PWM_CTL2 0x61250 /* 965+ only */ -#define BLM_COMBINATION_MODE (1 << 30) -/* - * This is the most significant 15 bits of the number of backlight cycles in a - * complete cycle of the modulated backlight control. - * - * The actual value is this field multiplied by two. - */ -#define BACKLIGHT_MODULATION_FREQ_MASK (0x7fff << 17) -#define BLM_LEGACY_MODE (1 << 16) /* * This is the number of cycles out of the backlight modulation cycle for which * the backlight is on. @@ -2182,8 +2295,10 @@ #define PIPE_6BPC (2 << 5) #define PIPE_12BPC (3 << 5) +#define PIPESRC(pipe) _PIPE(pipe, PIPEASRC, PIPEBSRC) #define PIPECONF(pipe) _PIPE(pipe, PIPEACONF, PIPEBCONF) #define PIPEDSL(pipe) _PIPE(pipe, PIPEADSL, PIPEBDSL) +#define PIPEFRAMEPIXEL(pipe) _PIPE(pipe, PIPEAFRAMEPIXEL, PIPEBFRAMEPIXEL) #define DSPARB 0x70030 #define DSPARB_CSTART_MASK (0x7f << 7) @@ -2271,8 +2386,13 @@ /* Memory latency timer register */ #define MLTR_ILK 0x11222 +#define MLTR_WM1_SHIFT 0 +#define MLTR_WM2_SHIFT 8 /* the unit of memory self-refresh latency time is 0.5us */ #define ILK_SRLT_MASK 0x3f +#define ILK_LATENCY(shift) (I915_READ(MLTR_ILK) >> (shift) & ILK_SRLT_MASK) +#define ILK_READ_WM1_LATENCY() ILK_LATENCY(MLTR_WM1_SHIFT) +#define ILK_READ_WM2_LATENCY() ILK_LATENCY(MLTR_WM2_SHIFT) /* define the fifo size on Ironlake */ #define ILK_DISPLAY_FIFO 128 @@ -2291,6 +2411,40 @@ #define ILK_FIFO_LINE_SIZE 64 +/* define the WM info on Sandybridge */ +#define SNB_DISPLAY_FIFO 128 +#define SNB_DISPLAY_MAXWM 0x7f /* bit 16:22 */ +#define SNB_DISPLAY_DFTWM 8 +#define SNB_CURSOR_FIFO 32 +#define SNB_CURSOR_MAXWM 0x1f /* bit 4:0 */ +#define SNB_CURSOR_DFTWM 8 + +#define SNB_DISPLAY_SR_FIFO 512 +#define SNB_DISPLAY_MAX_SRWM 0x1ff /* bit 16:8 */ +#define SNB_DISPLAY_DFT_SRWM 0x3f +#define SNB_CURSOR_SR_FIFO 64 +#define SNB_CURSOR_MAX_SRWM 0x3f /* bit 5:0 */ +#define SNB_CURSOR_DFT_SRWM 8 + +#define SNB_FBC_MAX_SRWM 0xf /* bit 23:20 */ + +#define SNB_FIFO_LINE_SIZE 64 + + +/* the address where we get all kinds of latency value */ +#define SSKPD 0x5d10 +#define SSKPD_WM_MASK 0x3f +#define SSKPD_WM0_SHIFT 0 +#define SSKPD_WM1_SHIFT 8 +#define SSKPD_WM2_SHIFT 16 +#define SSKPD_WM3_SHIFT 24 + +#define SNB_LATENCY(shift) (I915_READ(MCHBAR_MIRROR_BASE_SNB + SSKPD) >> (shift) & SSKPD_WM_MASK) +#define SNB_READ_WM0_LATENCY() SNB_LATENCY(SSKPD_WM0_SHIFT) +#define SNB_READ_WM1_LATENCY() SNB_LATENCY(SSKPD_WM1_SHIFT) +#define SNB_READ_WM2_LATENCY() SNB_LATENCY(SSKPD_WM2_SHIFT) +#define SNB_READ_WM3_LATENCY() SNB_LATENCY(SSKPD_WM3_SHIFT) + /* * The two pipe frame counter registers are not synchronized, so * reading a stable value is somewhat tricky. The following code @@ -2351,6 +2505,10 @@ #define CURBBASE 0x700c4 #define CURBPOS 0x700c8 +#define CURCNTR(pipe) _PIPE(pipe, CURACNTR, CURBCNTR) +#define CURBASE(pipe) _PIPE(pipe, CURABASE, CURBBASE) +#define CURPOS(pipe) _PIPE(pipe, CURAPOS, CURBPOS) + /* Display A control */ #define DSPACNTR 0x70180 #define DISPLAY_PLANE_ENABLE (1<<31) @@ -2464,6 +2622,8 @@ #define DISPLAY_PORT_PLL_BIOS_2 0x46014 #define PCH_DSPCLK_GATE_D 0x42020 +# define DPFCUNIT_CLOCK_GATE_DISABLE (1 << 9) +# define DPFCRUNIT_CLOCK_GATE_DISABLE (1 << 8) # define DPFDUNIT_CLOCK_GATE_DISABLE (1 << 7) # define DPARBUNIT_CLOCK_GATE_DISABLE (1 << 5) @@ -2589,6 +2749,8 @@ #define GTIER 0x4401c #define ILK_DISPLAY_CHICKEN2 0x42004 +/* Required on all Ironlake and Sandybridge according to the B-Spec. */ +#define ILK_ELPIN_409_SELECT (1 << 25) #define ILK_DPARB_GATE (1<<22) #define ILK_VSDPFD_FULL (1<<21) #define ILK_DISPLAY_CHICKEN_FUSES 0x42014 @@ -2600,6 +2762,8 @@ #define ILK_DESKTOP (1<<23) #define ILK_DSPCLK_GATE 0x42020 #define ILK_DPARB_CLK_GATE (1<<5) +#define ILK_DPFD_CLK_GATE (1<<7) + /* According to spec this bit 7/8/9 of 0x42020 should be set to enable FBC */ #define ILK_CLK_FBC (1<<7) #define ILK_DPFC_DIS1 (1<<8) @@ -2612,12 +2776,41 @@ /* PCH */ /* south display engine interrupt */ +#define SDE_AUDIO_POWER_D (1 << 27) +#define SDE_AUDIO_POWER_C (1 << 26) +#define SDE_AUDIO_POWER_B (1 << 25) +#define SDE_AUDIO_POWER_SHIFT (25) +#define SDE_AUDIO_POWER_MASK (7 << SDE_AUDIO_POWER_SHIFT) +#define SDE_GMBUS (1 << 24) +#define SDE_AUDIO_HDCP_TRANSB (1 << 23) +#define SDE_AUDIO_HDCP_TRANSA (1 << 22) +#define SDE_AUDIO_HDCP_MASK (3 << 22) +#define SDE_AUDIO_TRANSB (1 << 21) +#define SDE_AUDIO_TRANSA (1 << 20) +#define SDE_AUDIO_TRANS_MASK (3 << 20) +#define SDE_POISON (1 << 19) +/* 18 reserved */ +#define SDE_FDI_RXB (1 << 17) +#define SDE_FDI_RXA (1 << 16) +#define SDE_FDI_MASK (3 << 16) +#define SDE_AUXD (1 << 15) +#define SDE_AUXC (1 << 14) +#define SDE_AUXB (1 << 13) +#define SDE_AUX_MASK (7 << 13) +/* 12 reserved */ #define SDE_CRT_HOTPLUG (1 << 11) #define SDE_PORTD_HOTPLUG (1 << 10) #define SDE_PORTC_HOTPLUG (1 << 9) #define SDE_PORTB_HOTPLUG (1 << 8) #define SDE_SDVOB_HOTPLUG (1 << 6) #define SDE_HOTPLUG_MASK (0xf << 8) +#define SDE_TRANSB_CRC_DONE (1 << 5) +#define SDE_TRANSB_CRC_ERR (1 << 4) +#define SDE_TRANSB_FIFO_UNDER (1 << 3) +#define SDE_TRANSA_CRC_DONE (1 << 2) +#define SDE_TRANSA_CRC_ERR (1 << 1) +#define SDE_TRANSA_FIFO_UNDER (1 << 0) +#define SDE_TRANS_MASK (0x3f) /* CPT */ #define SDE_CRT_HOTPLUG_CPT (1 << 19) #define SDE_PORTD_HOTPLUG_CPT (1 << 23) @@ -2679,6 +2872,7 @@ #define PCH_DPLL(pipe) _PIPE(pipe, PCH_DPLL_A, PCH_DPLL_B) #define PCH_FPA0 0xc6040 +#define FP_CB_TUNE (0x3<<22) #define PCH_FPA1 0xc6044 #define PCH_FPB0 0xc6048 #define PCH_FPB1 0xc604c @@ -3057,10 +3251,74 @@ #define EDP_LINK_TRAIN_600MV_3_5DB_SNB_A (0x01<<22) #define EDP_LINK_TRAIN_800MV_0DB_SNB_A (0x0<<22) /* SNB B-stepping */ -#define EDP_LINK_TRAIN_400MV_0DB_SNB_B (0x0<<22) -#define EDP_LINK_TRAIN_400MV_6DB_SNB_B (0x3a<<22) -#define EDP_LINK_TRAIN_600MV_3_5DB_SNB_B (0x39<<22) -#define EDP_LINK_TRAIN_800MV_0DB_SNB_B (0x38<<22) +#define EDP_LINK_TRAIN_400_600MV_0DB_SNB_B (0x0<<22) +#define EDP_LINK_TRAIN_400MV_3_5DB_SNB_B (0x1<<22) +#define EDP_LINK_TRAIN_400_600MV_6DB_SNB_B (0x3a<<22) +#define EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B (0x39<<22) +#define EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B (0x38<<22) #define EDP_LINK_TRAIN_VOL_EMP_MASK_SNB (0x3f<<22) +#define FORCEWAKE 0xA18C +#define FORCEWAKE_ACK 0x130090 + +#define GEN6_RPNSWREQ 0xA008 +#define GEN6_TURBO_DISABLE (1<<31) +#define GEN6_FREQUENCY(x) ((x)<<25) +#define GEN6_OFFSET(x) ((x)<<19) +#define GEN6_AGGRESSIVE_TURBO (0<<15) +#define GEN6_RC_VIDEO_FREQ 0xA00C +#define GEN6_RC_CONTROL 0xA090 +#define GEN6_RC_CTL_RC6pp_ENABLE (1<<16) +#define GEN6_RC_CTL_RC6p_ENABLE (1<<17) +#define GEN6_RC_CTL_RC6_ENABLE (1<<18) +#define GEN6_RC_CTL_RC1e_ENABLE (1<<20) +#define GEN6_RC_CTL_RC7_ENABLE (1<<22) +#define GEN6_RC_CTL_EI_MODE(x) ((x)<<27) +#define GEN6_RC_CTL_HW_ENABLE (1<<31) +#define GEN6_RP_DOWN_TIMEOUT 0xA010 +#define GEN6_RP_INTERRUPT_LIMITS 0xA014 +#define GEN6_RPSTAT1 0xA01C +#define GEN6_RP_CONTROL 0xA024 +#define GEN6_RP_MEDIA_TURBO (1<<11) +#define GEN6_RP_USE_NORMAL_FREQ (1<<9) +#define GEN6_RP_MEDIA_IS_GFX (1<<8) +#define GEN6_RP_ENABLE (1<<7) +#define GEN6_RP_UP_BUSY_MAX (0x2<<3) +#define GEN6_RP_DOWN_BUSY_MIN (0x2<<0) +#define GEN6_RP_UP_THRESHOLD 0xA02C +#define GEN6_RP_DOWN_THRESHOLD 0xA030 +#define GEN6_RP_UP_EI 0xA068 +#define GEN6_RP_DOWN_EI 0xA06C +#define GEN6_RP_IDLE_HYSTERSIS 0xA070 +#define GEN6_RC_STATE 0xA094 +#define GEN6_RC1_WAKE_RATE_LIMIT 0xA098 +#define GEN6_RC6_WAKE_RATE_LIMIT 0xA09C +#define GEN6_RC6pp_WAKE_RATE_LIMIT 0xA0A0 +#define GEN6_RC_EVALUATION_INTERVAL 0xA0A8 +#define GEN6_RC_IDLE_HYSTERSIS 0xA0AC +#define GEN6_RC_SLEEP 0xA0B0 +#define GEN6_RC1e_THRESHOLD 0xA0B4 +#define GEN6_RC6_THRESHOLD 0xA0B8 +#define GEN6_RC6p_THRESHOLD 0xA0BC +#define GEN6_RC6pp_THRESHOLD 0xA0C0 +#define GEN6_PMINTRMSK 0xA168 + +#define GEN6_PMISR 0x44020 +#define GEN6_PMIMR 0x44024 +#define GEN6_PMIIR 0x44028 +#define GEN6_PMIER 0x4402C +#define GEN6_PM_MBOX_EVENT (1<<25) +#define GEN6_PM_THERMAL_EVENT (1<<24) +#define GEN6_PM_RP_DOWN_TIMEOUT (1<<6) +#define GEN6_PM_RP_UP_THRESHOLD (1<<5) +#define GEN6_PM_RP_DOWN_THRESHOLD (1<<4) +#define GEN6_PM_RP_UP_EI_EXPIRED (1<<2) +#define GEN6_PM_RP_DOWN_EI_EXPIRED (1<<1) + +#define GEN6_PCODE_MAILBOX 0x138124 +#define GEN6_PCODE_READY (1<<31) +#define GEN6_READ_OC_PARAMS 0xc +#define GEN6_PCODE_WRITE_MIN_FREQ_TABLE 0x9 +#define GEN6_PCODE_DATA 0x138128 + #endif /* _I915_REG_H_ */ diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c index 42729d25da58..0521ecf26017 100644 --- a/drivers/gpu/drm/i915/i915_suspend.c +++ b/drivers/gpu/drm/i915/i915_suspend.c @@ -235,6 +235,7 @@ static void i915_restore_vga(struct drm_device *dev) static void i915_save_modeset_reg(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + int i; if (drm_core_check_feature(dev, DRIVER_MODESET)) return; @@ -367,6 +368,28 @@ static void i915_save_modeset_reg(struct drm_device *dev) } i915_save_palette(dev, PIPE_B); dev_priv->savePIPEBSTAT = I915_READ(PIPEBSTAT); + + /* Fences */ + switch (INTEL_INFO(dev)->gen) { + case 6: + for (i = 0; i < 16; i++) + dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8)); + break; + case 5: + case 4: + for (i = 0; i < 16; i++) + dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8)); + break; + case 3: + if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) + for (i = 0; i < 8; i++) + dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4)); + case 2: + for (i = 0; i < 8; i++) + dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4)); + break; + } + return; } @@ -375,10 +398,33 @@ static void i915_restore_modeset_reg(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; int dpll_a_reg, fpa0_reg, fpa1_reg; int dpll_b_reg, fpb0_reg, fpb1_reg; + int i; if (drm_core_check_feature(dev, DRIVER_MODESET)) return; + /* Fences */ + switch (INTEL_INFO(dev)->gen) { + case 6: + for (i = 0; i < 16; i++) + I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]); + break; + case 5: + case 4: + for (i = 0; i < 16; i++) + I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]); + break; + case 3: + case 2: + if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) + for (i = 0; i < 8; i++) + I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]); + for (i = 0; i < 8; i++) + I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]); + break; + } + + if (HAS_PCH_SPLIT(dev)) { dpll_a_reg = PCH_DPLL_A; dpll_b_reg = PCH_DPLL_B; @@ -694,7 +740,7 @@ void i915_restore_display(struct drm_device *dev) I915_WRITE(PCH_PP_OFF_DELAYS, dev_priv->savePP_OFF_DELAYS); I915_WRITE(PCH_PP_DIVISOR, dev_priv->savePP_DIVISOR); I915_WRITE(PCH_PP_CONTROL, dev_priv->savePP_CONTROL); - I915_WRITE(MCHBAR_RENDER_STANDBY, + I915_WRITE(RSTDBYCTL, dev_priv->saveMCHBAR_RENDER_STANDBY); } else { I915_WRITE(PFIT_PGM_RATIOS, dev_priv->savePFIT_PGM_RATIOS); @@ -765,14 +811,16 @@ int i915_save_state(struct drm_device *dev) dev_priv->saveFDI_RXA_IMR = I915_READ(FDI_RXA_IMR); dev_priv->saveFDI_RXB_IMR = I915_READ(FDI_RXB_IMR); dev_priv->saveMCHBAR_RENDER_STANDBY = - I915_READ(MCHBAR_RENDER_STANDBY); + I915_READ(RSTDBYCTL); } else { dev_priv->saveIER = I915_READ(IER); dev_priv->saveIMR = I915_READ(IMR); } - if (HAS_PCH_SPLIT(dev)) + if (IS_IRONLAKE_M(dev)) ironlake_disable_drps(dev); + if (IS_GEN6(dev)) + gen6_disable_rps(dev); /* Cache mode state */ dev_priv->saveCACHE_MODE_0 = I915_READ(CACHE_MODE_0); @@ -788,28 +836,6 @@ int i915_save_state(struct drm_device *dev) for (i = 0; i < 3; i++) dev_priv->saveSWF2[i] = I915_READ(SWF30 + (i << 2)); - /* Fences */ - switch (INTEL_INFO(dev)->gen) { - case 6: - for (i = 0; i < 16; i++) - dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_SANDYBRIDGE_0 + (i * 8)); - break; - case 5: - case 4: - for (i = 0; i < 16; i++) - dev_priv->saveFENCE[i] = I915_READ64(FENCE_REG_965_0 + (i * 8)); - break; - case 3: - if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) - for (i = 0; i < 8; i++) - dev_priv->saveFENCE[i+8] = I915_READ(FENCE_REG_945_8 + (i * 4)); - case 2: - for (i = 0; i < 8; i++) - dev_priv->saveFENCE[i] = I915_READ(FENCE_REG_830_0 + (i * 4)); - break; - - } - return 0; } @@ -823,27 +849,6 @@ int i915_restore_state(struct drm_device *dev) /* Hardware status page */ I915_WRITE(HWS_PGA, dev_priv->saveHWS); - /* Fences */ - switch (INTEL_INFO(dev)->gen) { - case 6: - for (i = 0; i < 16; i++) - I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), dev_priv->saveFENCE[i]); - break; - case 5: - case 4: - for (i = 0; i < 16; i++) - I915_WRITE64(FENCE_REG_965_0 + (i * 8), dev_priv->saveFENCE[i]); - break; - case 3: - case 2: - if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) - for (i = 0; i < 8; i++) - I915_WRITE(FENCE_REG_945_8 + (i * 4), dev_priv->saveFENCE[i+8]); - for (i = 0; i < 8; i++) - I915_WRITE(FENCE_REG_830_0 + (i * 4), dev_priv->saveFENCE[i]); - break; - } - i915_restore_display(dev); /* Interrupt state */ @@ -860,13 +865,16 @@ int i915_restore_state(struct drm_device *dev) } /* Clock gating state */ - intel_init_clock_gating(dev); + intel_enable_clock_gating(dev); - if (HAS_PCH_SPLIT(dev)) { + if (IS_IRONLAKE_M(dev)) { ironlake_enable_drps(dev); intel_init_emon(dev); } + if (IS_GEN6(dev)) + gen6_enable_rps(dev_priv); + /* Cache mode state */ I915_WRITE (CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000); diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h index fea97a21cc14..7f0fc3ed61aa 100644 --- a/drivers/gpu/drm/i915/i915_trace.h +++ b/drivers/gpu/drm/i915/i915_trace.h @@ -6,6 +6,7 @@ #include <linux/tracepoint.h> #include <drm/drmP.h> +#include "i915_drv.h" #undef TRACE_SYSTEM #define TRACE_SYSTEM i915 @@ -16,18 +17,18 @@ TRACE_EVENT(i915_gem_object_create, - TP_PROTO(struct drm_gem_object *obj), + TP_PROTO(struct drm_i915_gem_object *obj), TP_ARGS(obj), TP_STRUCT__entry( - __field(struct drm_gem_object *, obj) + __field(struct drm_i915_gem_object *, obj) __field(u32, size) ), TP_fast_assign( __entry->obj = obj; - __entry->size = obj->size; + __entry->size = obj->base.size; ), TP_printk("obj=%p, size=%u", __entry->obj, __entry->size) @@ -35,40 +36,43 @@ TRACE_EVENT(i915_gem_object_create, TRACE_EVENT(i915_gem_object_bind, - TP_PROTO(struct drm_gem_object *obj, u32 gtt_offset), + TP_PROTO(struct drm_i915_gem_object *obj, u32 gtt_offset, bool mappable), - TP_ARGS(obj, gtt_offset), + TP_ARGS(obj, gtt_offset, mappable), TP_STRUCT__entry( - __field(struct drm_gem_object *, obj) + __field(struct drm_i915_gem_object *, obj) __field(u32, gtt_offset) + __field(bool, mappable) ), TP_fast_assign( __entry->obj = obj; __entry->gtt_offset = gtt_offset; + __entry->mappable = mappable; ), - TP_printk("obj=%p, gtt_offset=%08x", - __entry->obj, __entry->gtt_offset) + TP_printk("obj=%p, gtt_offset=%08x%s", + __entry->obj, __entry->gtt_offset, + __entry->mappable ? ", mappable" : "") ); TRACE_EVENT(i915_gem_object_change_domain, - TP_PROTO(struct drm_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain), + TP_PROTO(struct drm_i915_gem_object *obj, uint32_t old_read_domains, uint32_t old_write_domain), TP_ARGS(obj, old_read_domains, old_write_domain), TP_STRUCT__entry( - __field(struct drm_gem_object *, obj) + __field(struct drm_i915_gem_object *, obj) __field(u32, read_domains) __field(u32, write_domain) ), TP_fast_assign( __entry->obj = obj; - __entry->read_domains = obj->read_domains | (old_read_domains << 16); - __entry->write_domain = obj->write_domain | (old_write_domain << 16); + __entry->read_domains = obj->base.read_domains | (old_read_domains << 16); + __entry->write_domain = obj->base.write_domain | (old_write_domain << 16); ), TP_printk("obj=%p, read=%04x, write=%04x", @@ -76,36 +80,14 @@ TRACE_EVENT(i915_gem_object_change_domain, __entry->read_domains, __entry->write_domain) ); -TRACE_EVENT(i915_gem_object_get_fence, - - TP_PROTO(struct drm_gem_object *obj, int fence, int tiling_mode), - - TP_ARGS(obj, fence, tiling_mode), - - TP_STRUCT__entry( - __field(struct drm_gem_object *, obj) - __field(int, fence) - __field(int, tiling_mode) - ), - - TP_fast_assign( - __entry->obj = obj; - __entry->fence = fence; - __entry->tiling_mode = tiling_mode; - ), - - TP_printk("obj=%p, fence=%d, tiling=%d", - __entry->obj, __entry->fence, __entry->tiling_mode) -); - DECLARE_EVENT_CLASS(i915_gem_object, - TP_PROTO(struct drm_gem_object *obj), + TP_PROTO(struct drm_i915_gem_object *obj), TP_ARGS(obj), TP_STRUCT__entry( - __field(struct drm_gem_object *, obj) + __field(struct drm_i915_gem_object *, obj) ), TP_fast_assign( @@ -117,21 +99,21 @@ DECLARE_EVENT_CLASS(i915_gem_object, DEFINE_EVENT(i915_gem_object, i915_gem_object_clflush, - TP_PROTO(struct drm_gem_object *obj), + TP_PROTO(struct drm_i915_gem_object *obj), TP_ARGS(obj) ); DEFINE_EVENT(i915_gem_object, i915_gem_object_unbind, - TP_PROTO(struct drm_gem_object *obj), + TP_PROTO(struct drm_i915_gem_object *obj), TP_ARGS(obj) ); DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy, - TP_PROTO(struct drm_gem_object *obj), + TP_PROTO(struct drm_i915_gem_object *obj), TP_ARGS(obj) ); @@ -263,13 +245,13 @@ DEFINE_EVENT(i915_ring, i915_ring_wait_end, ); TRACE_EVENT(i915_flip_request, - TP_PROTO(int plane, struct drm_gem_object *obj), + TP_PROTO(int plane, struct drm_i915_gem_object *obj), TP_ARGS(plane, obj), TP_STRUCT__entry( __field(int, plane) - __field(struct drm_gem_object *, obj) + __field(struct drm_i915_gem_object *, obj) ), TP_fast_assign( @@ -281,13 +263,13 @@ TRACE_EVENT(i915_flip_request, ); TRACE_EVENT(i915_flip_complete, - TP_PROTO(int plane, struct drm_gem_object *obj), + TP_PROTO(int plane, struct drm_i915_gem_object *obj), TP_ARGS(plane, obj), TP_STRUCT__entry( __field(int, plane) - __field(struct drm_gem_object *, obj) + __field(struct drm_i915_gem_object *, obj) ), TP_fast_assign( @@ -298,6 +280,29 @@ TRACE_EVENT(i915_flip_complete, TP_printk("plane=%d, obj=%p", __entry->plane, __entry->obj) ); +TRACE_EVENT(i915_reg_rw, + TP_PROTO(int cmd, uint32_t reg, uint64_t val, int len), + + TP_ARGS(cmd, reg, val, len), + + TP_STRUCT__entry( + __field(int, cmd) + __field(uint32_t, reg) + __field(uint64_t, val) + __field(int, len) + ), + + TP_fast_assign( + __entry->cmd = cmd; + __entry->reg = reg; + __entry->val = (uint64_t)val; + __entry->len = len; + ), + + TP_printk("cmd=%c, reg=0x%x, val=0x%llx, len=%d", + __entry->cmd, __entry->reg, __entry->val, __entry->len) +); + #endif /* _I915_TRACE_H_ */ /* This part must be outside protection */ diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c index b0b1200ed650..0b44956c336b 100644 --- a/drivers/gpu/drm/i915/intel_bios.c +++ b/drivers/gpu/drm/i915/intel_bios.c @@ -264,17 +264,12 @@ parse_general_features(struct drm_i915_private *dev_priv, dev_priv->int_crt_support = general->int_crt_support; dev_priv->lvds_use_ssc = general->enable_ssc; - if (dev_priv->lvds_use_ssc) { - if (IS_I85X(dev)) - dev_priv->lvds_ssc_freq = - general->ssc_freq ? 66 : 48; - else if (IS_GEN5(dev) || IS_GEN6(dev)) - dev_priv->lvds_ssc_freq = - general->ssc_freq ? 100 : 120; - else - dev_priv->lvds_ssc_freq = - general->ssc_freq ? 100 : 96; - } + if (IS_I85X(dev)) + dev_priv->lvds_ssc_freq = general->ssc_freq ? 66 : 48; + else if (IS_GEN5(dev) || IS_GEN6(dev)) + dev_priv->lvds_ssc_freq = general->ssc_freq ? 100 : 120; + else + dev_priv->lvds_ssc_freq = general->ssc_freq ? 100 : 96; } } diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c index 8df574316063..8a77ff4a7237 100644 --- a/drivers/gpu/drm/i915/intel_crt.c +++ b/drivers/gpu/drm/i915/intel_crt.c @@ -30,6 +30,7 @@ #include "drm.h" #include "drm_crtc.h" #include "drm_crtc_helper.h" +#include "drm_edid.h" #include "intel_drv.h" #include "i915_drm.h" #include "i915_drv.h" @@ -287,8 +288,9 @@ static bool intel_crt_ddc_probe(struct drm_i915_private *dev_priv, int ddc_bus) return i2c_transfer(&dev_priv->gmbus[ddc_bus].adapter, msgs, 1) == 1; } -static bool intel_crt_detect_ddc(struct intel_crt *crt) +static bool intel_crt_detect_ddc(struct drm_connector *connector) { + struct intel_crt *crt = intel_attached_crt(connector); struct drm_i915_private *dev_priv = crt->base.base.dev->dev_private; /* CRT should always be at 0, but check anyway */ @@ -301,8 +303,26 @@ static bool intel_crt_detect_ddc(struct intel_crt *crt) } if (intel_ddc_probe(&crt->base, dev_priv->crt_ddc_pin)) { - DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n"); - return true; + struct edid *edid; + bool is_digital = false; + + edid = drm_get_edid(connector, + &dev_priv->gmbus[dev_priv->crt_ddc_pin].adapter); + /* + * This may be a DVI-I connector with a shared DDC + * link between analog and digital outputs, so we + * have to check the EDID input spec of the attached device. + */ + if (edid != NULL) { + is_digital = edid->input & DRM_EDID_INPUT_DIGITAL; + connector->display_info.raw_edid = NULL; + kfree(edid); + } + + if (!is_digital) { + DRM_DEBUG_KMS("CRT detected via DDC:0x50 [EDID]\n"); + return true; + } } return false; @@ -458,7 +478,7 @@ intel_crt_detect(struct drm_connector *connector, bool force) } } - if (intel_crt_detect_ddc(crt)) + if (intel_crt_detect_ddc(connector)) return connector_status_connected; if (!force) @@ -472,7 +492,7 @@ intel_crt_detect(struct drm_connector *connector, bool force) crtc = intel_get_load_detect_pipe(&crt->base, connector, NULL, &dpms_mode); if (crtc) { - if (intel_crt_detect_ddc(crt)) + if (intel_crt_detect_ddc(connector)) status = connector_status_connected; else status = intel_crt_load_detect(crtc, crt); @@ -515,6 +535,15 @@ static int intel_crt_set_property(struct drm_connector *connector, return 0; } +static void intel_crt_reset(struct drm_connector *connector) +{ + struct drm_device *dev = connector->dev; + struct intel_crt *crt = intel_attached_crt(connector); + + if (HAS_PCH_SPLIT(dev)) + crt->force_hotplug_required = 1; +} + /* * Routines for controlling stuff on the analog port */ @@ -528,6 +557,7 @@ static const struct drm_encoder_helper_funcs intel_crt_helper_funcs = { }; static const struct drm_connector_funcs intel_crt_connector_funcs = { + .reset = intel_crt_reset, .dpms = drm_helper_connector_dpms, .detect = intel_crt_detect, .fill_modes = drm_helper_probe_single_connector_modes, diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c index fca523288aca..e79b25bbee6c 100644 --- a/drivers/gpu/drm/i915/intel_display.c +++ b/drivers/gpu/drm/i915/intel_display.c @@ -642,26 +642,23 @@ static const intel_limit_t intel_limits_ironlake_display_port = { .find_pll = intel_find_pll_ironlake_dp, }; -static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc) +static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc, + int refclk) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; const intel_limit_t *limit; - int refclk = 120; if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) { - if (dev_priv->lvds_use_ssc && dev_priv->lvds_ssc_freq == 100) - refclk = 100; - if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP) { /* LVDS dual channel */ - if (refclk == 100) + if (refclk == 100000) limit = &intel_limits_ironlake_dual_lvds_100m; else limit = &intel_limits_ironlake_dual_lvds; } else { - if (refclk == 100) + if (refclk == 100000) limit = &intel_limits_ironlake_single_lvds_100m; else limit = &intel_limits_ironlake_single_lvds; @@ -702,13 +699,13 @@ static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc) return limit; } -static const intel_limit_t *intel_limit(struct drm_crtc *crtc) +static const intel_limit_t *intel_limit(struct drm_crtc *crtc, int refclk) { struct drm_device *dev = crtc->dev; const intel_limit_t *limit; if (HAS_PCH_SPLIT(dev)) - limit = intel_ironlake_limit(crtc); + limit = intel_ironlake_limit(crtc, refclk); else if (IS_G4X(dev)) { limit = intel_g4x_limit(crtc); } else if (IS_PINEVIEW(dev)) { @@ -773,11 +770,10 @@ bool intel_pipe_has_type(struct drm_crtc *crtc, int type) * the given connectors. */ -static bool intel_PLL_is_valid(struct drm_crtc *crtc, intel_clock_t *clock) +static bool intel_PLL_is_valid(struct drm_device *dev, + const intel_limit_t *limit, + const intel_clock_t *clock) { - const intel_limit_t *limit = intel_limit (crtc); - struct drm_device *dev = crtc->dev; - if (clock->p1 < limit->p1.min || limit->p1.max < clock->p1) INTELPllInvalid ("p1 out of range\n"); if (clock->p < limit->p.min || limit->p.max < clock->p) @@ -849,8 +845,8 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc, int this_err; intel_clock(dev, refclk, &clock); - - if (!intel_PLL_is_valid(crtc, &clock)) + if (!intel_PLL_is_valid(dev, limit, + &clock)) continue; this_err = abs(clock.dot - target); @@ -912,9 +908,11 @@ intel_g4x_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc, int this_err; intel_clock(dev, refclk, &clock); - if (!intel_PLL_is_valid(crtc, &clock)) + if (!intel_PLL_is_valid(dev, limit, + &clock)) continue; - this_err = abs(clock.dot - target) ; + + this_err = abs(clock.dot - target); if (this_err < err_most) { *best_clock = clock; err_most = this_err; @@ -1066,13 +1064,13 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval) struct drm_i915_private *dev_priv = dev->dev_private; struct drm_framebuffer *fb = crtc->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); + struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int plane, i; u32 fbc_ctl, fbc_ctl2; if (fb->pitch == dev_priv->cfb_pitch && - obj_priv->fence_reg == dev_priv->cfb_fence && + obj->fence_reg == dev_priv->cfb_fence && intel_crtc->plane == dev_priv->cfb_plane && I915_READ(FBC_CONTROL) & FBC_CTL_EN) return; @@ -1086,7 +1084,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval) /* FBC_CTL wants 64B units */ dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; - dev_priv->cfb_fence = obj_priv->fence_reg; + dev_priv->cfb_fence = obj->fence_reg; dev_priv->cfb_plane = intel_crtc->plane; plane = dev_priv->cfb_plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB; @@ -1096,7 +1094,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval) /* Set it up... */ fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | plane; - if (obj_priv->tiling_mode != I915_TILING_NONE) + if (obj->tiling_mode != I915_TILING_NONE) fbc_ctl2 |= FBC_CTL_CPU_FENCE; I915_WRITE(FBC_CONTROL2, fbc_ctl2); I915_WRITE(FBC_FENCE_OFF, crtc->y); @@ -1107,7 +1105,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval) fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */ fbc_ctl |= (dev_priv->cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT; fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT; - if (obj_priv->tiling_mode != I915_TILING_NONE) + if (obj->tiling_mode != I915_TILING_NONE) fbc_ctl |= dev_priv->cfb_fence; I915_WRITE(FBC_CONTROL, fbc_ctl); @@ -1150,7 +1148,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval) struct drm_i915_private *dev_priv = dev->dev_private; struct drm_framebuffer *fb = crtc->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); + struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; unsigned long stall_watermark = 200; @@ -1159,7 +1157,7 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval) dpfc_ctl = I915_READ(DPFC_CONTROL); if (dpfc_ctl & DPFC_CTL_EN) { if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 && - dev_priv->cfb_fence == obj_priv->fence_reg && + dev_priv->cfb_fence == obj->fence_reg && dev_priv->cfb_plane == intel_crtc->plane && dev_priv->cfb_y == crtc->y) return; @@ -1170,12 +1168,12 @@ static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval) } dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; - dev_priv->cfb_fence = obj_priv->fence_reg; + dev_priv->cfb_fence = obj->fence_reg; dev_priv->cfb_plane = intel_crtc->plane; dev_priv->cfb_y = crtc->y; dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X; - if (obj_priv->tiling_mode != I915_TILING_NONE) { + if (obj->tiling_mode != I915_TILING_NONE) { dpfc_ctl |= DPFC_CTL_FENCE_EN | dev_priv->cfb_fence; I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY); } else { @@ -1215,13 +1213,33 @@ static bool g4x_fbc_enabled(struct drm_device *dev) return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN; } +static void sandybridge_blit_fbc_update(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 blt_ecoskpd; + + /* Make sure blitter notifies FBC of writes */ + __gen6_force_wake_get(dev_priv); + blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD); + blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY << + GEN6_BLITTER_LOCK_SHIFT; + I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); + blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY; + I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); + blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY << + GEN6_BLITTER_LOCK_SHIFT); + I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd); + POSTING_READ(GEN6_BLITTER_ECOSKPD); + __gen6_force_wake_put(dev_priv); +} + static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct drm_framebuffer *fb = crtc->fb; struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct drm_i915_gem_object *obj_priv = to_intel_bo(intel_fb->obj); + struct drm_i915_gem_object *obj = intel_fb->obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB; unsigned long stall_watermark = 200; @@ -1230,9 +1248,9 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval) dpfc_ctl = I915_READ(ILK_DPFC_CONTROL); if (dpfc_ctl & DPFC_CTL_EN) { if (dev_priv->cfb_pitch == dev_priv->cfb_pitch / 64 - 1 && - dev_priv->cfb_fence == obj_priv->fence_reg && + dev_priv->cfb_fence == obj->fence_reg && dev_priv->cfb_plane == intel_crtc->plane && - dev_priv->cfb_offset == obj_priv->gtt_offset && + dev_priv->cfb_offset == obj->gtt_offset && dev_priv->cfb_y == crtc->y) return; @@ -1242,14 +1260,14 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval) } dev_priv->cfb_pitch = (dev_priv->cfb_pitch / 64) - 1; - dev_priv->cfb_fence = obj_priv->fence_reg; + dev_priv->cfb_fence = obj->fence_reg; dev_priv->cfb_plane = intel_crtc->plane; - dev_priv->cfb_offset = obj_priv->gtt_offset; + dev_priv->cfb_offset = obj->gtt_offset; dev_priv->cfb_y = crtc->y; dpfc_ctl &= DPFC_RESERVED; dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X); - if (obj_priv->tiling_mode != I915_TILING_NONE) { + if (obj->tiling_mode != I915_TILING_NONE) { dpfc_ctl |= (DPFC_CTL_FENCE_EN | dev_priv->cfb_fence); I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY); } else { @@ -1260,10 +1278,17 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval) (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) | (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT)); I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y); - I915_WRITE(ILK_FBC_RT_BASE, obj_priv->gtt_offset | ILK_FBC_RT_VALID); + I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID); /* enable it... */ I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN); + if (IS_GEN6(dev)) { + I915_WRITE(SNB_DPFC_CTL_SA, + SNB_CPU_FENCE_ENABLE | dev_priv->cfb_fence); + I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y); + sandybridge_blit_fbc_update(dev); + } + DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane); } @@ -1345,7 +1370,7 @@ static void intel_update_fbc(struct drm_device *dev) struct intel_crtc *intel_crtc; struct drm_framebuffer *fb; struct intel_framebuffer *intel_fb; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; DRM_DEBUG_KMS("\n"); @@ -1384,9 +1409,9 @@ static void intel_update_fbc(struct drm_device *dev) intel_crtc = to_intel_crtc(crtc); fb = crtc->fb; intel_fb = to_intel_framebuffer(fb); - obj_priv = to_intel_bo(intel_fb->obj); + obj = intel_fb->obj; - if (intel_fb->obj->size > dev_priv->cfb_size) { + if (intel_fb->obj->base.size > dev_priv->cfb_size) { DRM_DEBUG_KMS("framebuffer too large, disabling " "compression\n"); dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL; @@ -1410,7 +1435,7 @@ static void intel_update_fbc(struct drm_device *dev) dev_priv->no_fbc_reason = FBC_BAD_PLANE; goto out_disable; } - if (obj_priv->tiling_mode != I915_TILING_X) { + if (obj->tiling_mode != I915_TILING_X) { DRM_DEBUG_KMS("framebuffer not tiled, disabling compression\n"); dev_priv->no_fbc_reason = FBC_NOT_TILED; goto out_disable; @@ -1433,14 +1458,13 @@ out_disable: int intel_pin_and_fence_fb_obj(struct drm_device *dev, - struct drm_gem_object *obj, - bool pipelined) + struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); u32 alignment; int ret; - switch (obj_priv->tiling_mode) { + switch (obj->tiling_mode) { case I915_TILING_NONE: if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) alignment = 128 * 1024; @@ -1461,7 +1485,7 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev, BUG(); } - ret = i915_gem_object_pin(obj, alignment); + ret = i915_gem_object_pin(obj, alignment, true); if (ret) return ret; @@ -1474,9 +1498,8 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev, * framebuffer compression. For simplicity, we always install * a fence as the cost is not that onerous. */ - if (obj_priv->fence_reg == I915_FENCE_REG_NONE && - obj_priv->tiling_mode != I915_TILING_NONE) { - ret = i915_gem_object_get_fence_reg(obj, false); + if (obj->tiling_mode != I915_TILING_NONE) { + ret = i915_gem_object_get_fence(obj, pipelined, false); if (ret) goto err_unpin; } @@ -1497,8 +1520,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_framebuffer *intel_fb; - struct drm_i915_gem_object *obj_priv; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; int plane = intel_crtc->plane; unsigned long Start, Offset; u32 dspcntr; @@ -1515,7 +1537,6 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, intel_fb = to_intel_framebuffer(fb); obj = intel_fb->obj; - obj_priv = to_intel_bo(obj); reg = DSPCNTR(plane); dspcntr = I915_READ(reg); @@ -1540,7 +1561,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, return -EINVAL; } if (INTEL_INFO(dev)->gen >= 4) { - if (obj_priv->tiling_mode != I915_TILING_NONE) + if (obj->tiling_mode != I915_TILING_NONE) dspcntr |= DISPPLANE_TILED; else dspcntr &= ~DISPPLANE_TILED; @@ -1552,7 +1573,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb, I915_WRITE(reg, dspcntr); - Start = obj_priv->gtt_offset; + Start = obj->gtt_offset; Offset = y * fb->pitch + x * (fb->bits_per_pixel / 8); DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n", @@ -1598,7 +1619,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, mutex_lock(&dev->struct_mutex); ret = intel_pin_and_fence_fb_obj(dev, to_intel_framebuffer(crtc->fb)->obj, - false); + NULL); if (ret != 0) { mutex_unlock(&dev->struct_mutex); return ret; @@ -1606,23 +1627,22 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, if (old_fb) { struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_gem_object *obj = to_intel_framebuffer(old_fb)->obj; - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); + struct drm_i915_gem_object *obj = to_intel_framebuffer(old_fb)->obj; wait_event(dev_priv->pending_flip_queue, - atomic_read(&obj_priv->pending_flip) == 0); + atomic_read(&dev_priv->mm.wedged) || + atomic_read(&obj->pending_flip) == 0); /* Big Hammer, we also need to ensure that any pending * MI_WAIT_FOR_EVENT inside a user batch buffer on the * current scanout is retired before unpinning the old * framebuffer. + * + * This should only fail upon a hung GPU, in which case we + * can safely continue. */ - ret = i915_gem_object_flush_gpu(obj_priv, false); - if (ret) { - i915_gem_object_unpin(to_intel_framebuffer(crtc->fb)->obj); - mutex_unlock(&dev->struct_mutex); - return ret; - } + ret = i915_gem_object_flush_gpu(obj, false); + (void) ret; } ret = intel_pipe_set_base_atomic(crtc, crtc->fb, x, y, @@ -1633,8 +1653,10 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y, return ret; } - if (old_fb) + if (old_fb) { + intel_wait_for_vblank(dev, intel_crtc->pipe); i915_gem_object_unpin(to_intel_framebuffer(old_fb)->obj); + } mutex_unlock(&dev->struct_mutex); @@ -1996,31 +2018,56 @@ static void intel_flush_display_plane(struct drm_device *dev, static void intel_clear_scanline_wait(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring; u32 tmp; if (IS_GEN2(dev)) /* Can't break the hang on i8xx */ return; - tmp = I915_READ(PRB0_CTL); - if (tmp & RING_WAIT) { - I915_WRITE(PRB0_CTL, tmp); - POSTING_READ(PRB0_CTL); - } + ring = LP_RING(dev_priv); + tmp = I915_READ_CTL(ring); + if (tmp & RING_WAIT) + I915_WRITE_CTL(ring, tmp); } static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc) { - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; struct drm_i915_private *dev_priv; if (crtc->fb == NULL) return; - obj_priv = to_intel_bo(to_intel_framebuffer(crtc->fb)->obj); + obj = to_intel_framebuffer(crtc->fb)->obj; dev_priv = crtc->dev->dev_private; wait_event(dev_priv->pending_flip_queue, - atomic_read(&obj_priv->pending_flip) == 0); + atomic_read(&obj->pending_flip) == 0); +} + +static bool intel_crtc_driving_pch(struct drm_crtc *crtc) +{ + struct drm_device *dev = crtc->dev; + struct drm_mode_config *mode_config = &dev->mode_config; + struct intel_encoder *encoder; + + /* + * If there's a non-PCH eDP on this crtc, it must be DP_A, and that + * must be driven by its own crtc; no sharing is possible. + */ + list_for_each_entry(encoder, &mode_config->encoder_list, base.head) { + if (encoder->base.crtc != crtc) + continue; + + switch (encoder->type) { + case INTEL_OUTPUT_EDP: + if (!intel_encoder_is_pch_edp(&encoder->base)) + return false; + continue; + } + } + + return true; } static void ironlake_crtc_enable(struct drm_crtc *crtc) @@ -2031,6 +2078,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) int pipe = intel_crtc->pipe; int plane = intel_crtc->plane; u32 reg, temp; + bool is_pch_port = false; if (intel_crtc->active) return; @@ -2044,7 +2092,56 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) I915_WRITE(PCH_LVDS, temp | LVDS_PORT_EN); } - ironlake_fdi_enable(crtc); + is_pch_port = intel_crtc_driving_pch(crtc); + + if (is_pch_port) + ironlake_fdi_enable(crtc); + else { + /* disable CPU FDI tx and PCH FDI rx */ + reg = FDI_TX_CTL(pipe); + temp = I915_READ(reg); + I915_WRITE(reg, temp & ~FDI_TX_ENABLE); + POSTING_READ(reg); + + reg = FDI_RX_CTL(pipe); + temp = I915_READ(reg); + temp &= ~(0x7 << 16); + temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11; + I915_WRITE(reg, temp & ~FDI_RX_ENABLE); + + POSTING_READ(reg); + udelay(100); + + /* Ironlake workaround, disable clock pointer after downing FDI */ + if (HAS_PCH_IBX(dev)) + I915_WRITE(FDI_RX_CHICKEN(pipe), + I915_READ(FDI_RX_CHICKEN(pipe) & + ~FDI_RX_PHASE_SYNC_POINTER_ENABLE)); + + /* still set train pattern 1 */ + reg = FDI_TX_CTL(pipe); + temp = I915_READ(reg); + temp &= ~FDI_LINK_TRAIN_NONE; + temp |= FDI_LINK_TRAIN_PATTERN_1; + I915_WRITE(reg, temp); + + reg = FDI_RX_CTL(pipe); + temp = I915_READ(reg); + if (HAS_PCH_CPT(dev)) { + temp &= ~FDI_LINK_TRAIN_PATTERN_MASK_CPT; + temp |= FDI_LINK_TRAIN_PATTERN_1_CPT; + } else { + temp &= ~FDI_LINK_TRAIN_NONE; + temp |= FDI_LINK_TRAIN_PATTERN_1; + } + /* BPC in FDI rx is consistent with that in PIPECONF */ + temp &= ~(0x07 << 16); + temp |= (I915_READ(PIPECONF(pipe)) & PIPE_BPC_MASK) << 11; + I915_WRITE(reg, temp); + + POSTING_READ(reg); + udelay(100); + } /* Enable panel fitting for LVDS */ if (dev_priv->pch_pf_size && @@ -2078,6 +2175,10 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) intel_flush_display_plane(dev, plane); } + /* Skip the PCH stuff if possible */ + if (!is_pch_port) + goto done; + /* For PCH output, training FDI link */ if (IS_GEN6(dev)) gen6_fdi_link_train(crtc); @@ -2162,7 +2263,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc) I915_WRITE(reg, temp | TRANS_ENABLE); if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100)) DRM_ERROR("failed to enable transcoder %d\n", pipe); - +done: intel_crtc_load_lut(crtc); intel_update_fbc(dev); intel_crtc_update_cursor(crtc, true); @@ -2850,6 +2951,39 @@ static struct intel_watermark_params ironlake_cursor_srwm_info = { ILK_FIFO_LINE_SIZE }; +static struct intel_watermark_params sandybridge_display_wm_info = { + SNB_DISPLAY_FIFO, + SNB_DISPLAY_MAXWM, + SNB_DISPLAY_DFTWM, + 2, + SNB_FIFO_LINE_SIZE +}; + +static struct intel_watermark_params sandybridge_cursor_wm_info = { + SNB_CURSOR_FIFO, + SNB_CURSOR_MAXWM, + SNB_CURSOR_DFTWM, + 2, + SNB_FIFO_LINE_SIZE +}; + +static struct intel_watermark_params sandybridge_display_srwm_info = { + SNB_DISPLAY_SR_FIFO, + SNB_DISPLAY_MAX_SRWM, + SNB_DISPLAY_DFT_SRWM, + 2, + SNB_FIFO_LINE_SIZE +}; + +static struct intel_watermark_params sandybridge_cursor_srwm_info = { + SNB_CURSOR_SR_FIFO, + SNB_CURSOR_MAX_SRWM, + SNB_CURSOR_DFT_SRWM, + 2, + SNB_FIFO_LINE_SIZE +}; + + /** * intel_calculate_wm - calculate watermark level * @clock_in_khz: pixel clock @@ -3383,12 +3517,17 @@ static void i830_update_wm(struct drm_device *dev, int planea_clock, int unused, static bool ironlake_compute_wm0(struct drm_device *dev, int pipe, + const struct intel_watermark_params *display, + int display_latency_ns, + const struct intel_watermark_params *cursor, + int cursor_latency_ns, int *plane_wm, int *cursor_wm) { struct drm_crtc *crtc; - int htotal, hdisplay, clock, pixel_size = 0; - int line_time_us, line_count, entries; + int htotal, hdisplay, clock, pixel_size; + int line_time_us, line_count; + int entries, tlb_miss; crtc = intel_get_crtc_for_pipe(dev, pipe); if (crtc->fb == NULL || !crtc->enabled) @@ -3400,37 +3539,141 @@ static bool ironlake_compute_wm0(struct drm_device *dev, pixel_size = crtc->fb->bits_per_pixel / 8; /* Use the small buffer method to calculate plane watermark */ - entries = ((clock * pixel_size / 1000) * ILK_LP0_PLANE_LATENCY) / 1000; - entries = DIV_ROUND_UP(entries, - ironlake_display_wm_info.cacheline_size); - *plane_wm = entries + ironlake_display_wm_info.guard_size; - if (*plane_wm > (int)ironlake_display_wm_info.max_wm) - *plane_wm = ironlake_display_wm_info.max_wm; + entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000; + tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8; + if (tlb_miss > 0) + entries += tlb_miss; + entries = DIV_ROUND_UP(entries, display->cacheline_size); + *plane_wm = entries + display->guard_size; + if (*plane_wm > (int)display->max_wm) + *plane_wm = display->max_wm; /* Use the large buffer method to calculate cursor watermark */ line_time_us = ((htotal * 1000) / clock); - line_count = (ILK_LP0_CURSOR_LATENCY / line_time_us + 1000) / 1000; + line_count = (cursor_latency_ns / line_time_us + 1000) / 1000; entries = line_count * 64 * pixel_size; - entries = DIV_ROUND_UP(entries, - ironlake_cursor_wm_info.cacheline_size); - *cursor_wm = entries + ironlake_cursor_wm_info.guard_size; - if (*cursor_wm > ironlake_cursor_wm_info.max_wm) - *cursor_wm = ironlake_cursor_wm_info.max_wm; + tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8; + if (tlb_miss > 0) + entries += tlb_miss; + entries = DIV_ROUND_UP(entries, cursor->cacheline_size); + *cursor_wm = entries + cursor->guard_size; + if (*cursor_wm > (int)cursor->max_wm) + *cursor_wm = (int)cursor->max_wm; + + return true; +} + +/* + * Check the wm result. + * + * If any calculated watermark values is larger than the maximum value that + * can be programmed into the associated watermark register, that watermark + * must be disabled. + */ +static bool ironlake_check_srwm(struct drm_device *dev, int level, + int fbc_wm, int display_wm, int cursor_wm, + const struct intel_watermark_params *display, + const struct intel_watermark_params *cursor) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d," + " cursor %d\n", level, display_wm, fbc_wm, cursor_wm); + + if (fbc_wm > SNB_FBC_MAX_SRWM) { + DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n", + fbc_wm, SNB_FBC_MAX_SRWM, level); + + /* fbc has it's own way to disable FBC WM */ + I915_WRITE(DISP_ARB_CTL, + I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS); + return false; + } + + if (display_wm > display->max_wm) { + DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n", + display_wm, SNB_DISPLAY_MAX_SRWM, level); + return false; + } + + if (cursor_wm > cursor->max_wm) { + DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n", + cursor_wm, SNB_CURSOR_MAX_SRWM, level); + return false; + } + + if (!(fbc_wm || display_wm || cursor_wm)) { + DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level); + return false; + } return true; } +/* + * Compute watermark values of WM[1-3], + */ +static bool ironlake_compute_srwm(struct drm_device *dev, int level, + int hdisplay, int htotal, + int pixel_size, int clock, int latency_ns, + const struct intel_watermark_params *display, + const struct intel_watermark_params *cursor, + int *fbc_wm, int *display_wm, int *cursor_wm) +{ + + unsigned long line_time_us; + int line_count, line_size; + int small, large; + int entries; + + if (!latency_ns) { + *fbc_wm = *display_wm = *cursor_wm = 0; + return false; + } + + line_time_us = (htotal * 1000) / clock; + line_count = (latency_ns / line_time_us + 1000) / 1000; + line_size = hdisplay * pixel_size; + + /* Use the minimum of the small and large buffer method for primary */ + small = ((clock * pixel_size / 1000) * latency_ns) / 1000; + large = line_count * line_size; + + entries = DIV_ROUND_UP(min(small, large), display->cacheline_size); + *display_wm = entries + display->guard_size; + + /* + * Spec says: + * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2 + */ + *fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2; + + /* calculate the self-refresh watermark for display cursor */ + entries = line_count * pixel_size * 64; + entries = DIV_ROUND_UP(entries, cursor->cacheline_size); + *cursor_wm = entries + cursor->guard_size; + + return ironlake_check_srwm(dev, level, + *fbc_wm, *display_wm, *cursor_wm, + display, cursor); +} + static void ironlake_update_wm(struct drm_device *dev, int planea_clock, int planeb_clock, - int sr_hdisplay, int sr_htotal, + int hdisplay, int htotal, int pixel_size) { struct drm_i915_private *dev_priv = dev->dev_private; - int plane_wm, cursor_wm, enabled; - int tmp; + int fbc_wm, plane_wm, cursor_wm, enabled; + int clock; enabled = 0; - if (ironlake_compute_wm0(dev, 0, &plane_wm, &cursor_wm)) { + if (ironlake_compute_wm0(dev, 0, + &ironlake_display_wm_info, + ILK_LP0_PLANE_LATENCY, + &ironlake_cursor_wm_info, + ILK_LP0_CURSOR_LATENCY, + &plane_wm, &cursor_wm)) { I915_WRITE(WM0_PIPEA_ILK, (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); DRM_DEBUG_KMS("FIFO watermarks For pipe A -" @@ -3439,7 +3682,12 @@ static void ironlake_update_wm(struct drm_device *dev, enabled++; } - if (ironlake_compute_wm0(dev, 1, &plane_wm, &cursor_wm)) { + if (ironlake_compute_wm0(dev, 1, + &ironlake_display_wm_info, + ILK_LP0_PLANE_LATENCY, + &ironlake_cursor_wm_info, + ILK_LP0_CURSOR_LATENCY, + &plane_wm, &cursor_wm)) { I915_WRITE(WM0_PIPEB_ILK, (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); DRM_DEBUG_KMS("FIFO watermarks For pipe B -" @@ -3452,57 +3700,151 @@ static void ironlake_update_wm(struct drm_device *dev, * Calculate and update the self-refresh watermark only when one * display plane is used. */ - tmp = 0; - if (enabled == 1 && /* XXX disabled due to buggy implmentation? */ 0) { - unsigned long line_time_us; - int small, large, plane_fbc; - int sr_clock, entries; - int line_count, line_size; - /* Read the self-refresh latency. The unit is 0.5us */ - int ilk_sr_latency = I915_READ(MLTR_ILK) & ILK_SRLT_MASK; + I915_WRITE(WM3_LP_ILK, 0); + I915_WRITE(WM2_LP_ILK, 0); + I915_WRITE(WM1_LP_ILK, 0); - sr_clock = planea_clock ? planea_clock : planeb_clock; - line_time_us = (sr_htotal * 1000) / sr_clock; + if (enabled != 1) + return; - /* Use ns/us then divide to preserve precision */ - line_count = ((ilk_sr_latency * 500) / line_time_us + 1000) - / 1000; - line_size = sr_hdisplay * pixel_size; + clock = planea_clock ? planea_clock : planeb_clock; - /* Use the minimum of the small and large buffer method for primary */ - small = ((sr_clock * pixel_size / 1000) * (ilk_sr_latency * 500)) / 1000; - large = line_count * line_size; + /* WM1 */ + if (!ironlake_compute_srwm(dev, 1, hdisplay, htotal, pixel_size, + clock, ILK_READ_WM1_LATENCY() * 500, + &ironlake_display_srwm_info, + &ironlake_cursor_srwm_info, + &fbc_wm, &plane_wm, &cursor_wm)) + return; - entries = DIV_ROUND_UP(min(small, large), - ironlake_display_srwm_info.cacheline_size); + I915_WRITE(WM1_LP_ILK, + WM1_LP_SR_EN | + (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) | + (fbc_wm << WM1_LP_FBC_SHIFT) | + (plane_wm << WM1_LP_SR_SHIFT) | + cursor_wm); + + /* WM2 */ + if (!ironlake_compute_srwm(dev, 2, hdisplay, htotal, pixel_size, + clock, ILK_READ_WM2_LATENCY() * 500, + &ironlake_display_srwm_info, + &ironlake_cursor_srwm_info, + &fbc_wm, &plane_wm, &cursor_wm)) + return; - plane_fbc = entries * 64; - plane_fbc = DIV_ROUND_UP(plane_fbc, line_size); + I915_WRITE(WM2_LP_ILK, + WM2_LP_EN | + (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) | + (fbc_wm << WM1_LP_FBC_SHIFT) | + (plane_wm << WM1_LP_SR_SHIFT) | + cursor_wm); - plane_wm = entries + ironlake_display_srwm_info.guard_size; - if (plane_wm > (int)ironlake_display_srwm_info.max_wm) - plane_wm = ironlake_display_srwm_info.max_wm; + /* + * WM3 is unsupported on ILK, probably because we don't have latency + * data for that power state + */ +} - /* calculate the self-refresh watermark for display cursor */ - entries = line_count * pixel_size * 64; - entries = DIV_ROUND_UP(entries, - ironlake_cursor_srwm_info.cacheline_size); +static void sandybridge_update_wm(struct drm_device *dev, + int planea_clock, int planeb_clock, + int hdisplay, int htotal, + int pixel_size) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int latency = SNB_READ_WM0_LATENCY() * 100; /* In unit 0.1us */ + int fbc_wm, plane_wm, cursor_wm, enabled; + int clock; - cursor_wm = entries + ironlake_cursor_srwm_info.guard_size; - if (cursor_wm > (int)ironlake_cursor_srwm_info.max_wm) - cursor_wm = ironlake_cursor_srwm_info.max_wm; + enabled = 0; + if (ironlake_compute_wm0(dev, 0, + &sandybridge_display_wm_info, latency, + &sandybridge_cursor_wm_info, latency, + &plane_wm, &cursor_wm)) { + I915_WRITE(WM0_PIPEA_ILK, + (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); + DRM_DEBUG_KMS("FIFO watermarks For pipe A -" + " plane %d, " "cursor: %d\n", + plane_wm, cursor_wm); + enabled++; + } - /* configure watermark and enable self-refresh */ - tmp = (WM1_LP_SR_EN | - (ilk_sr_latency << WM1_LP_LATENCY_SHIFT) | - (plane_fbc << WM1_LP_FBC_SHIFT) | - (plane_wm << WM1_LP_SR_SHIFT) | - cursor_wm); - DRM_DEBUG_KMS("self-refresh watermark: display plane %d, fbc lines %d," - " cursor %d\n", plane_wm, plane_fbc, cursor_wm); + if (ironlake_compute_wm0(dev, 1, + &sandybridge_display_wm_info, latency, + &sandybridge_cursor_wm_info, latency, + &plane_wm, &cursor_wm)) { + I915_WRITE(WM0_PIPEB_ILK, + (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm); + DRM_DEBUG_KMS("FIFO watermarks For pipe B -" + " plane %d, cursor: %d\n", + plane_wm, cursor_wm); + enabled++; } - I915_WRITE(WM1_LP_ILK, tmp); - /* XXX setup WM2 and WM3 */ + + /* + * Calculate and update the self-refresh watermark only when one + * display plane is used. + * + * SNB support 3 levels of watermark. + * + * WM1/WM2/WM2 watermarks have to be enabled in the ascending order, + * and disabled in the descending order + * + */ + I915_WRITE(WM3_LP_ILK, 0); + I915_WRITE(WM2_LP_ILK, 0); + I915_WRITE(WM1_LP_ILK, 0); + + if (enabled != 1) + return; + + clock = planea_clock ? planea_clock : planeb_clock; + + /* WM1 */ + if (!ironlake_compute_srwm(dev, 1, hdisplay, htotal, pixel_size, + clock, SNB_READ_WM1_LATENCY() * 500, + &sandybridge_display_srwm_info, + &sandybridge_cursor_srwm_info, + &fbc_wm, &plane_wm, &cursor_wm)) + return; + + I915_WRITE(WM1_LP_ILK, + WM1_LP_SR_EN | + (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) | + (fbc_wm << WM1_LP_FBC_SHIFT) | + (plane_wm << WM1_LP_SR_SHIFT) | + cursor_wm); + + /* WM2 */ + if (!ironlake_compute_srwm(dev, 2, + hdisplay, htotal, pixel_size, + clock, SNB_READ_WM2_LATENCY() * 500, + &sandybridge_display_srwm_info, + &sandybridge_cursor_srwm_info, + &fbc_wm, &plane_wm, &cursor_wm)) + return; + + I915_WRITE(WM2_LP_ILK, + WM2_LP_EN | + (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) | + (fbc_wm << WM1_LP_FBC_SHIFT) | + (plane_wm << WM1_LP_SR_SHIFT) | + cursor_wm); + + /* WM3 */ + if (!ironlake_compute_srwm(dev, 3, + hdisplay, htotal, pixel_size, + clock, SNB_READ_WM3_LATENCY() * 500, + &sandybridge_display_srwm_info, + &sandybridge_cursor_srwm_info, + &fbc_wm, &plane_wm, &cursor_wm)) + return; + + I915_WRITE(WM3_LP_ILK, + WM3_LP_EN | + (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) | + (fbc_wm << WM1_LP_FBC_SHIFT) | + (plane_wm << WM1_LP_SR_SHIFT) | + cursor_wm); } /** @@ -3580,6 +3922,11 @@ static void intel_update_watermarks(struct drm_device *dev) sr_hdisplay, sr_htotal, pixel_size); } +static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv) +{ + return dev_priv->lvds_use_ssc && i915_panel_use_ssc; +} + static int intel_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode, struct drm_display_mode *adjusted_mode, @@ -3642,7 +3989,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, num_connectors++; } - if (is_lvds && dev_priv->lvds_use_ssc && num_connectors < 2) { + if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) { refclk = dev_priv->lvds_ssc_freq * 1000; DRM_DEBUG_KMS("using SSC reference clock of %d MHz\n", refclk / 1000); @@ -3660,7 +4007,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, * refclk, or FALSE. The returned values represent the clock equation: * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2. */ - limit = intel_limit(crtc); + limit = intel_limit(crtc, refclk); ok = limit->find_pll(limit, crtc, adjusted_mode->clock, refclk, &clock); if (!ok) { DRM_ERROR("Couldn't find PLL settings for mode!\n"); @@ -3714,7 +4061,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, int lane = 0, link_bw, bpp; /* CPU eDP doesn't require FDI link, so just set DP M/N according to current link config */ - if (has_edp_encoder && !intel_encoder_is_pch_edp(&encoder->base)) { + if (has_edp_encoder && !intel_encoder_is_pch_edp(&has_edp_encoder->base)) { target_clock = mode->clock; intel_edp_link_config(has_edp_encoder, &lane, &link_bw); @@ -3817,7 +4164,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, udelay(200); if (has_edp_encoder) { - if (dev_priv->lvds_use_ssc) { + if (intel_panel_use_ssc(dev_priv)) { temp |= DREF_SSC1_ENABLE; I915_WRITE(PCH_DREF_CONTROL, temp); @@ -3828,13 +4175,13 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, /* Enable CPU source on CPU attached eDP */ if (!intel_encoder_is_pch_edp(&has_edp_encoder->base)) { - if (dev_priv->lvds_use_ssc) + if (intel_panel_use_ssc(dev_priv)) temp |= DREF_CPU_SOURCE_OUTPUT_DOWNSPREAD; else temp |= DREF_CPU_SOURCE_OUTPUT_NONSPREAD; } else { /* Enable SSC on PCH eDP if needed */ - if (dev_priv->lvds_use_ssc) { + if (intel_panel_use_ssc(dev_priv)) { DRM_ERROR("enabling SSC on PCH\n"); temp |= DREF_SUPERSPREAD_SOURCE_ENABLE; } @@ -3857,6 +4204,22 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, reduced_clock.m2; } + /* Enable autotuning of the PLL clock (if permissible) */ + if (HAS_PCH_SPLIT(dev)) { + int factor = 21; + + if (is_lvds) { + if ((intel_panel_use_ssc(dev_priv) && + dev_priv->lvds_ssc_freq == 100) || + (I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP) + factor = 25; + } else if (is_sdvo && is_tv) + factor = 20; + + if (clock.m1 < factor * clock.n) + fp |= FP_CB_TUNE; + } + dpll = 0; if (!HAS_PCH_SPLIT(dev)) dpll = DPLL_VGA_MODE_DIS; @@ -3925,7 +4288,7 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, /* XXX: just matching BIOS for now */ /* dpll |= PLL_REF_INPUT_TVCLKINBC; */ dpll |= 3; - else if (is_lvds && dev_priv->lvds_use_ssc && num_connectors < 2) + else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2) dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN; else dpll |= PLL_REF_INPUT_DREFCLK; @@ -4071,7 +4434,6 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, } if (!has_edp_encoder || intel_encoder_is_pch_edp(&has_edp_encoder->base)) { - I915_WRITE(fp_reg, fp); I915_WRITE(dpll_reg, dpll); /* Wait for the clocks to stabilize. */ @@ -4089,13 +4451,13 @@ static int intel_crtc_mode_set(struct drm_crtc *crtc, } I915_WRITE(DPLL_MD(pipe), temp); } else { - /* write it again -- the BIOS does, after all */ + /* The pixel multiplier can only be updated once the + * DPLL is enabled and the clocks are stable. + * + * So write it again. + */ I915_WRITE(dpll_reg, dpll); } - - /* Wait for the clocks to stabilize. */ - POSTING_READ(dpll_reg); - udelay(150); } intel_crtc->lowfreq_avail = false; @@ -4331,15 +4693,14 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc, } static int intel_crtc_cursor_set(struct drm_crtc *crtc, - struct drm_file *file_priv, + struct drm_file *file, uint32_t handle, uint32_t width, uint32_t height) { struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); - struct drm_gem_object *bo; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; uint32_t addr; int ret; @@ -4349,7 +4710,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, if (!handle) { DRM_DEBUG_KMS("cursor off\n"); addr = 0; - bo = NULL; + obj = NULL; mutex_lock(&dev->struct_mutex); goto finish; } @@ -4360,13 +4721,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, return -EINVAL; } - bo = drm_gem_object_lookup(dev, file_priv, handle); - if (!bo) + obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); + if (!obj) return -ENOENT; - obj_priv = to_intel_bo(bo); - - if (bo->size < width * height * 4) { + if (obj->base.size < width * height * 4) { DRM_ERROR("buffer is to small\n"); ret = -ENOMEM; goto fail; @@ -4375,29 +4734,41 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, /* we only need to pin inside GTT if cursor is non-phy */ mutex_lock(&dev->struct_mutex); if (!dev_priv->info->cursor_needs_physical) { - ret = i915_gem_object_pin(bo, PAGE_SIZE); + if (obj->tiling_mode) { + DRM_ERROR("cursor cannot be tiled\n"); + ret = -EINVAL; + goto fail_locked; + } + + ret = i915_gem_object_pin(obj, PAGE_SIZE, true); if (ret) { DRM_ERROR("failed to pin cursor bo\n"); goto fail_locked; } - ret = i915_gem_object_set_to_gtt_domain(bo, 0); + ret = i915_gem_object_set_to_gtt_domain(obj, 0); if (ret) { DRM_ERROR("failed to move cursor bo into the GTT\n"); goto fail_unpin; } - addr = obj_priv->gtt_offset; + ret = i915_gem_object_put_fence(obj); + if (ret) { + DRM_ERROR("failed to move cursor bo into the GTT\n"); + goto fail_unpin; + } + + addr = obj->gtt_offset; } else { int align = IS_I830(dev) ? 16 * 1024 : 256; - ret = i915_gem_attach_phys_object(dev, bo, + ret = i915_gem_attach_phys_object(dev, obj, (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1, align); if (ret) { DRM_ERROR("failed to attach phys object\n"); goto fail_locked; } - addr = obj_priv->phys_obj->handle->busaddr; + addr = obj->phys_obj->handle->busaddr; } if (IS_GEN2(dev)) @@ -4406,17 +4777,17 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, finish: if (intel_crtc->cursor_bo) { if (dev_priv->info->cursor_needs_physical) { - if (intel_crtc->cursor_bo != bo) + if (intel_crtc->cursor_bo != obj) i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo); } else i915_gem_object_unpin(intel_crtc->cursor_bo); - drm_gem_object_unreference(intel_crtc->cursor_bo); + drm_gem_object_unreference(&intel_crtc->cursor_bo->base); } mutex_unlock(&dev->struct_mutex); intel_crtc->cursor_addr = addr; - intel_crtc->cursor_bo = bo; + intel_crtc->cursor_bo = obj; intel_crtc->cursor_width = width; intel_crtc->cursor_height = height; @@ -4424,11 +4795,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc, return 0; fail_unpin: - i915_gem_object_unpin(bo); + i915_gem_object_unpin(obj); fail_locked: mutex_unlock(&dev->struct_mutex); fail: - drm_gem_object_unreference_unlocked(bo); + drm_gem_object_unreference_unlocked(&obj->base); return ret; } @@ -4739,8 +5110,14 @@ static void intel_gpu_idle_timer(unsigned long arg) struct drm_device *dev = (struct drm_device *)arg; drm_i915_private_t *dev_priv = dev->dev_private; - dev_priv->busy = false; + if (!list_empty(&dev_priv->mm.active_list)) { + /* Still processing requests, so just re-arm the timer. */ + mod_timer(&dev_priv->idle_timer, jiffies + + msecs_to_jiffies(GPU_IDLE_TIMEOUT)); + return; + } + dev_priv->busy = false; queue_work(dev_priv->wq, &dev_priv->idle_work); } @@ -4751,9 +5128,17 @@ static void intel_crtc_idle_timer(unsigned long arg) struct intel_crtc *intel_crtc = (struct intel_crtc *)arg; struct drm_crtc *crtc = &intel_crtc->base; drm_i915_private_t *dev_priv = crtc->dev->dev_private; + struct intel_framebuffer *intel_fb; - intel_crtc->busy = false; + intel_fb = to_intel_framebuffer(crtc->fb); + if (intel_fb && intel_fb->obj->active) { + /* The framebuffer is still being accessed by the GPU. */ + mod_timer(&intel_crtc->idle_timer, jiffies + + msecs_to_jiffies(CRTC_IDLE_TIMEOUT)); + return; + } + intel_crtc->busy = false; queue_work(dev_priv->wq, &dev_priv->idle_work); } @@ -4763,8 +5148,8 @@ static void intel_increase_pllclock(struct drm_crtc *crtc) drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); int pipe = intel_crtc->pipe; - int dpll_reg = (pipe == 0) ? DPLL_A : DPLL_B; - int dpll = I915_READ(dpll_reg); + int dpll_reg = DPLL(pipe); + int dpll; if (HAS_PCH_SPLIT(dev)) return; @@ -4772,17 +5157,19 @@ static void intel_increase_pllclock(struct drm_crtc *crtc) if (!dev_priv->lvds_downclock_avail) return; + dpll = I915_READ(dpll_reg); if (!HAS_PIPE_CXSR(dev) && (dpll & DISPLAY_RATE_SELECT_FPA1)) { DRM_DEBUG_DRIVER("upclocking LVDS\n"); /* Unlock panel regs */ - I915_WRITE(PP_CONTROL, I915_READ(PP_CONTROL) | - PANEL_UNLOCK_REGS); + I915_WRITE(PP_CONTROL, + I915_READ(PP_CONTROL) | PANEL_UNLOCK_REGS); dpll &= ~DISPLAY_RATE_SELECT_FPA1; I915_WRITE(dpll_reg, dpll); - dpll = I915_READ(dpll_reg); + POSTING_READ(dpll_reg); intel_wait_for_vblank(dev, pipe); + dpll = I915_READ(dpll_reg); if (dpll & DISPLAY_RATE_SELECT_FPA1) DRM_DEBUG_DRIVER("failed to upclock LVDS!\n"); @@ -4888,7 +5275,7 @@ static void intel_idle_update(struct work_struct *work) * buffer), we'll also mark the display as busy, so we know to increase its * clock frequency. */ -void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj) +void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_crtc *crtc = NULL; @@ -4969,8 +5356,9 @@ static void intel_unpin_work_fn(struct work_struct *__work) mutex_lock(&work->dev->struct_mutex); i915_gem_object_unpin(work->old_fb_obj); - drm_gem_object_unreference(work->pending_flip_obj); - drm_gem_object_unreference(work->old_fb_obj); + drm_gem_object_unreference(&work->pending_flip_obj->base); + drm_gem_object_unreference(&work->old_fb_obj->base); + mutex_unlock(&work->dev->struct_mutex); kfree(work); } @@ -4981,15 +5369,17 @@ static void do_intel_finish_page_flip(struct drm_device *dev, drm_i915_private_t *dev_priv = dev->dev_private; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; struct drm_pending_vblank_event *e; - struct timeval now; + struct timeval tnow, tvbl; unsigned long flags; /* Ignore early vblank irqs */ if (intel_crtc == NULL) return; + do_gettimeofday(&tnow); + spin_lock_irqsave(&dev->event_lock, flags); work = intel_crtc->unpin_work; if (work == NULL || !work->pending) { @@ -4998,26 +5388,49 @@ static void do_intel_finish_page_flip(struct drm_device *dev, } intel_crtc->unpin_work = NULL; - drm_vblank_put(dev, intel_crtc->pipe); if (work->event) { e = work->event; - do_gettimeofday(&now); - e->event.sequence = drm_vblank_count(dev, intel_crtc->pipe); - e->event.tv_sec = now.tv_sec; - e->event.tv_usec = now.tv_usec; + e->event.sequence = drm_vblank_count_and_time(dev, intel_crtc->pipe, &tvbl); + + /* Called before vblank count and timestamps have + * been updated for the vblank interval of flip + * completion? Need to increment vblank count and + * add one videorefresh duration to returned timestamp + * to account for this. We assume this happened if we + * get called over 0.9 frame durations after the last + * timestamped vblank. + * + * This calculation can not be used with vrefresh rates + * below 5Hz (10Hz to be on the safe side) without + * promoting to 64 integers. + */ + if (10 * (timeval_to_ns(&tnow) - timeval_to_ns(&tvbl)) > + 9 * crtc->framedur_ns) { + e->event.sequence++; + tvbl = ns_to_timeval(timeval_to_ns(&tvbl) + + crtc->framedur_ns); + } + + e->event.tv_sec = tvbl.tv_sec; + e->event.tv_usec = tvbl.tv_usec; + list_add_tail(&e->base.link, &e->base.file_priv->event_list); wake_up_interruptible(&e->base.file_priv->event_wait); } + drm_vblank_put(dev, intel_crtc->pipe); + spin_unlock_irqrestore(&dev->event_lock, flags); - obj_priv = to_intel_bo(work->old_fb_obj); + obj = work->old_fb_obj; + atomic_clear_mask(1 << intel_crtc->plane, - &obj_priv->pending_flip.counter); - if (atomic_read(&obj_priv->pending_flip) == 0) + &obj->pending_flip.counter); + if (atomic_read(&obj->pending_flip) == 0) wake_up(&dev_priv->pending_flip_queue); + schedule_work(&work->work); trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj); @@ -5063,8 +5476,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, struct drm_device *dev = crtc->dev; struct drm_i915_private *dev_priv = dev->dev_private; struct intel_framebuffer *intel_fb; - struct drm_i915_gem_object *obj_priv; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); struct intel_unpin_work *work; unsigned long flags, offset; @@ -5098,13 +5510,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, obj = intel_fb->obj; mutex_lock(&dev->struct_mutex); - ret = intel_pin_and_fence_fb_obj(dev, obj, true); + ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv)); if (ret) goto cleanup_work; /* Reference the objects for the scheduled work. */ - drm_gem_object_reference(work->old_fb_obj); - drm_gem_object_reference(obj); + drm_gem_object_reference(&work->old_fb_obj->base); + drm_gem_object_reference(&obj->base); crtc->fb = fb; @@ -5112,22 +5524,16 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, if (ret) goto cleanup_objs; - /* Block clients from rendering to the new back buffer until - * the flip occurs and the object is no longer visible. - */ - atomic_add(1 << intel_crtc->plane, - &to_intel_bo(work->old_fb_obj)->pending_flip); - - work->pending_flip_obj = obj; - obj_priv = to_intel_bo(obj); - if (IS_GEN3(dev) || IS_GEN2(dev)) { u32 flip_mask; /* Can't queue multiple flips, so wait for the previous * one to finish before executing the next. */ - BEGIN_LP_RING(2); + ret = BEGIN_LP_RING(2); + if (ret) + goto cleanup_objs; + if (intel_crtc->plane) flip_mask = MI_WAIT_FOR_PLANE_B_FLIP; else @@ -5137,18 +5543,28 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, ADVANCE_LP_RING(); } + work->pending_flip_obj = obj; + work->enable_stall_check = true; /* Offset into the new buffer for cases of shared fbs between CRTCs */ offset = crtc->y * fb->pitch + crtc->x * fb->bits_per_pixel/8; - BEGIN_LP_RING(4); - switch(INTEL_INFO(dev)->gen) { + ret = BEGIN_LP_RING(4); + if (ret) + goto cleanup_objs; + + /* Block clients from rendering to the new back buffer until + * the flip occurs and the object is no longer visible. + */ + atomic_add(1 << intel_crtc->plane, &work->old_fb_obj->pending_flip); + + switch (INTEL_INFO(dev)->gen) { case 2: OUT_RING(MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); OUT_RING(fb->pitch); - OUT_RING(obj_priv->gtt_offset + offset); + OUT_RING(obj->gtt_offset + offset); OUT_RING(MI_NOOP); break; @@ -5156,7 +5572,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, OUT_RING(MI_DISPLAY_FLIP_I915 | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); OUT_RING(fb->pitch); - OUT_RING(obj_priv->gtt_offset + offset); + OUT_RING(obj->gtt_offset + offset); OUT_RING(MI_NOOP); break; @@ -5169,7 +5585,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, OUT_RING(MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); OUT_RING(fb->pitch); - OUT_RING(obj_priv->gtt_offset | obj_priv->tiling_mode); + OUT_RING(obj->gtt_offset | obj->tiling_mode); /* XXX Enabling the panel-fitter across page-flip is so far * untested on non-native modes, so ignore it for now. @@ -5183,8 +5599,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, case 6: OUT_RING(MI_DISPLAY_FLIP | MI_DISPLAY_FLIP_PLANE(intel_crtc->plane)); - OUT_RING(fb->pitch | obj_priv->tiling_mode); - OUT_RING(obj_priv->gtt_offset); + OUT_RING(fb->pitch | obj->tiling_mode); + OUT_RING(obj->gtt_offset); pf = I915_READ(pipe == 0 ? PFA_CTL_1 : PFB_CTL_1) & PF_ENABLE; pipesrc = I915_READ(pipe == 0 ? PIPEASRC : PIPEBSRC) & 0x0fff0fff; @@ -5200,8 +5616,8 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc, return 0; cleanup_objs: - drm_gem_object_unreference(work->old_fb_obj); - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&work->old_fb_obj->base); + drm_gem_object_unreference(&obj->base); cleanup_work: mutex_unlock(&dev->struct_mutex); @@ -5214,6 +5630,16 @@ cleanup_work: return ret; } +static void intel_crtc_reset(struct drm_crtc *crtc) +{ + struct intel_crtc *intel_crtc = to_intel_crtc(crtc); + + /* Reset flags back to the 'unknown' status so that they + * will be correctly set on the initial modeset. + */ + intel_crtc->dpms_mode = -1; +} + static struct drm_crtc_helper_funcs intel_helper_funcs = { .dpms = intel_crtc_dpms, .mode_fixup = intel_crtc_mode_fixup, @@ -5225,6 +5651,7 @@ static struct drm_crtc_helper_funcs intel_helper_funcs = { }; static const struct drm_crtc_funcs intel_crtc_funcs = { + .reset = intel_crtc_reset, .cursor_set = intel_crtc_cursor_set, .cursor_move = intel_crtc_cursor_move, .gamma_set = intel_crtc_gamma_set, @@ -5315,8 +5742,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) dev_priv->plane_to_crtc_mapping[intel_crtc->plane] = &intel_crtc->base; dev_priv->pipe_to_crtc_mapping[intel_crtc->pipe] = &intel_crtc->base; - intel_crtc->cursor_addr = 0; - intel_crtc->dpms_mode = -1; + intel_crtc_reset(&intel_crtc->base); intel_crtc->active = true; /* force the pipe off on setup_init_config */ if (HAS_PCH_SPLIT(dev)) { @@ -5338,7 +5764,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe) } int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data, - struct drm_file *file_priv) + struct drm_file *file) { drm_i915_private_t *dev_priv = dev->dev_private; struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data; @@ -5498,6 +5924,8 @@ static void intel_setup_outputs(struct drm_device *dev) encoder->base.possible_clones = intel_encoder_clones(dev, encoder->clone_mask); } + + intel_panel_setup_backlight(dev); } static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) @@ -5505,19 +5933,19 @@ static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb) struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); drm_framebuffer_cleanup(fb); - drm_gem_object_unreference_unlocked(intel_fb->obj); + drm_gem_object_unreference_unlocked(&intel_fb->obj->base); kfree(intel_fb); } static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, - struct drm_file *file_priv, + struct drm_file *file, unsigned int *handle) { struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb); - struct drm_gem_object *object = intel_fb->obj; + struct drm_i915_gem_object *obj = intel_fb->obj; - return drm_gem_handle_create(file_priv, object, handle); + return drm_gem_handle_create(file, &obj->base, handle); } static const struct drm_framebuffer_funcs intel_fb_funcs = { @@ -5528,12 +5956,11 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = { int intel_framebuffer_init(struct drm_device *dev, struct intel_framebuffer *intel_fb, struct drm_mode_fb_cmd *mode_cmd, - struct drm_gem_object *obj) + struct drm_i915_gem_object *obj) { - struct drm_i915_gem_object *obj_priv = to_intel_bo(obj); int ret; - if (obj_priv->tiling_mode == I915_TILING_Y) + if (obj->tiling_mode == I915_TILING_Y) return -EINVAL; if (mode_cmd->pitch & 63) @@ -5565,11 +5992,11 @@ intel_user_framebuffer_create(struct drm_device *dev, struct drm_file *filp, struct drm_mode_fb_cmd *mode_cmd) { - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; struct intel_framebuffer *intel_fb; int ret; - obj = drm_gem_object_lookup(dev, filp, mode_cmd->handle); + obj = to_intel_bo(drm_gem_object_lookup(dev, filp, mode_cmd->handle)); if (!obj) return ERR_PTR(-ENOENT); @@ -5577,10 +6004,9 @@ intel_user_framebuffer_create(struct drm_device *dev, if (!intel_fb) return ERR_PTR(-ENOMEM); - ret = intel_framebuffer_init(dev, intel_fb, - mode_cmd, obj); + ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj); if (ret) { - drm_gem_object_unreference_unlocked(obj); + drm_gem_object_unreference_unlocked(&obj->base); kfree(intel_fb); return ERR_PTR(ret); } @@ -5593,10 +6019,10 @@ static const struct drm_mode_config_funcs intel_mode_funcs = { .output_poll_changed = intel_fb_output_poll_changed, }; -static struct drm_gem_object * +static struct drm_i915_gem_object * intel_alloc_context_page(struct drm_device *dev) { - struct drm_gem_object *ctx; + struct drm_i915_gem_object *ctx; int ret; ctx = i915_gem_alloc_object(dev, 4096); @@ -5606,7 +6032,7 @@ intel_alloc_context_page(struct drm_device *dev) } mutex_lock(&dev->struct_mutex); - ret = i915_gem_object_pin(ctx, 4096); + ret = i915_gem_object_pin(ctx, 4096, true); if (ret) { DRM_ERROR("failed to pin power context: %d\n", ret); goto err_unref; @@ -5624,7 +6050,7 @@ intel_alloc_context_page(struct drm_device *dev) err_unpin: i915_gem_object_unpin(ctx); err_unref: - drm_gem_object_unreference(ctx); + drm_gem_object_unreference(&ctx->base); mutex_unlock(&dev->struct_mutex); return NULL; } @@ -5736,6 +6162,25 @@ void ironlake_disable_drps(struct drm_device *dev) } +void gen6_set_rps(struct drm_device *dev, u8 val) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + u32 swreq; + + swreq = (val & 0x3ff) << 25; + I915_WRITE(GEN6_RPNSWREQ, swreq); +} + +void gen6_disable_rps(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + I915_WRITE(GEN6_RPNSWREQ, 1 << 31); + I915_WRITE(GEN6_PMINTRMSK, 0xffffffff); + I915_WRITE(GEN6_PMIER, 0); + I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR)); +} + static unsigned long intel_pxfreq(u32 vidfreq) { unsigned long freq; @@ -5822,7 +6267,123 @@ void intel_init_emon(struct drm_device *dev) dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK); } -void intel_init_clock_gating(struct drm_device *dev) +void gen6_enable_rps(struct drm_i915_private *dev_priv) +{ + u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP); + u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS); + u32 pcu_mbox; + int cur_freq, min_freq, max_freq; + int i; + + /* Here begins a magic sequence of register writes to enable + * auto-downclocking. + * + * Perhaps there might be some value in exposing these to + * userspace... + */ + I915_WRITE(GEN6_RC_STATE, 0); + __gen6_force_wake_get(dev_priv); + + /* disable the counters and set deterministic thresholds */ + I915_WRITE(GEN6_RC_CONTROL, 0); + + I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16); + I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30); + I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30); + I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); + I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); + + for (i = 0; i < I915_NUM_RINGS; i++) + I915_WRITE(RING_MAX_IDLE(dev_priv->ring[i].mmio_base), 10); + + I915_WRITE(GEN6_RC_SLEEP, 0); + I915_WRITE(GEN6_RC1e_THRESHOLD, 1000); + I915_WRITE(GEN6_RC6_THRESHOLD, 50000); + I915_WRITE(GEN6_RC6p_THRESHOLD, 100000); + I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */ + + I915_WRITE(GEN6_RC_CONTROL, + GEN6_RC_CTL_RC6p_ENABLE | + GEN6_RC_CTL_RC6_ENABLE | + GEN6_RC_CTL_EI_MODE(1) | + GEN6_RC_CTL_HW_ENABLE); + + I915_WRITE(GEN6_RPNSWREQ, + GEN6_FREQUENCY(10) | + GEN6_OFFSET(0) | + GEN6_AGGRESSIVE_TURBO); + I915_WRITE(GEN6_RC_VIDEO_FREQ, + GEN6_FREQUENCY(12)); + + I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000); + I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, + 18 << 24 | + 6 << 16); + I915_WRITE(GEN6_RP_UP_THRESHOLD, 90000); + I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 100000); + I915_WRITE(GEN6_RP_UP_EI, 100000); + I915_WRITE(GEN6_RP_DOWN_EI, 300000); + I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10); + I915_WRITE(GEN6_RP_CONTROL, + GEN6_RP_MEDIA_TURBO | + GEN6_RP_USE_NORMAL_FREQ | + GEN6_RP_MEDIA_IS_GFX | + GEN6_RP_ENABLE | + GEN6_RP_UP_BUSY_MAX | + GEN6_RP_DOWN_BUSY_MIN); + + if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, + 500)) + DRM_ERROR("timeout waiting for pcode mailbox to become idle\n"); + + I915_WRITE(GEN6_PCODE_DATA, 0); + I915_WRITE(GEN6_PCODE_MAILBOX, + GEN6_PCODE_READY | + GEN6_PCODE_WRITE_MIN_FREQ_TABLE); + if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, + 500)) + DRM_ERROR("timeout waiting for pcode mailbox to finish\n"); + + min_freq = (rp_state_cap & 0xff0000) >> 16; + max_freq = rp_state_cap & 0xff; + cur_freq = (gt_perf_status & 0xff00) >> 8; + + /* Check for overclock support */ + if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, + 500)) + DRM_ERROR("timeout waiting for pcode mailbox to become idle\n"); + I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS); + pcu_mbox = I915_READ(GEN6_PCODE_DATA); + if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, + 500)) + DRM_ERROR("timeout waiting for pcode mailbox to finish\n"); + if (pcu_mbox & (1<<31)) { /* OC supported */ + max_freq = pcu_mbox & 0xff; + DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 100); + } + + /* In units of 100MHz */ + dev_priv->max_delay = max_freq; + dev_priv->min_delay = min_freq; + dev_priv->cur_delay = cur_freq; + + /* requires MSI enabled */ + I915_WRITE(GEN6_PMIER, + GEN6_PM_MBOX_EVENT | + GEN6_PM_THERMAL_EVENT | + GEN6_PM_RP_DOWN_TIMEOUT | + GEN6_PM_RP_UP_THRESHOLD | + GEN6_PM_RP_DOWN_THRESHOLD | + GEN6_PM_RP_UP_EI_EXPIRED | + GEN6_PM_RP_DOWN_EI_EXPIRED); + I915_WRITE(GEN6_PMIMR, 0); + /* enable all PM interrupts */ + I915_WRITE(GEN6_PMINTRMSK, 0); + + __gen6_force_wake_put(dev_priv); +} + +void intel_enable_clock_gating(struct drm_device *dev) { struct drm_i915_private *dev_priv = dev->dev_private; @@ -5835,7 +6396,9 @@ void intel_init_clock_gating(struct drm_device *dev) if (IS_GEN5(dev)) { /* Required for FBC */ - dspclk_gate |= DPFDUNIT_CLOCK_GATE_DISABLE; + dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE | + DPFCRUNIT_CLOCK_GATE_DISABLE | + DPFDUNIT_CLOCK_GATE_DISABLE; /* Required for CxSR */ dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE; @@ -5872,9 +6435,9 @@ void intel_init_clock_gating(struct drm_device *dev) I915_WRITE(DISP_ARB_CTL, (I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS)); - I915_WRITE(WM3_LP_ILK, 0); - I915_WRITE(WM2_LP_ILK, 0); - I915_WRITE(WM1_LP_ILK, 0); + I915_WRITE(WM3_LP_ILK, 0); + I915_WRITE(WM2_LP_ILK, 0); + I915_WRITE(WM1_LP_ILK, 0); } /* * Based on the document from hardware guys the following bits @@ -5896,7 +6459,49 @@ void intel_init_clock_gating(struct drm_device *dev) ILK_DPFC_DIS2 | ILK_CLK_FBC); } - return; + + I915_WRITE(ILK_DISPLAY_CHICKEN2, + I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_ELPIN_409_SELECT); + + if (IS_GEN5(dev)) { + I915_WRITE(_3D_CHICKEN2, + _3D_CHICKEN2_WM_READ_PIPELINED << 16 | + _3D_CHICKEN2_WM_READ_PIPELINED); + } + + if (IS_GEN6(dev)) { + I915_WRITE(WM3_LP_ILK, 0); + I915_WRITE(WM2_LP_ILK, 0); + I915_WRITE(WM1_LP_ILK, 0); + + /* + * According to the spec the following bits should be + * set in order to enable memory self-refresh and fbc: + * The bit21 and bit22 of 0x42000 + * The bit21 and bit22 of 0x42004 + * The bit5 and bit7 of 0x42020 + * The bit14 of 0x70180 + * The bit14 of 0x71180 + */ + I915_WRITE(ILK_DISPLAY_CHICKEN1, + I915_READ(ILK_DISPLAY_CHICKEN1) | + ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS); + I915_WRITE(ILK_DISPLAY_CHICKEN2, + I915_READ(ILK_DISPLAY_CHICKEN2) | + ILK_DPARB_GATE | ILK_VSDPFD_FULL); + I915_WRITE(ILK_DSPCLK_GATE, + I915_READ(ILK_DSPCLK_GATE) | + ILK_DPARB_CLK_GATE | + ILK_DPFD_CLK_GATE); + + I915_WRITE(DSPACNTR, + I915_READ(DSPACNTR) | + DISPPLANE_TRICKLE_FEED_DISABLE); + I915_WRITE(DSPBCNTR, + I915_READ(DSPBCNTR) | + DISPPLANE_TRICKLE_FEED_DISABLE); + } } else if (IS_G4X(dev)) { uint32_t dspclk_gate; I915_WRITE(RENCLK_GATE_D1, 0); @@ -5934,57 +6539,106 @@ void intel_init_clock_gating(struct drm_device *dev) } else if (IS_I830(dev)) { I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE); } +} + +static void ironlake_teardown_rc6(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (dev_priv->renderctx) { + i915_gem_object_unpin(dev_priv->renderctx); + drm_gem_object_unreference(&dev_priv->renderctx->base); + dev_priv->renderctx = NULL; + } + + if (dev_priv->pwrctx) { + i915_gem_object_unpin(dev_priv->pwrctx); + drm_gem_object_unreference(&dev_priv->pwrctx->base); + dev_priv->pwrctx = NULL; + } +} + +static void ironlake_disable_rc6(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (I915_READ(PWRCTXA)) { + /* Wake the GPU, prevent RC6, then restore RSTDBYCTL */ + I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT); + wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON), + 50); + + I915_WRITE(PWRCTXA, 0); + POSTING_READ(PWRCTXA); + + I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT); + POSTING_READ(RSTDBYCTL); + } + + ironlake_teardown_rc6(dev); +} + +static int ironlake_setup_rc6(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (dev_priv->renderctx == NULL) + dev_priv->renderctx = intel_alloc_context_page(dev); + if (!dev_priv->renderctx) + return -ENOMEM; + + if (dev_priv->pwrctx == NULL) + dev_priv->pwrctx = intel_alloc_context_page(dev); + if (!dev_priv->pwrctx) { + ironlake_teardown_rc6(dev); + return -ENOMEM; + } + + return 0; +} + +void ironlake_enable_rc6(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + int ret; + + /* rc6 disabled by default due to repeated reports of hanging during + * boot and resume. + */ + if (!i915_enable_rc6) + return; + + ret = ironlake_setup_rc6(dev); + if (ret) + return; /* * GPU can automatically power down the render unit if given a page * to save state. */ - if (IS_IRONLAKE_M(dev)) { - if (dev_priv->renderctx == NULL) - dev_priv->renderctx = intel_alloc_context_page(dev); - if (dev_priv->renderctx) { - struct drm_i915_gem_object *obj_priv; - obj_priv = to_intel_bo(dev_priv->renderctx); - if (obj_priv) { - BEGIN_LP_RING(4); - OUT_RING(MI_SET_CONTEXT); - OUT_RING(obj_priv->gtt_offset | - MI_MM_SPACE_GTT | - MI_SAVE_EXT_STATE_EN | - MI_RESTORE_EXT_STATE_EN | - MI_RESTORE_INHIBIT); - OUT_RING(MI_NOOP); - OUT_RING(MI_FLUSH); - ADVANCE_LP_RING(); - } - } else - DRM_DEBUG_KMS("Failed to allocate render context." - "Disable RC6\n"); + ret = BEGIN_LP_RING(6); + if (ret) { + ironlake_teardown_rc6(dev); + return; } - if (I915_HAS_RC6(dev) && drm_core_check_feature(dev, DRIVER_MODESET)) { - struct drm_i915_gem_object *obj_priv = NULL; - - if (dev_priv->pwrctx) { - obj_priv = to_intel_bo(dev_priv->pwrctx); - } else { - struct drm_gem_object *pwrctx; - - pwrctx = intel_alloc_context_page(dev); - if (pwrctx) { - dev_priv->pwrctx = pwrctx; - obj_priv = to_intel_bo(pwrctx); - } - } + OUT_RING(MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN); + OUT_RING(MI_SET_CONTEXT); + OUT_RING(dev_priv->renderctx->gtt_offset | + MI_MM_SPACE_GTT | + MI_SAVE_EXT_STATE_EN | + MI_RESTORE_EXT_STATE_EN | + MI_RESTORE_INHIBIT); + OUT_RING(MI_SUSPEND_FLUSH); + OUT_RING(MI_NOOP); + OUT_RING(MI_FLUSH); + ADVANCE_LP_RING(); - if (obj_priv) { - I915_WRITE(PWRCTXA, obj_priv->gtt_offset | PWRCTX_EN); - I915_WRITE(MCHBAR_RENDER_STANDBY, - I915_READ(MCHBAR_RENDER_STANDBY) & ~RCX_SW_EXIT); - } - } + I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN); + I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT); } + /* Set up chip specific display functions */ static void intel_init_display(struct drm_device *dev) { @@ -5997,7 +6651,7 @@ static void intel_init_display(struct drm_device *dev) dev_priv->display.dpms = i9xx_crtc_dpms; if (I915_HAS_FBC(dev)) { - if (IS_IRONLAKE_M(dev)) { + if (HAS_PCH_SPLIT(dev)) { dev_priv->display.fbc_enabled = ironlake_fbc_enabled; dev_priv->display.enable_fbc = ironlake_enable_fbc; dev_priv->display.disable_fbc = ironlake_disable_fbc; @@ -6046,6 +6700,14 @@ static void intel_init_display(struct drm_device *dev) "Disable CxSR\n"); dev_priv->display.update_wm = NULL; } + } else if (IS_GEN6(dev)) { + if (SNB_READ_WM0_LATENCY()) { + dev_priv->display.update_wm = sandybridge_update_wm; + } else { + DRM_DEBUG_KMS("Failed to read display plane latency. " + "Disable CxSR\n"); + dev_priv->display.update_wm = NULL; + } } else dev_priv->display.update_wm = NULL; } else if (IS_PINEVIEW(dev)) { @@ -6191,12 +6853,7 @@ void intel_modeset_init(struct drm_device *dev) dev->mode_config.max_width = 8192; dev->mode_config.max_height = 8192; } - - /* set memory base */ - if (IS_GEN2(dev)) - dev->mode_config.fb_base = pci_resource_start(dev->pdev, 0); - else - dev->mode_config.fb_base = pci_resource_start(dev->pdev, 2); + dev->mode_config.fb_base = dev->agp->base; if (IS_MOBILE(dev) || !IS_GEN2(dev)) dev_priv->num_pipe = 2; @@ -6211,7 +6868,7 @@ void intel_modeset_init(struct drm_device *dev) intel_setup_outputs(dev); - intel_init_clock_gating(dev); + intel_enable_clock_gating(dev); /* Just disable it once at startup */ i915_disable_vga(dev); @@ -6221,6 +6878,12 @@ void intel_modeset_init(struct drm_device *dev) intel_init_emon(dev); } + if (IS_GEN6(dev)) + gen6_enable_rps(dev_priv); + + if (IS_IRONLAKE_M(dev)) + ironlake_enable_rc6(dev); + INIT_WORK(&dev_priv->idle_work, intel_idle_update); setup_timer(&dev_priv->idle_timer, intel_gpu_idle_timer, (unsigned long)dev); @@ -6252,28 +6915,13 @@ void intel_modeset_cleanup(struct drm_device *dev) if (dev_priv->display.disable_fbc) dev_priv->display.disable_fbc(dev); - if (dev_priv->renderctx) { - struct drm_i915_gem_object *obj_priv; - - obj_priv = to_intel_bo(dev_priv->renderctx); - I915_WRITE(CCID, obj_priv->gtt_offset &~ CCID_EN); - I915_READ(CCID); - i915_gem_object_unpin(dev_priv->renderctx); - drm_gem_object_unreference(dev_priv->renderctx); - } - - if (dev_priv->pwrctx) { - struct drm_i915_gem_object *obj_priv; - - obj_priv = to_intel_bo(dev_priv->pwrctx); - I915_WRITE(PWRCTXA, obj_priv->gtt_offset &~ PWRCTX_EN); - I915_READ(PWRCTXA); - i915_gem_object_unpin(dev_priv->pwrctx); - drm_gem_object_unreference(dev_priv->pwrctx); - } - if (IS_IRONLAKE_M(dev)) ironlake_disable_drps(dev); + if (IS_GEN6(dev)) + gen6_disable_rps(dev); + + if (IS_IRONLAKE_M(dev)) + ironlake_disable_rc6(dev); mutex_unlock(&dev->struct_mutex); @@ -6325,3 +6973,113 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state) pci_write_config_word(dev_priv->bridge_dev, INTEL_GMCH_CTRL, gmch_ctrl); return 0; } + +#ifdef CONFIG_DEBUG_FS +#include <linux/seq_file.h> + +struct intel_display_error_state { + struct intel_cursor_error_state { + u32 control; + u32 position; + u32 base; + u32 size; + } cursor[2]; + + struct intel_pipe_error_state { + u32 conf; + u32 source; + + u32 htotal; + u32 hblank; + u32 hsync; + u32 vtotal; + u32 vblank; + u32 vsync; + } pipe[2]; + + struct intel_plane_error_state { + u32 control; + u32 stride; + u32 size; + u32 pos; + u32 addr; + u32 surface; + u32 tile_offset; + } plane[2]; +}; + +struct intel_display_error_state * +intel_display_capture_error_state(struct drm_device *dev) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_display_error_state *error; + int i; + + error = kmalloc(sizeof(*error), GFP_ATOMIC); + if (error == NULL) + return NULL; + + for (i = 0; i < 2; i++) { + error->cursor[i].control = I915_READ(CURCNTR(i)); + error->cursor[i].position = I915_READ(CURPOS(i)); + error->cursor[i].base = I915_READ(CURBASE(i)); + + error->plane[i].control = I915_READ(DSPCNTR(i)); + error->plane[i].stride = I915_READ(DSPSTRIDE(i)); + error->plane[i].size = I915_READ(DSPSIZE(i)); + error->plane[i].pos= I915_READ(DSPPOS(i)); + error->plane[i].addr = I915_READ(DSPADDR(i)); + if (INTEL_INFO(dev)->gen >= 4) { + error->plane[i].surface = I915_READ(DSPSURF(i)); + error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i)); + } + + error->pipe[i].conf = I915_READ(PIPECONF(i)); + error->pipe[i].source = I915_READ(PIPESRC(i)); + error->pipe[i].htotal = I915_READ(HTOTAL(i)); + error->pipe[i].hblank = I915_READ(HBLANK(i)); + error->pipe[i].hsync = I915_READ(HSYNC(i)); + error->pipe[i].vtotal = I915_READ(VTOTAL(i)); + error->pipe[i].vblank = I915_READ(VBLANK(i)); + error->pipe[i].vsync = I915_READ(VSYNC(i)); + } + + return error; +} + +void +intel_display_print_error_state(struct seq_file *m, + struct drm_device *dev, + struct intel_display_error_state *error) +{ + int i; + + for (i = 0; i < 2; i++) { + seq_printf(m, "Pipe [%d]:\n", i); + seq_printf(m, " CONF: %08x\n", error->pipe[i].conf); + seq_printf(m, " SRC: %08x\n", error->pipe[i].source); + seq_printf(m, " HTOTAL: %08x\n", error->pipe[i].htotal); + seq_printf(m, " HBLANK: %08x\n", error->pipe[i].hblank); + seq_printf(m, " HSYNC: %08x\n", error->pipe[i].hsync); + seq_printf(m, " VTOTAL: %08x\n", error->pipe[i].vtotal); + seq_printf(m, " VBLANK: %08x\n", error->pipe[i].vblank); + seq_printf(m, " VSYNC: %08x\n", error->pipe[i].vsync); + + seq_printf(m, "Plane [%d]:\n", i); + seq_printf(m, " CNTR: %08x\n", error->plane[i].control); + seq_printf(m, " STRIDE: %08x\n", error->plane[i].stride); + seq_printf(m, " SIZE: %08x\n", error->plane[i].size); + seq_printf(m, " POS: %08x\n", error->plane[i].pos); + seq_printf(m, " ADDR: %08x\n", error->plane[i].addr); + if (INTEL_INFO(dev)->gen >= 4) { + seq_printf(m, " SURF: %08x\n", error->plane[i].surface); + seq_printf(m, " TILEOFF: %08x\n", error->plane[i].tile_offset); + } + + seq_printf(m, "Cursor [%d]:\n", i); + seq_printf(m, " CNTR: %08x\n", error->cursor[i].control); + seq_printf(m, " POS: %08x\n", error->cursor[i].position); + seq_printf(m, " BASE: %08x\n", error->cursor[i].base); + } +} +#endif diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c index 864417cffe9a..51cb4e36997f 100644 --- a/drivers/gpu/drm/i915/intel_dp.c +++ b/drivers/gpu/drm/i915/intel_dp.c @@ -1153,18 +1153,27 @@ intel_dp_signal_levels(uint8_t train_set, int lane_count) static uint32_t intel_gen6_edp_signal_levels(uint8_t train_set) { - switch (train_set & (DP_TRAIN_VOLTAGE_SWING_MASK|DP_TRAIN_PRE_EMPHASIS_MASK)) { + int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK | + DP_TRAIN_PRE_EMPHASIS_MASK); + switch (signal_levels) { case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0: - return EDP_LINK_TRAIN_400MV_0DB_SNB_B; + case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0: + return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B; + case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5: + return EDP_LINK_TRAIN_400MV_3_5DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6: - return EDP_LINK_TRAIN_400MV_6DB_SNB_B; + case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6: + return EDP_LINK_TRAIN_400_600MV_6DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5: - return EDP_LINK_TRAIN_600MV_3_5DB_SNB_B; + case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5: + return EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B; case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0: - return EDP_LINK_TRAIN_800MV_0DB_SNB_B; + case DP_TRAIN_VOLTAGE_SWING_1200 | DP_TRAIN_PRE_EMPHASIS_0: + return EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B; default: - DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level\n"); - return EDP_LINK_TRAIN_400MV_0DB_SNB_B; + DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:" + "0x%x\n", signal_levels); + return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B; } } @@ -1334,17 +1343,24 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp) struct drm_device *dev = intel_dp->base.base.dev; struct drm_i915_private *dev_priv = dev->dev_private; bool channel_eq = false; - int tries; + int tries, cr_tries; u32 reg; uint32_t DP = intel_dp->DP; /* channel equalization */ tries = 0; + cr_tries = 0; channel_eq = false; for (;;) { /* Use intel_dp->train_set[0] to set the voltage and pre emphasis values */ uint32_t signal_levels; + if (cr_tries > 5) { + DRM_ERROR("failed to train DP, aborting\n"); + intel_dp_link_down(intel_dp); + break; + } + if (IS_GEN6(dev) && is_edp(intel_dp)) { signal_levels = intel_gen6_edp_signal_levels(intel_dp->train_set[0]); DP = (DP & ~EDP_LINK_TRAIN_VOL_EMP_MASK_SNB) | signal_levels; @@ -1367,14 +1383,26 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp) if (!intel_dp_get_link_status(intel_dp)) break; + /* Make sure clock is still ok */ + if (!intel_clock_recovery_ok(intel_dp->link_status, intel_dp->lane_count)) { + intel_dp_start_link_train(intel_dp); + cr_tries++; + continue; + } + if (intel_channel_eq_ok(intel_dp)) { channel_eq = true; break; } - /* Try 5 times */ - if (tries > 5) - break; + /* Try 5 times, then try clock recovery if that fails */ + if (tries > 5) { + intel_dp_link_down(intel_dp); + intel_dp_start_link_train(intel_dp); + tries = 0; + cr_tries++; + continue; + } /* Compute new intel_dp->train_set as requested by target */ intel_get_adjust_train(intel_dp); @@ -1442,8 +1470,7 @@ intel_dp_link_down(struct intel_dp *intel_dp) /* Changes to enable or select take place the vblank * after being written. */ - intel_wait_for_vblank(intel_dp->base.base.dev, - intel_crtc->pipe); + intel_wait_for_vblank(dev, intel_crtc->pipe); } I915_WRITE(intel_dp->output_reg, DP & ~DP_PORT_EN); @@ -1612,6 +1639,24 @@ static int intel_dp_get_modes(struct drm_connector *connector) return 0; } +static bool +intel_dp_detect_audio(struct drm_connector *connector) +{ + struct intel_dp *intel_dp = intel_attached_dp(connector); + struct edid *edid; + bool has_audio = false; + + edid = drm_get_edid(connector, &intel_dp->adapter); + if (edid) { + has_audio = drm_detect_monitor_audio(edid); + + connector->display_info.raw_edid = NULL; + kfree(edid); + } + + return has_audio; +} + static int intel_dp_set_property(struct drm_connector *connector, struct drm_property *property, @@ -1625,17 +1670,23 @@ intel_dp_set_property(struct drm_connector *connector, return ret; if (property == intel_dp->force_audio_property) { - if (val == intel_dp->force_audio) + int i = val; + bool has_audio; + + if (i == intel_dp->force_audio) return 0; - intel_dp->force_audio = val; + intel_dp->force_audio = i; - if (val > 0 && intel_dp->has_audio) - return 0; - if (val < 0 && !intel_dp->has_audio) + if (i == 0) + has_audio = intel_dp_detect_audio(connector); + else + has_audio = i > 0; + + if (has_audio == intel_dp->has_audio) return 0; - intel_dp->has_audio = val > 0; + intel_dp->has_audio = has_audio; goto done; } diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h index e52c6125bb1f..2c431049963c 100644 --- a/drivers/gpu/drm/i915/intel_drv.h +++ b/drivers/gpu/drm/i915/intel_drv.h @@ -127,7 +127,7 @@ intel_mode_get_pixel_multiplier(const struct drm_display_mode *mode) struct intel_framebuffer { struct drm_framebuffer base; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; }; struct intel_fbdev { @@ -166,7 +166,7 @@ struct intel_crtc { struct intel_unpin_work *unpin_work; int fdi_lanes; - struct drm_gem_object *cursor_bo; + struct drm_i915_gem_object *cursor_bo; uint32_t cursor_addr; int16_t cursor_x, cursor_y; int16_t cursor_width, cursor_height; @@ -220,8 +220,8 @@ intel_get_crtc_for_pipe(struct drm_device *dev, int pipe) struct intel_unpin_work { struct work_struct work; struct drm_device *dev; - struct drm_gem_object *old_fb_obj; - struct drm_gem_object *pending_flip_obj; + struct drm_i915_gem_object *old_fb_obj; + struct drm_i915_gem_object *pending_flip_obj; struct drm_pending_vblank_event *event; int pending; bool enable_stall_check; @@ -236,7 +236,8 @@ void intel_dip_infoframe_csum(struct dip_infoframe *avi_if); extern bool intel_sdvo_init(struct drm_device *dev, int output_device); extern void intel_dvo_init(struct drm_device *dev); extern void intel_tv_init(struct drm_device *dev); -extern void intel_mark_busy(struct drm_device *dev, struct drm_gem_object *obj); +extern void intel_mark_busy(struct drm_device *dev, + struct drm_i915_gem_object *obj); extern bool intel_lvds_init(struct drm_device *dev); extern void intel_dp_init(struct drm_device *dev, int dp_reg); void @@ -256,6 +257,9 @@ extern void intel_pch_panel_fitting(struct drm_device *dev, extern u32 intel_panel_get_max_backlight(struct drm_device *dev); extern u32 intel_panel_get_backlight(struct drm_device *dev); extern void intel_panel_set_backlight(struct drm_device *dev, u32 level); +extern void intel_panel_setup_backlight(struct drm_device *dev); +extern void intel_panel_enable_backlight(struct drm_device *dev); +extern void intel_panel_disable_backlight(struct drm_device *dev); extern void intel_crtc_load_lut(struct drm_crtc *crtc); extern void intel_encoder_prepare (struct drm_encoder *encoder); @@ -293,19 +297,21 @@ extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green, u16 blue, int regno); extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue, int regno); -extern void intel_init_clock_gating(struct drm_device *dev); +extern void intel_enable_clock_gating(struct drm_device *dev); extern void ironlake_enable_drps(struct drm_device *dev); extern void ironlake_disable_drps(struct drm_device *dev); +extern void gen6_enable_rps(struct drm_i915_private *dev_priv); +extern void gen6_disable_rps(struct drm_device *dev); extern void intel_init_emon(struct drm_device *dev); extern int intel_pin_and_fence_fb_obj(struct drm_device *dev, - struct drm_gem_object *obj, - bool pipelined); + struct drm_i915_gem_object *obj, + struct intel_ring_buffer *pipelined); extern int intel_framebuffer_init(struct drm_device *dev, struct intel_framebuffer *ifb, struct drm_mode_fb_cmd *mode_cmd, - struct drm_gem_object *obj); + struct drm_i915_gem_object *obj); extern int intel_fbdev_init(struct drm_device *dev); extern void intel_fbdev_fini(struct drm_device *dev); diff --git a/drivers/gpu/drm/i915/intel_fb.c b/drivers/gpu/drm/i915/intel_fb.c index af2a1dddc28e..512782728e51 100644 --- a/drivers/gpu/drm/i915/intel_fb.c +++ b/drivers/gpu/drm/i915/intel_fb.c @@ -62,13 +62,13 @@ static int intelfb_create(struct intel_fbdev *ifbdev, struct drm_fb_helper_surface_size *sizes) { struct drm_device *dev = ifbdev->helper.dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct fb_info *info; struct drm_framebuffer *fb; struct drm_mode_fb_cmd mode_cmd; - struct drm_gem_object *fbo = NULL; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; struct device *device = &dev->pdev->dev; - int size, ret, mmio_bar = IS_GEN2(dev) ? 1 : 0; + int size, ret; /* we don't do packed 24bpp */ if (sizes->surface_bpp == 24) @@ -78,23 +78,22 @@ static int intelfb_create(struct intel_fbdev *ifbdev, mode_cmd.height = sizes->surface_height; mode_cmd.bpp = sizes->surface_bpp; - mode_cmd.pitch = ALIGN(mode_cmd.width * ((mode_cmd.bpp + 1) / 8), 64); + mode_cmd.pitch = ALIGN(mode_cmd.width * ((mode_cmd.bpp + 7) / 8), 64); mode_cmd.depth = sizes->surface_depth; size = mode_cmd.pitch * mode_cmd.height; size = ALIGN(size, PAGE_SIZE); - fbo = i915_gem_alloc_object(dev, size); - if (!fbo) { + obj = i915_gem_alloc_object(dev, size); + if (!obj) { DRM_ERROR("failed to allocate framebuffer\n"); ret = -ENOMEM; goto out; } - obj_priv = to_intel_bo(fbo); mutex_lock(&dev->struct_mutex); /* Flush everything out, we'll be doing GTT only from now on */ - ret = intel_pin_and_fence_fb_obj(dev, fbo, false); + ret = intel_pin_and_fence_fb_obj(dev, obj, false); if (ret) { DRM_ERROR("failed to pin fb: %d\n", ret); goto out_unref; @@ -108,7 +107,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev, info->par = ifbdev; - ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, fbo); + ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj); if (ret) goto out_unpin; @@ -122,6 +121,11 @@ static int intelfb_create(struct intel_fbdev *ifbdev, info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT; info->fbops = &intelfb_ops; + ret = fb_alloc_cmap(&info->cmap, 256, 0); + if (ret) { + ret = -ENOMEM; + goto out_unpin; + } /* setup aperture base/size for vesafb takeover */ info->apertures = alloc_apertures(1); if (!info->apertures) { @@ -129,26 +133,17 @@ static int intelfb_create(struct intel_fbdev *ifbdev, goto out_unpin; } info->apertures->ranges[0].base = dev->mode_config.fb_base; - if (!IS_GEN2(dev)) - info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 2); - else - info->apertures->ranges[0].size = pci_resource_len(dev->pdev, 0); + info->apertures->ranges[0].size = + dev_priv->mm.gtt->gtt_mappable_entries << PAGE_SHIFT; - info->fix.smem_start = dev->mode_config.fb_base + obj_priv->gtt_offset; + info->fix.smem_start = dev->mode_config.fb_base + obj->gtt_offset; info->fix.smem_len = size; - info->screen_base = ioremap_wc(dev->agp->base + obj_priv->gtt_offset, - size); + info->screen_base = ioremap_wc(dev->agp->base + obj->gtt_offset, size); if (!info->screen_base) { ret = -ENOSPC; goto out_unpin; } - - ret = fb_alloc_cmap(&info->cmap, 256, 0); - if (ret) { - ret = -ENOMEM; - goto out_unpin; - } info->screen_size = size; // memset(info->screen_base, 0, size); @@ -156,10 +151,6 @@ static int intelfb_create(struct intel_fbdev *ifbdev, drm_fb_helper_fill_fix(info, fb->pitch, fb->depth); drm_fb_helper_fill_var(info, &ifbdev->helper, sizes->fb_width, sizes->fb_height); - /* FIXME: we really shouldn't expose mmio space at all */ - info->fix.mmio_start = pci_resource_start(dev->pdev, mmio_bar); - info->fix.mmio_len = pci_resource_len(dev->pdev, mmio_bar); - info->pixmap.size = 64*1024; info->pixmap.buf_align = 8; info->pixmap.access_align = 32; @@ -168,7 +159,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev, DRM_DEBUG_KMS("allocated %dx%d fb: 0x%08x, bo %p\n", fb->width, fb->height, - obj_priv->gtt_offset, fbo); + obj->gtt_offset, obj); mutex_unlock(&dev->struct_mutex); @@ -176,9 +167,9 @@ static int intelfb_create(struct intel_fbdev *ifbdev, return 0; out_unpin: - i915_gem_object_unpin(fbo); + i915_gem_object_unpin(obj); out_unref: - drm_gem_object_unreference(fbo); + drm_gem_object_unreference(&obj->base); mutex_unlock(&dev->struct_mutex); out: return ret; @@ -225,7 +216,7 @@ static void intel_fbdev_destroy(struct drm_device *dev, drm_framebuffer_cleanup(&ifb->base); if (ifb->obj) { - drm_gem_object_unreference_unlocked(ifb->obj); + drm_gem_object_unreference_unlocked(&ifb->obj->base); ifb->obj = NULL; } } diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c index 0d0273e7b029..c635c9e357b9 100644 --- a/drivers/gpu/drm/i915/intel_hdmi.c +++ b/drivers/gpu/drm/i915/intel_hdmi.c @@ -251,6 +251,27 @@ static int intel_hdmi_get_modes(struct drm_connector *connector) &dev_priv->gmbus[intel_hdmi->ddc_bus].adapter); } +static bool +intel_hdmi_detect_audio(struct drm_connector *connector) +{ + struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector); + struct drm_i915_private *dev_priv = connector->dev->dev_private; + struct edid *edid; + bool has_audio = false; + + edid = drm_get_edid(connector, + &dev_priv->gmbus[intel_hdmi->ddc_bus].adapter); + if (edid) { + if (edid->input & DRM_EDID_INPUT_DIGITAL) + has_audio = drm_detect_monitor_audio(edid); + + connector->display_info.raw_edid = NULL; + kfree(edid); + } + + return has_audio; +} + static int intel_hdmi_set_property(struct drm_connector *connector, struct drm_property *property, @@ -264,17 +285,23 @@ intel_hdmi_set_property(struct drm_connector *connector, return ret; if (property == intel_hdmi->force_audio_property) { - if (val == intel_hdmi->force_audio) + int i = val; + bool has_audio; + + if (i == intel_hdmi->force_audio) return 0; - intel_hdmi->force_audio = val; + intel_hdmi->force_audio = i; - if (val > 0 && intel_hdmi->has_audio) - return 0; - if (val < 0 && !intel_hdmi->has_audio) + if (i == 0) + has_audio = intel_hdmi_detect_audio(connector); + else + has_audio = i > 0; + + if (has_audio == intel_hdmi->has_audio) return 0; - intel_hdmi->has_audio = val > 0; + intel_hdmi->has_audio = has_audio; goto done; } diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c index 3dba086e7eea..58040f68ed7a 100644 --- a/drivers/gpu/drm/i915/intel_i2c.c +++ b/drivers/gpu/drm/i915/intel_i2c.c @@ -85,8 +85,9 @@ static u32 get_reserved(struct intel_gpio *gpio) /* On most chips, these bits must be preserved in software. */ if (!IS_I830(dev) && !IS_845G(dev)) - reserved = I915_READ(gpio->reg) & (GPIO_DATA_PULLUP_DISABLE | - GPIO_CLOCK_PULLUP_DISABLE); + reserved = I915_READ_NOTRACE(gpio->reg) & + (GPIO_DATA_PULLUP_DISABLE | + GPIO_CLOCK_PULLUP_DISABLE); return reserved; } @@ -96,9 +97,9 @@ static int get_clock(void *data) struct intel_gpio *gpio = data; struct drm_i915_private *dev_priv = gpio->dev_priv; u32 reserved = get_reserved(gpio); - I915_WRITE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK); - I915_WRITE(gpio->reg, reserved); - return (I915_READ(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0; + I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK); + I915_WRITE_NOTRACE(gpio->reg, reserved); + return (I915_READ_NOTRACE(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0; } static int get_data(void *data) @@ -106,9 +107,9 @@ static int get_data(void *data) struct intel_gpio *gpio = data; struct drm_i915_private *dev_priv = gpio->dev_priv; u32 reserved = get_reserved(gpio); - I915_WRITE(gpio->reg, reserved | GPIO_DATA_DIR_MASK); - I915_WRITE(gpio->reg, reserved); - return (I915_READ(gpio->reg) & GPIO_DATA_VAL_IN) != 0; + I915_WRITE_NOTRACE(gpio->reg, reserved | GPIO_DATA_DIR_MASK); + I915_WRITE_NOTRACE(gpio->reg, reserved); + return (I915_READ_NOTRACE(gpio->reg) & GPIO_DATA_VAL_IN) != 0; } static void set_clock(void *data, int state_high) @@ -124,7 +125,7 @@ static void set_clock(void *data, int state_high) clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK | GPIO_CLOCK_VAL_MASK; - I915_WRITE(gpio->reg, reserved | clock_bits); + I915_WRITE_NOTRACE(gpio->reg, reserved | clock_bits); POSTING_READ(gpio->reg); } @@ -141,7 +142,7 @@ static void set_data(void *data, int state_high) data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK | GPIO_DATA_VAL_MASK; - I915_WRITE(gpio->reg, reserved | data_bits); + I915_WRITE_NOTRACE(gpio->reg, reserved | data_bits); POSTING_READ(gpio->reg); } diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c index 25bcedf386fd..bcdba7bd5cfa 100644 --- a/drivers/gpu/drm/i915/intel_lvds.c +++ b/drivers/gpu/drm/i915/intel_lvds.c @@ -106,7 +106,7 @@ static void intel_lvds_enable(struct intel_lvds *intel_lvds) I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON); POSTING_READ(lvds_reg); - intel_panel_set_backlight(dev, dev_priv->backlight_level); + intel_panel_enable_backlight(dev); } static void intel_lvds_disable(struct intel_lvds *intel_lvds) @@ -123,8 +123,7 @@ static void intel_lvds_disable(struct intel_lvds *intel_lvds) lvds_reg = LVDS; } - dev_priv->backlight_level = intel_panel_get_backlight(dev); - intel_panel_set_backlight(dev, 0); + intel_panel_disable_backlight(dev); I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON); @@ -262,12 +261,6 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder, return true; } - /* Make sure pre-965s set dither correctly */ - if (INTEL_INFO(dev)->gen < 4) { - if (dev_priv->lvds_dither) - pfit_control |= PANEL_8TO6_DITHER_ENABLE; - } - /* Native modes don't need fitting */ if (adjusted_mode->hdisplay == mode->hdisplay && adjusted_mode->vdisplay == mode->vdisplay) @@ -304,14 +297,13 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder, u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay; u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay; - pfit_control |= PFIT_ENABLE; /* 965+ is easy, it does everything in hw */ if (scaled_width > scaled_height) - pfit_control |= PFIT_SCALING_PILLAR; + pfit_control |= PFIT_ENABLE | PFIT_SCALING_PILLAR; else if (scaled_width < scaled_height) - pfit_control |= PFIT_SCALING_LETTER; - else - pfit_control |= PFIT_SCALING_AUTO; + pfit_control |= PFIT_ENABLE | PFIT_SCALING_LETTER; + else if (adjusted_mode->hdisplay != mode->hdisplay) + pfit_control |= PFIT_ENABLE | PFIT_SCALING_AUTO; } else { u32 scaled_width = adjusted_mode->hdisplay * mode->vdisplay; u32 scaled_height = mode->hdisplay * adjusted_mode->vdisplay; @@ -358,13 +350,17 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder, * Full scaling, even if it changes the aspect ratio. * Fortunately this is all done for us in hw. */ - pfit_control |= PFIT_ENABLE; - if (INTEL_INFO(dev)->gen >= 4) - pfit_control |= PFIT_SCALING_AUTO; - else - pfit_control |= (VERT_AUTO_SCALE | HORIZ_AUTO_SCALE | - VERT_INTERP_BILINEAR | - HORIZ_INTERP_BILINEAR); + if (mode->vdisplay != adjusted_mode->vdisplay || + mode->hdisplay != adjusted_mode->hdisplay) { + pfit_control |= PFIT_ENABLE; + if (INTEL_INFO(dev)->gen >= 4) + pfit_control |= PFIT_SCALING_AUTO; + else + pfit_control |= (VERT_AUTO_SCALE | + VERT_INTERP_BILINEAR | + HORIZ_AUTO_SCALE | + HORIZ_INTERP_BILINEAR); + } break; default: @@ -372,6 +368,16 @@ static bool intel_lvds_mode_fixup(struct drm_encoder *encoder, } out: + /* If not enabling scaling, be consistent and always use 0. */ + if ((pfit_control & PFIT_ENABLE) == 0) { + pfit_control = 0; + pfit_pgm_ratios = 0; + } + + /* Make sure pre-965 set dither correctly */ + if (INTEL_INFO(dev)->gen < 4 && dev_priv->lvds_dither) + pfit_control |= PANEL_8TO6_DITHER_ENABLE; + if (pfit_control != intel_lvds->pfit_control || pfit_pgm_ratios != intel_lvds->pfit_pgm_ratios) { intel_lvds->pfit_control = pfit_control; @@ -395,8 +401,6 @@ static void intel_lvds_prepare(struct drm_encoder *encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_lvds *intel_lvds = to_intel_lvds(encoder); - dev_priv->backlight_level = intel_panel_get_backlight(dev); - /* We try to do the minimum that is necessary in order to unlock * the registers for mode setting. * @@ -427,9 +431,6 @@ static void intel_lvds_commit(struct drm_encoder *encoder) struct drm_i915_private *dev_priv = dev->dev_private; struct intel_lvds *intel_lvds = to_intel_lvds(encoder); - if (dev_priv->backlight_level == 0) - dev_priv->backlight_level = intel_panel_get_max_backlight(dev); - /* Undo any unlocking done in prepare to prevent accidental * adjustment of the registers. */ @@ -703,6 +704,14 @@ static const struct dmi_system_id intel_no_lvds[] = { }, { .callback = intel_no_lvds_dmi_callback, + .ident = "AOpen i915GMm-HFS", + .matches = { + DMI_MATCH(DMI_BOARD_VENDOR, "AOpen"), + DMI_MATCH(DMI_BOARD_NAME, "i915GMm-HFS"), + }, + }, + { + .callback = intel_no_lvds_dmi_callback, .ident = "Aopen i945GTt-VFA", .matches = { DMI_MATCH(DMI_PRODUCT_VERSION, "AO00001JW"), @@ -914,6 +923,8 @@ bool intel_lvds_init(struct drm_device *dev) intel_encoder->clone_mask = (1 << INTEL_LVDS_CLONE_BIT); intel_encoder->crtc_mask = (1 << 1); + if (INTEL_INFO(dev)->gen >= 5) + intel_encoder->crtc_mask |= (1 << 0); drm_encoder_helper_add(encoder, &intel_lvds_helper_funcs); drm_connector_helper_add(connector, &intel_lvds_connector_helper_funcs); connector->display_info.subpixel_order = SubPixelHorizontalRGB; @@ -1019,10 +1030,18 @@ bool intel_lvds_init(struct drm_device *dev) out: if (HAS_PCH_SPLIT(dev)) { u32 pwm; - /* make sure PWM is enabled */ + + pipe = (I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT) ? 1 : 0; + + /* make sure PWM is enabled and locked to the LVDS pipe */ pwm = I915_READ(BLC_PWM_CPU_CTL2); - pwm |= (PWM_ENABLE | PWM_PIPE_B); - I915_WRITE(BLC_PWM_CPU_CTL2, pwm); + if (pipe == 0 && (pwm & PWM_PIPE_B)) + I915_WRITE(BLC_PWM_CPU_CTL2, pwm & ~PWM_ENABLE); + if (pipe) + pwm |= PWM_PIPE_B; + else + pwm &= ~PWM_PIPE_B; + I915_WRITE(BLC_PWM_CPU_CTL2, pwm | PWM_ENABLE); pwm = I915_READ(BLC_PWM_PCH_CTL1); pwm |= PWM_PCH_ENABLE; diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c index 9b0d9a867aea..64fd64443ca6 100644 --- a/drivers/gpu/drm/i915/intel_opregion.c +++ b/drivers/gpu/drm/i915/intel_opregion.c @@ -26,6 +26,7 @@ */ #include <linux/acpi.h> +#include <linux/acpi_io.h> #include <acpi/video.h> #include "drmP.h" @@ -273,14 +274,8 @@ void intel_opregion_enable_asle(struct drm_device *dev) struct opregion_asle *asle = dev_priv->opregion.asle; if (asle) { - if (IS_MOBILE(dev)) { - unsigned long irqflags; - - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); + if (IS_MOBILE(dev)) intel_enable_asle(dev); - spin_unlock_irqrestore(&dev_priv->user_irq_lock, - irqflags); - } asle->tche = ASLE_ALS_EN | ASLE_BLC_EN | ASLE_PFIT_EN | ASLE_PFMB_EN; @@ -482,7 +477,7 @@ int intel_opregion_setup(struct drm_device *dev) return -ENOTSUPP; } - base = ioremap(asls, OPREGION_SIZE); + base = acpi_os_ioremap(asls, OPREGION_SIZE); if (!base) return -ENOMEM; diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c index 02ff0a481f47..3fbb98b948d6 100644 --- a/drivers/gpu/drm/i915/intel_overlay.c +++ b/drivers/gpu/drm/i915/intel_overlay.c @@ -221,15 +221,16 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay, int ret; BUG_ON(overlay->last_flip_req); - overlay->last_flip_req = - i915_add_request(dev, NULL, request, &dev_priv->render_ring); - if (overlay->last_flip_req == 0) - return -ENOMEM; - + ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv)); + if (ret) { + kfree(request); + return ret; + } + overlay->last_flip_req = request->seqno; overlay->flip_tail = tail; ret = i915_do_wait_request(dev, overlay->last_flip_req, true, - &dev_priv->render_ring); + LP_RING(dev_priv)); if (ret) return ret; @@ -289,6 +290,7 @@ i830_deactivate_pipe_a(struct drm_device *dev) static int intel_overlay_on(struct intel_overlay *overlay) { struct drm_device *dev = overlay->dev; + struct drm_i915_private *dev_priv = dev->dev_private; struct drm_i915_gem_request *request; int pipe_a_quirk = 0; int ret; @@ -308,7 +310,12 @@ static int intel_overlay_on(struct intel_overlay *overlay) goto out; } - BEGIN_LP_RING(4); + ret = BEGIN_LP_RING(4); + if (ret) { + kfree(request); + goto out; + } + OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON); OUT_RING(overlay->flip_addr | OFC_UPDATE); OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); @@ -332,6 +339,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay, struct drm_i915_gem_request *request; u32 flip_addr = overlay->flip_addr; u32 tmp; + int ret; BUG_ON(!overlay->active); @@ -347,36 +355,44 @@ static int intel_overlay_continue(struct intel_overlay *overlay, if (tmp & (1 << 17)) DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp); - BEGIN_LP_RING(2); + ret = BEGIN_LP_RING(2); + if (ret) { + kfree(request); + return ret; + } OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); OUT_RING(flip_addr); ADVANCE_LP_RING(); - overlay->last_flip_req = - i915_add_request(dev, NULL, request, &dev_priv->render_ring); + ret = i915_add_request(dev, NULL, request, LP_RING(dev_priv)); + if (ret) { + kfree(request); + return ret; + } + + overlay->last_flip_req = request->seqno; return 0; } static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay) { - struct drm_gem_object *obj = &overlay->old_vid_bo->base; + struct drm_i915_gem_object *obj = overlay->old_vid_bo; i915_gem_object_unpin(obj); - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); overlay->old_vid_bo = NULL; } static void intel_overlay_off_tail(struct intel_overlay *overlay) { - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj = overlay->vid_bo; /* never have the overlay hw on without showing a frame */ BUG_ON(!overlay->vid_bo); - obj = &overlay->vid_bo->base; i915_gem_object_unpin(obj); - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); overlay->vid_bo = NULL; overlay->crtc->overlay = NULL; @@ -389,8 +405,10 @@ static int intel_overlay_off(struct intel_overlay *overlay, bool interruptible) { struct drm_device *dev = overlay->dev; + struct drm_i915_private *dev_priv = dev->dev_private; u32 flip_addr = overlay->flip_addr; struct drm_i915_gem_request *request; + int ret; BUG_ON(!overlay->active); @@ -404,7 +422,11 @@ static int intel_overlay_off(struct intel_overlay *overlay, * of the hw. Do it in both cases */ flip_addr |= OFC_UPDATE; - BEGIN_LP_RING(6); + ret = BEGIN_LP_RING(6); + if (ret) { + kfree(request); + return ret; + } /* wait for overlay to go idle */ OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE); OUT_RING(flip_addr); @@ -432,7 +454,7 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay, return 0; ret = i915_do_wait_request(dev, overlay->last_flip_req, - interruptible, &dev_priv->render_ring); + interruptible, LP_RING(dev_priv)); if (ret) return ret; @@ -467,7 +489,12 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay) if (request == NULL) return -ENOMEM; - BEGIN_LP_RING(2); + ret = BEGIN_LP_RING(2); + if (ret) { + kfree(request); + return ret; + } + OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP); OUT_RING(MI_NOOP); ADVANCE_LP_RING(); @@ -736,13 +763,12 @@ static u32 overlay_cmd_reg(struct put_image_params *params) } static int intel_overlay_do_put_image(struct intel_overlay *overlay, - struct drm_gem_object *new_bo, + struct drm_i915_gem_object *new_bo, struct put_image_params *params) { int ret, tmp_width; struct overlay_registers *regs; bool scale_changed = false; - struct drm_i915_gem_object *bo_priv = to_intel_bo(new_bo); struct drm_device *dev = overlay->dev; BUG_ON(!mutex_is_locked(&dev->struct_mutex)); @@ -753,7 +779,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, if (ret != 0) return ret; - ret = i915_gem_object_pin(new_bo, PAGE_SIZE); + ret = i915_gem_object_pin(new_bo, PAGE_SIZE, true); if (ret != 0) return ret; @@ -761,6 +787,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, if (ret != 0) goto out_unpin; + ret = i915_gem_object_put_fence(new_bo); + if (ret) + goto out_unpin; + if (!overlay->active) { regs = intel_overlay_map_regs(overlay); if (!regs) { @@ -797,7 +827,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, regs->SWIDTHSW = calc_swidthsw(overlay->dev, params->offset_Y, tmp_width); regs->SHEIGHT = params->src_h; - regs->OBUF_0Y = bo_priv->gtt_offset + params-> offset_Y; + regs->OBUF_0Y = new_bo->gtt_offset + params-> offset_Y; regs->OSTRIDE = params->stride_Y; if (params->format & I915_OVERLAY_YUV_PLANAR) { @@ -811,8 +841,8 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, params->src_w/uv_hscale); regs->SWIDTHSW |= max_t(u32, tmp_U, tmp_V) << 16; regs->SHEIGHT |= (params->src_h/uv_vscale) << 16; - regs->OBUF_0U = bo_priv->gtt_offset + params->offset_U; - regs->OBUF_0V = bo_priv->gtt_offset + params->offset_V; + regs->OBUF_0U = new_bo->gtt_offset + params->offset_U; + regs->OBUF_0V = new_bo->gtt_offset + params->offset_V; regs->OSTRIDE |= params->stride_UV << 16; } @@ -829,7 +859,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, goto out_unpin; overlay->old_vid_bo = overlay->vid_bo; - overlay->vid_bo = to_intel_bo(new_bo); + overlay->vid_bo = new_bo; return 0; @@ -942,7 +972,7 @@ static int check_overlay_scaling(struct put_image_params *rec) static int check_overlay_src(struct drm_device *dev, struct drm_intel_overlay_put_image *rec, - struct drm_gem_object *new_bo) + struct drm_i915_gem_object *new_bo) { int uv_hscale = uv_hsubsampling(rec->flags); int uv_vscale = uv_vsubsampling(rec->flags); @@ -1027,7 +1057,7 @@ static int check_overlay_src(struct drm_device *dev, return -EINVAL; tmp = rec->stride_Y*rec->src_height; - if (rec->offset_Y + tmp > new_bo->size) + if (rec->offset_Y + tmp > new_bo->base.size) return -EINVAL; break; @@ -1038,12 +1068,12 @@ static int check_overlay_src(struct drm_device *dev, return -EINVAL; tmp = rec->stride_Y * rec->src_height; - if (rec->offset_Y + tmp > new_bo->size) + if (rec->offset_Y + tmp > new_bo->base.size) return -EINVAL; tmp = rec->stride_UV * (rec->src_height / uv_vscale); - if (rec->offset_U + tmp > new_bo->size || - rec->offset_V + tmp > new_bo->size) + if (rec->offset_U + tmp > new_bo->base.size || + rec->offset_V + tmp > new_bo->base.size) return -EINVAL; break; } @@ -1086,7 +1116,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, struct intel_overlay *overlay; struct drm_mode_object *drmmode_obj; struct intel_crtc *crtc; - struct drm_gem_object *new_bo; + struct drm_i915_gem_object *new_bo; struct put_image_params *params; int ret; @@ -1125,8 +1155,8 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, } crtc = to_intel_crtc(obj_to_crtc(drmmode_obj)); - new_bo = drm_gem_object_lookup(dev, file_priv, - put_image_rec->bo_handle); + new_bo = to_intel_bo(drm_gem_object_lookup(dev, file_priv, + put_image_rec->bo_handle)); if (!new_bo) { ret = -ENOENT; goto out_free; @@ -1135,6 +1165,12 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, mutex_lock(&dev->mode_config.mutex); mutex_lock(&dev->struct_mutex); + if (new_bo->tiling_mode) { + DRM_ERROR("buffer used for overlay image can not be tiled\n"); + ret = -EINVAL; + goto out_unlock; + } + ret = intel_overlay_recover_from_interrupt(overlay, true); if (ret != 0) goto out_unlock; @@ -1217,7 +1253,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data, out_unlock: mutex_unlock(&dev->struct_mutex); mutex_unlock(&dev->mode_config.mutex); - drm_gem_object_unreference_unlocked(new_bo); + drm_gem_object_unreference_unlocked(&new_bo->base); out_free: kfree(params); @@ -1370,7 +1406,7 @@ void intel_setup_overlay(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; struct intel_overlay *overlay; - struct drm_gem_object *reg_bo; + struct drm_i915_gem_object *reg_bo; struct overlay_registers *regs; int ret; @@ -1385,7 +1421,7 @@ void intel_setup_overlay(struct drm_device *dev) reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE); if (!reg_bo) goto out_free; - overlay->reg_bo = to_intel_bo(reg_bo); + overlay->reg_bo = reg_bo; if (OVERLAY_NEEDS_PHYSICAL(dev)) { ret = i915_gem_attach_phys_object(dev, reg_bo, @@ -1395,14 +1431,14 @@ void intel_setup_overlay(struct drm_device *dev) DRM_ERROR("failed to attach phys overlay regs\n"); goto out_free_bo; } - overlay->flip_addr = overlay->reg_bo->phys_obj->handle->busaddr; + overlay->flip_addr = reg_bo->phys_obj->handle->busaddr; } else { - ret = i915_gem_object_pin(reg_bo, PAGE_SIZE); + ret = i915_gem_object_pin(reg_bo, PAGE_SIZE, true); if (ret) { DRM_ERROR("failed to pin overlay register bo\n"); goto out_free_bo; } - overlay->flip_addr = overlay->reg_bo->gtt_offset; + overlay->flip_addr = reg_bo->gtt_offset; ret = i915_gem_object_set_to_gtt_domain(reg_bo, true); if (ret) { @@ -1434,7 +1470,7 @@ void intel_setup_overlay(struct drm_device *dev) out_unpin_bo: i915_gem_object_unpin(reg_bo); out_free_bo: - drm_gem_object_unreference(reg_bo); + drm_gem_object_unreference(®_bo->base); out_free: kfree(overlay); return; diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c index 92ff8f385278..d860abeda70f 100644 --- a/drivers/gpu/drm/i915/intel_panel.c +++ b/drivers/gpu/drm/i915/intel_panel.c @@ -30,8 +30,6 @@ #include "intel_drv.h" -#define PCI_LBPC 0xf4 /* legacy/combination backlight modes */ - void intel_fixed_panel_mode(struct drm_display_mode *fixed_mode, struct drm_display_mode *adjusted_mode) @@ -112,17 +110,36 @@ done: dev_priv->pch_pf_size = (width << 16) | height; } -static int is_backlight_combination_mode(struct drm_device *dev) +static u32 i915_read_blc_pwm_ctl(struct drm_i915_private *dev_priv) { - struct drm_i915_private *dev_priv = dev->dev_private; + u32 val; - if (INTEL_INFO(dev)->gen >= 4) - return I915_READ(BLC_PWM_CTL2) & BLM_COMBINATION_MODE; + /* Restore the CTL value if it lost, e.g. GPU reset */ - if (IS_GEN2(dev)) - return I915_READ(BLC_PWM_CTL) & BLM_LEGACY_MODE; + if (HAS_PCH_SPLIT(dev_priv->dev)) { + val = I915_READ(BLC_PWM_PCH_CTL2); + if (dev_priv->saveBLC_PWM_CTL2 == 0) { + dev_priv->saveBLC_PWM_CTL2 = val; + } else if (val == 0) { + I915_WRITE(BLC_PWM_PCH_CTL2, + dev_priv->saveBLC_PWM_CTL); + val = dev_priv->saveBLC_PWM_CTL; + } + } else { + val = I915_READ(BLC_PWM_CTL); + if (dev_priv->saveBLC_PWM_CTL == 0) { + dev_priv->saveBLC_PWM_CTL = val; + dev_priv->saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2); + } else if (val == 0) { + I915_WRITE(BLC_PWM_CTL, + dev_priv->saveBLC_PWM_CTL); + I915_WRITE(BLC_PWM_CTL2, + dev_priv->saveBLC_PWM_CTL2); + val = dev_priv->saveBLC_PWM_CTL; + } + } - return 0; + return val; } u32 intel_panel_get_max_backlight(struct drm_device *dev) @@ -130,10 +147,18 @@ u32 intel_panel_get_max_backlight(struct drm_device *dev) struct drm_i915_private *dev_priv = dev->dev_private; u32 max; + max = i915_read_blc_pwm_ctl(dev_priv); + if (max == 0) { + /* XXX add code here to query mode clock or hardware clock + * and program max PWM appropriately. + */ + printk_once(KERN_WARNING "fixme: max PWM is zero.\n"); + return 1; + } + if (HAS_PCH_SPLIT(dev)) { - max = I915_READ(BLC_PWM_PCH_CTL2) >> 16; + max >>= 16; } else { - max = I915_READ(BLC_PWM_CTL); if (IS_PINEVIEW(dev)) { max >>= 17; } else { @@ -141,17 +166,6 @@ u32 intel_panel_get_max_backlight(struct drm_device *dev) if (INTEL_INFO(dev)->gen < 4) max &= ~1; } - - if (is_backlight_combination_mode(dev)) - max *= 0xff; - } - - if (max == 0) { - /* XXX add code here to query mode clock or hardware clock - * and program max PWM appropriately. - */ - DRM_ERROR("fixme: max PWM is zero.\n"); - max = 1; } DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max); @@ -169,15 +183,6 @@ u32 intel_panel_get_backlight(struct drm_device *dev) val = I915_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK; if (IS_PINEVIEW(dev)) val >>= 1; - - if (is_backlight_combination_mode(dev)){ - u8 lbpc; - - val &= ~1; - pci_read_config_byte(dev->pdev, PCI_LBPC, &lbpc); - val *= lbpc; - val >>= 1; - } } DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val); @@ -200,16 +205,6 @@ void intel_panel_set_backlight(struct drm_device *dev, u32 level) if (HAS_PCH_SPLIT(dev)) return intel_pch_panel_set_backlight(dev, level); - - if (is_backlight_combination_mode(dev)){ - u32 max = intel_panel_get_max_backlight(dev); - u8 lpbc; - - lpbc = level * 0xfe / max + 1; - level /= lpbc; - pci_write_config_byte(dev->pdev, PCI_LBPC, lpbc); - } - tmp = I915_READ(BLC_PWM_CTL); if (IS_PINEVIEW(dev)) { tmp &= ~(BACKLIGHT_DUTY_CYCLE_MASK - 1); @@ -218,3 +213,34 @@ void intel_panel_set_backlight(struct drm_device *dev, u32 level) tmp &= ~BACKLIGHT_DUTY_CYCLE_MASK; I915_WRITE(BLC_PWM_CTL, tmp | level); } + +void intel_panel_disable_backlight(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (dev_priv->backlight_enabled) { + dev_priv->backlight_level = intel_panel_get_backlight(dev); + dev_priv->backlight_enabled = false; + } + + intel_panel_set_backlight(dev, 0); +} + +void intel_panel_enable_backlight(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + if (dev_priv->backlight_level == 0) + dev_priv->backlight_level = intel_panel_get_max_backlight(dev); + + intel_panel_set_backlight(dev, dev_priv->backlight_level); + dev_priv->backlight_enabled = true; +} + +void intel_panel_setup_backlight(struct drm_device *dev) +{ + struct drm_i915_private *dev_priv = dev->dev_private; + + dev_priv->backlight_level = intel_panel_get_backlight(dev); + dev_priv->backlight_enabled = dev_priv->backlight_level != 0; +} diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 31cd7e33e820..445f27efe677 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -34,6 +34,14 @@ #include "i915_trace.h" #include "intel_drv.h" +static inline int ring_space(struct intel_ring_buffer *ring) +{ + int space = (ring->head & HEAD_ADDR) - (ring->tail + 8); + if (space < 0) + space += ring->size; + return space; +} + static u32 i915_gem_get_seqno(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; @@ -48,14 +56,15 @@ static u32 i915_gem_get_seqno(struct drm_device *dev) return seqno; } -static void -render_ring_flush(struct drm_device *dev, - struct intel_ring_buffer *ring, +static int +render_ring_flush(struct intel_ring_buffer *ring, u32 invalidate_domains, u32 flush_domains) { + struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; u32 cmd; + int ret; #if WATCH_EXEC DRM_INFO("%s: invalidate %08x flush %08x\n", __func__, @@ -109,49 +118,54 @@ render_ring_flush(struct drm_device *dev, if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION) cmd |= MI_EXE_FLUSH; + if (invalidate_domains & I915_GEM_DOMAIN_COMMAND && + (IS_G4X(dev) || IS_GEN5(dev))) + cmd |= MI_INVALIDATE_ISP; + #if WATCH_EXEC DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd); #endif - intel_ring_begin(dev, ring, 2); - intel_ring_emit(dev, ring, cmd); - intel_ring_emit(dev, ring, MI_NOOP); - intel_ring_advance(dev, ring); + ret = intel_ring_begin(ring, 2); + if (ret) + return ret; + + intel_ring_emit(ring, cmd); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); } + + return 0; } -static void ring_write_tail(struct drm_device *dev, - struct intel_ring_buffer *ring, +static void ring_write_tail(struct intel_ring_buffer *ring, u32 value) { - drm_i915_private_t *dev_priv = dev->dev_private; + drm_i915_private_t *dev_priv = ring->dev->dev_private; I915_WRITE_TAIL(ring, value); } -u32 intel_ring_get_active_head(struct drm_device *dev, - struct intel_ring_buffer *ring) +u32 intel_ring_get_active_head(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; - u32 acthd_reg = INTEL_INFO(dev)->gen >= 4 ? + drm_i915_private_t *dev_priv = ring->dev->dev_private; + u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ? RING_ACTHD(ring->mmio_base) : ACTHD; return I915_READ(acthd_reg); } -static int init_ring_common(struct drm_device *dev, - struct intel_ring_buffer *ring) +static int init_ring_common(struct intel_ring_buffer *ring) { + drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_gem_object *obj = ring->obj; u32 head; - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; - obj_priv = to_intel_bo(ring->gem_object); /* Stop the ring if it's running. */ I915_WRITE_CTL(ring, 0); I915_WRITE_HEAD(ring, 0); - ring->write_tail(dev, ring, 0); + ring->write_tail(ring, 0); /* Initialize the ring. */ - I915_WRITE_START(ring, obj_priv->gtt_offset); + I915_WRITE_START(ring, obj->gtt_offset); head = I915_READ_HEAD(ring) & HEAD_ADDR; /* G45 ring initialization fails to reset head to zero */ @@ -178,12 +192,13 @@ static int init_ring_common(struct drm_device *dev, } I915_WRITE_CTL(ring, - ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES) + ((ring->size - PAGE_SIZE) & RING_NR_PAGES) | RING_REPORT_64K | RING_VALID); - head = I915_READ_HEAD(ring) & HEAD_ADDR; /* If the head is still not zero, the ring is dead */ - if (head != 0) { + if ((I915_READ_CTL(ring) & RING_VALID) == 0 || + I915_READ_START(ring) != obj->gtt_offset || + (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) { DRM_ERROR("%s initialization failed " "ctl %08x head %08x tail %08x start %08x\n", ring->name, @@ -194,344 +209,569 @@ static int init_ring_common(struct drm_device *dev, return -EIO; } - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - i915_kernel_lost_context(dev); + if (!drm_core_check_feature(ring->dev, DRIVER_MODESET)) + i915_kernel_lost_context(ring->dev); else { - ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; + ring->head = I915_READ_HEAD(ring); ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; - ring->space = ring->head - (ring->tail + 8); - if (ring->space < 0) - ring->space += ring->size; + ring->space = ring_space(ring); } + return 0; } -static int init_render_ring(struct drm_device *dev, - struct intel_ring_buffer *ring) +/* + * 965+ support PIPE_CONTROL commands, which provide finer grained control + * over cache flushing. + */ +struct pipe_control { + struct drm_i915_gem_object *obj; + volatile u32 *cpu_page; + u32 gtt_offset; +}; + +static int +init_pipe_control(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; - int ret = init_ring_common(dev, ring); - int mode; + struct pipe_control *pc; + struct drm_i915_gem_object *obj; + int ret; + + if (ring->private) + return 0; + + pc = kmalloc(sizeof(*pc), GFP_KERNEL); + if (!pc) + return -ENOMEM; + + obj = i915_gem_alloc_object(ring->dev, 4096); + if (obj == NULL) { + DRM_ERROR("Failed to allocate seqno page\n"); + ret = -ENOMEM; + goto err; + } + obj->agp_type = AGP_USER_CACHED_MEMORY; + + ret = i915_gem_object_pin(obj, 4096, true); + if (ret) + goto err_unref; + + pc->gtt_offset = obj->gtt_offset; + pc->cpu_page = kmap(obj->pages[0]); + if (pc->cpu_page == NULL) + goto err_unpin; + + pc->obj = obj; + ring->private = pc; + return 0; + +err_unpin: + i915_gem_object_unpin(obj); +err_unref: + drm_gem_object_unreference(&obj->base); +err: + kfree(pc); + return ret; +} + +static void +cleanup_pipe_control(struct intel_ring_buffer *ring) +{ + struct pipe_control *pc = ring->private; + struct drm_i915_gem_object *obj; + + if (!ring->private) + return; + + obj = pc->obj; + kunmap(obj->pages[0]); + i915_gem_object_unpin(obj); + drm_gem_object_unreference(&obj->base); + + kfree(pc); + ring->private = NULL; +} + +static int init_render_ring(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int ret = init_ring_common(ring); if (INTEL_INFO(dev)->gen > 3) { - mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; + int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH; if (IS_GEN6(dev)) mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE; I915_WRITE(MI_MODE, mode); } + + if (INTEL_INFO(dev)->gen >= 6) { + } else if (IS_GEN5(dev)) { + ret = init_pipe_control(ring); + if (ret) + return ret; + } + return ret; } -#define PIPE_CONTROL_FLUSH(addr) \ +static void render_ring_cleanup(struct intel_ring_buffer *ring) +{ + if (!ring->private) + return; + + cleanup_pipe_control(ring); +} + +static void +update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno) +{ + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; + int id; + + /* + * cs -> 1 = vcs, 0 = bcs + * vcs -> 1 = bcs, 0 = cs, + * bcs -> 1 = cs, 0 = vcs. + */ + id = ring - dev_priv->ring; + id += 2 - i; + id %= 3; + + intel_ring_emit(ring, + MI_SEMAPHORE_MBOX | + MI_SEMAPHORE_REGISTER | + MI_SEMAPHORE_UPDATE); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, + RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i); +} + +static int +gen6_add_request(struct intel_ring_buffer *ring, + u32 *result) +{ + u32 seqno; + int ret; + + ret = intel_ring_begin(ring, 10); + if (ret) + return ret; + + seqno = i915_gem_get_seqno(ring->dev); + update_semaphore(ring, 0, seqno); + update_semaphore(ring, 1, seqno); + + intel_ring_emit(ring, MI_STORE_DWORD_INDEX); + intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, MI_USER_INTERRUPT); + intel_ring_advance(ring); + + *result = seqno; + return 0; +} + +int +intel_ring_sync(struct intel_ring_buffer *ring, + struct intel_ring_buffer *to, + u32 seqno) +{ + int ret; + + ret = intel_ring_begin(ring, 4); + if (ret) + return ret; + + intel_ring_emit(ring, + MI_SEMAPHORE_MBOX | + MI_SEMAPHORE_REGISTER | + intel_ring_sync_index(ring, to) << 17 | + MI_SEMAPHORE_COMPARE); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, 0); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); + + return 0; +} + +#define PIPE_CONTROL_FLUSH(ring__, addr__) \ do { \ - OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ + intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \ PIPE_CONTROL_DEPTH_STALL | 2); \ - OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \ - OUT_RING(0); \ - OUT_RING(0); \ + intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \ + intel_ring_emit(ring__, 0); \ + intel_ring_emit(ring__, 0); \ } while (0) -/** - * Creates a new sequence number, emitting a write of it to the status page - * plus an interrupt, which will trigger i915_user_interrupt_handler. - * - * Must be called with struct_lock held. - * - * Returned sequence numbers are nonzero on success. - */ -static u32 -render_ring_add_request(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 flush_domains) +static int +pc_render_add_request(struct intel_ring_buffer *ring, + u32 *result) { - drm_i915_private_t *dev_priv = dev->dev_private; - u32 seqno; + struct drm_device *dev = ring->dev; + u32 seqno = i915_gem_get_seqno(dev); + struct pipe_control *pc = ring->private; + u32 scratch_addr = pc->gtt_offset + 128; + int ret; - seqno = i915_gem_get_seqno(dev); - - if (IS_GEN6(dev)) { - BEGIN_LP_RING(6); - OUT_RING(GFX_OP_PIPE_CONTROL | 3); - OUT_RING(PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH | - PIPE_CONTROL_NOTIFY); - OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); - OUT_RING(seqno); - OUT_RING(0); - OUT_RING(0); - ADVANCE_LP_RING(); - } else if (HAS_PIPE_CONTROL(dev)) { - u32 scratch_addr = dev_priv->seqno_gfx_addr + 128; + /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently + * incoherent with writes to memory, i.e. completely fubar, + * so we need to use PIPE_NOTIFY instead. + * + * However, we also need to workaround the qword write + * incoherence by flushing the 6 PIPE_NOTIFY buffers out to + * memory before requesting an interrupt. + */ + ret = intel_ring_begin(ring, 32); + if (ret) + return ret; + + intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH); + intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, 0); + PIPE_CONTROL_FLUSH(ring, scratch_addr); + scratch_addr += 128; /* write to separate cachelines */ + PIPE_CONTROL_FLUSH(ring, scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(ring, scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(ring, scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(ring, scratch_addr); + scratch_addr += 128; + PIPE_CONTROL_FLUSH(ring, scratch_addr); + intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH | + PIPE_CONTROL_NOTIFY); + intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, 0); + intel_ring_advance(ring); + + *result = seqno; + return 0; +} - /* - * Workaround qword write incoherence by flushing the - * PIPE_NOTIFY buffers out to memory before requesting - * an interrupt. - */ - BEGIN_LP_RING(32); - OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH); - OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); - OUT_RING(seqno); - OUT_RING(0); - PIPE_CONTROL_FLUSH(scratch_addr); - scratch_addr += 128; /* write to separate cachelines */ - PIPE_CONTROL_FLUSH(scratch_addr); - scratch_addr += 128; - PIPE_CONTROL_FLUSH(scratch_addr); - scratch_addr += 128; - PIPE_CONTROL_FLUSH(scratch_addr); - scratch_addr += 128; - PIPE_CONTROL_FLUSH(scratch_addr); - scratch_addr += 128; - PIPE_CONTROL_FLUSH(scratch_addr); - OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | - PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH | - PIPE_CONTROL_NOTIFY); - OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT); - OUT_RING(seqno); - OUT_RING(0); - ADVANCE_LP_RING(); - } else { - BEGIN_LP_RING(4); - OUT_RING(MI_STORE_DWORD_INDEX); - OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - OUT_RING(seqno); +static int +render_ring_add_request(struct intel_ring_buffer *ring, + u32 *result) +{ + struct drm_device *dev = ring->dev; + u32 seqno = i915_gem_get_seqno(dev); + int ret; - OUT_RING(MI_USER_INTERRUPT); - ADVANCE_LP_RING(); - } - return seqno; + ret = intel_ring_begin(ring, 4); + if (ret) + return ret; + + intel_ring_emit(ring, MI_STORE_DWORD_INDEX); + intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, MI_USER_INTERRUPT); + intel_ring_advance(ring); + + *result = seqno; + return 0; } static u32 -render_ring_get_seqno(struct drm_device *dev, - struct intel_ring_buffer *ring) +ring_get_seqno(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - if (HAS_PIPE_CONTROL(dev)) - return ((volatile u32 *)(dev_priv->seqno_page))[0]; - else - return intel_read_status_page(ring, I915_GEM_HWS_INDEX); + return intel_read_status_page(ring, I915_GEM_HWS_INDEX); +} + +static u32 +pc_render_get_seqno(struct intel_ring_buffer *ring) +{ + struct pipe_control *pc = ring->private; + return pc->cpu_page[0]; } static void -render_ring_get_user_irq(struct drm_device *dev, - struct intel_ring_buffer *ring) +ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - unsigned long irqflags; + dev_priv->gt_irq_mask &= ~mask; + I915_WRITE(GTIMR, dev_priv->gt_irq_mask); + POSTING_READ(GTIMR); +} - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); - if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) { +static void +ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask) +{ + dev_priv->gt_irq_mask |= mask; + I915_WRITE(GTIMR, dev_priv->gt_irq_mask); + POSTING_READ(GTIMR); +} + +static void +i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask) +{ + dev_priv->irq_mask &= ~mask; + I915_WRITE(IMR, dev_priv->irq_mask); + POSTING_READ(IMR); +} + +static void +i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask) +{ + dev_priv->irq_mask |= mask; + I915_WRITE(IMR, dev_priv->irq_mask); + POSTING_READ(IMR); +} + +static bool +render_ring_get_irq(struct intel_ring_buffer *ring) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + if (!dev->irq_enabled) + return false; + + spin_lock(&ring->irq_lock); + if (ring->irq_refcount++ == 0) { if (HAS_PCH_SPLIT(dev)) - ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); + ironlake_enable_irq(dev_priv, + GT_PIPE_NOTIFY | GT_USER_INTERRUPT); else i915_enable_irq(dev_priv, I915_USER_INTERRUPT); } - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + spin_unlock(&ring->irq_lock); + + return true; } static void -render_ring_put_user_irq(struct drm_device *dev, - struct intel_ring_buffer *ring) +render_ring_put_irq(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private; - unsigned long irqflags; + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); - BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0); - if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) { + spin_lock(&ring->irq_lock); + if (--ring->irq_refcount == 0) { if (HAS_PCH_SPLIT(dev)) - ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY); + ironlake_disable_irq(dev_priv, + GT_USER_INTERRUPT | + GT_PIPE_NOTIFY); else i915_disable_irq(dev_priv, I915_USER_INTERRUPT); } - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + spin_unlock(&ring->irq_lock); } -void intel_ring_setup_status_page(struct drm_device *dev, - struct intel_ring_buffer *ring) +void intel_ring_setup_status_page(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; - if (IS_GEN6(dev)) { - I915_WRITE(RING_HWS_PGA_GEN6(ring->mmio_base), - ring->status_page.gfx_addr); - I915_READ(RING_HWS_PGA_GEN6(ring->mmio_base)); /* posting read */ - } else { - I915_WRITE(RING_HWS_PGA(ring->mmio_base), - ring->status_page.gfx_addr); - I915_READ(RING_HWS_PGA(ring->mmio_base)); /* posting read */ - } - + drm_i915_private_t *dev_priv = ring->dev->dev_private; + u32 mmio = IS_GEN6(ring->dev) ? + RING_HWS_PGA_GEN6(ring->mmio_base) : + RING_HWS_PGA(ring->mmio_base); + I915_WRITE(mmio, (u32)ring->status_page.gfx_addr); + POSTING_READ(mmio); } -static void -bsd_ring_flush(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 invalidate_domains, - u32 flush_domains) +static int +bsd_ring_flush(struct intel_ring_buffer *ring, + u32 invalidate_domains, + u32 flush_domains) { - intel_ring_begin(dev, ring, 2); - intel_ring_emit(dev, ring, MI_FLUSH); - intel_ring_emit(dev, ring, MI_NOOP); - intel_ring_advance(dev, ring); -} + int ret; -static int init_bsd_ring(struct drm_device *dev, - struct intel_ring_buffer *ring) -{ - return init_ring_common(dev, ring); + if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0) + return 0; + + ret = intel_ring_begin(ring, 2); + if (ret) + return ret; + + intel_ring_emit(ring, MI_FLUSH); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); + return 0; } -static u32 -ring_add_request(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 flush_domains) +static int +ring_add_request(struct intel_ring_buffer *ring, + u32 *result) { u32 seqno; + int ret; - seqno = i915_gem_get_seqno(dev); + ret = intel_ring_begin(ring, 4); + if (ret) + return ret; - intel_ring_begin(dev, ring, 4); - intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX); - intel_ring_emit(dev, ring, - I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(dev, ring, seqno); - intel_ring_emit(dev, ring, MI_USER_INTERRUPT); - intel_ring_advance(dev, ring); + seqno = i915_gem_get_seqno(ring->dev); + + intel_ring_emit(ring, MI_STORE_DWORD_INDEX); + intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); + intel_ring_emit(ring, seqno); + intel_ring_emit(ring, MI_USER_INTERRUPT); + intel_ring_advance(ring); DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); + *result = seqno; + return 0; +} - return seqno; +static bool +ring_get_irq(struct intel_ring_buffer *ring, u32 flag) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + if (!dev->irq_enabled) + return false; + + spin_lock(&ring->irq_lock); + if (ring->irq_refcount++ == 0) + ironlake_enable_irq(dev_priv, flag); + spin_unlock(&ring->irq_lock); + + return true; } static void -bsd_ring_get_user_irq(struct drm_device *dev, - struct intel_ring_buffer *ring) +ring_put_irq(struct intel_ring_buffer *ring, u32 flag) +{ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + spin_lock(&ring->irq_lock); + if (--ring->irq_refcount == 0) + ironlake_disable_irq(dev_priv, flag); + spin_unlock(&ring->irq_lock); +} + +static bool +gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag) { - /* do nothing */ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + if (!dev->irq_enabled) + return false; + + spin_lock(&ring->irq_lock); + if (ring->irq_refcount++ == 0) { + ring->irq_mask &= ~rflag; + I915_WRITE_IMR(ring, ring->irq_mask); + ironlake_enable_irq(dev_priv, gflag); + } + spin_unlock(&ring->irq_lock); + + return true; } + static void -bsd_ring_put_user_irq(struct drm_device *dev, - struct intel_ring_buffer *ring) +gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag) { - /* do nothing */ + struct drm_device *dev = ring->dev; + drm_i915_private_t *dev_priv = dev->dev_private; + + spin_lock(&ring->irq_lock); + if (--ring->irq_refcount == 0) { + ring->irq_mask |= rflag; + I915_WRITE_IMR(ring, ring->irq_mask); + ironlake_disable_irq(dev_priv, gflag); + } + spin_unlock(&ring->irq_lock); } -static u32 -ring_status_page_get_seqno(struct drm_device *dev, - struct intel_ring_buffer *ring) +static bool +bsd_ring_get_irq(struct intel_ring_buffer *ring) { - return intel_read_status_page(ring, I915_GEM_HWS_INDEX); + return ring_get_irq(ring, GT_BSD_USER_INTERRUPT); +} +static void +bsd_ring_put_irq(struct intel_ring_buffer *ring) +{ + ring_put_irq(ring, GT_BSD_USER_INTERRUPT); } static int -ring_dispatch_gem_execbuffer(struct drm_device *dev, - struct intel_ring_buffer *ring, - struct drm_i915_gem_execbuffer2 *exec, - struct drm_clip_rect *cliprects, - uint64_t exec_offset) +ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length) { - uint32_t exec_start; - exec_start = (uint32_t) exec_offset + exec->batch_start_offset; - intel_ring_begin(dev, ring, 2); - intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START | - (2 << 6) | MI_BATCH_NON_SECURE_I965); - intel_ring_emit(dev, ring, exec_start); - intel_ring_advance(dev, ring); + int ret; + + ret = intel_ring_begin(ring, 2); + if (ret) + return ret; + + intel_ring_emit(ring, + MI_BATCH_BUFFER_START | (2 << 6) | + MI_BATCH_NON_SECURE_I965); + intel_ring_emit(ring, offset); + intel_ring_advance(ring); + return 0; } static int -render_ring_dispatch_gem_execbuffer(struct drm_device *dev, - struct intel_ring_buffer *ring, - struct drm_i915_gem_execbuffer2 *exec, - struct drm_clip_rect *cliprects, - uint64_t exec_offset) +render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, + u32 offset, u32 len) { + struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; - int nbox = exec->num_cliprects; - int i = 0, count; - uint32_t exec_start, exec_len; - exec_start = (uint32_t) exec_offset + exec->batch_start_offset; - exec_len = (uint32_t) exec->batch_len; + int ret; trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1); - count = nbox ? nbox : 1; + if (IS_I830(dev) || IS_845G(dev)) { + ret = intel_ring_begin(ring, 4); + if (ret) + return ret; - for (i = 0; i < count; i++) { - if (i < nbox) { - int ret = i915_emit_box(dev, cliprects, i, - exec->DR1, exec->DR4); - if (ret) - return ret; - } + intel_ring_emit(ring, MI_BATCH_BUFFER); + intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE); + intel_ring_emit(ring, offset + len - 8); + intel_ring_emit(ring, 0); + } else { + ret = intel_ring_begin(ring, 2); + if (ret) + return ret; - if (IS_I830(dev) || IS_845G(dev)) { - intel_ring_begin(dev, ring, 4); - intel_ring_emit(dev, ring, MI_BATCH_BUFFER); - intel_ring_emit(dev, ring, - exec_start | MI_BATCH_NON_SECURE); - intel_ring_emit(dev, ring, exec_start + exec_len - 4); - intel_ring_emit(dev, ring, 0); + if (INTEL_INFO(dev)->gen >= 4) { + intel_ring_emit(ring, + MI_BATCH_BUFFER_START | (2 << 6) | + MI_BATCH_NON_SECURE_I965); + intel_ring_emit(ring, offset); } else { - intel_ring_begin(dev, ring, 2); - if (INTEL_INFO(dev)->gen >= 4) { - intel_ring_emit(dev, ring, - MI_BATCH_BUFFER_START | (2 << 6) - | MI_BATCH_NON_SECURE_I965); - intel_ring_emit(dev, ring, exec_start); - } else { - intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START - | (2 << 6)); - intel_ring_emit(dev, ring, exec_start | - MI_BATCH_NON_SECURE); - } + intel_ring_emit(ring, + MI_BATCH_BUFFER_START | (2 << 6)); + intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE); } - intel_ring_advance(dev, ring); } - - if (IS_G4X(dev) || IS_GEN5(dev)) { - intel_ring_begin(dev, ring, 2); - intel_ring_emit(dev, ring, MI_FLUSH | - MI_NO_WRITE_FLUSH | - MI_INVALIDATE_ISP ); - intel_ring_emit(dev, ring, MI_NOOP); - intel_ring_advance(dev, ring); - } - /* XXX breadcrumb */ + intel_ring_advance(ring); return 0; } -static void cleanup_status_page(struct drm_device *dev, - struct intel_ring_buffer *ring) +static void cleanup_status_page(struct intel_ring_buffer *ring) { - drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + drm_i915_private_t *dev_priv = ring->dev->dev_private; + struct drm_i915_gem_object *obj; obj = ring->status_page.obj; if (obj == NULL) return; - obj_priv = to_intel_bo(obj); - kunmap(obj_priv->pages[0]); + kunmap(obj->pages[0]); i915_gem_object_unpin(obj); - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); ring->status_page.obj = NULL; memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); } -static int init_status_page(struct drm_device *dev, - struct intel_ring_buffer *ring) +static int init_status_page(struct intel_ring_buffer *ring) { + struct drm_device *dev = ring->dev; drm_i915_private_t *dev_priv = dev->dev_private; - struct drm_gem_object *obj; - struct drm_i915_gem_object *obj_priv; + struct drm_i915_gem_object *obj; int ret; obj = i915_gem_alloc_object(dev, 4096); @@ -540,16 +780,15 @@ static int init_status_page(struct drm_device *dev, ret = -ENOMEM; goto err; } - obj_priv = to_intel_bo(obj); - obj_priv->agp_type = AGP_USER_CACHED_MEMORY; + obj->agp_type = AGP_USER_CACHED_MEMORY; - ret = i915_gem_object_pin(obj, 4096); + ret = i915_gem_object_pin(obj, 4096, true); if (ret != 0) { goto err_unref; } - ring->status_page.gfx_addr = obj_priv->gtt_offset; - ring->status_page.page_addr = kmap(obj_priv->pages[0]); + ring->status_page.gfx_addr = obj->gtt_offset; + ring->status_page.page_addr = kmap(obj->pages[0]); if (ring->status_page.page_addr == NULL) { memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map)); goto err_unpin; @@ -557,7 +796,7 @@ static int init_status_page(struct drm_device *dev, ring->status_page.obj = obj; memset(ring->status_page.page_addr, 0, PAGE_SIZE); - intel_ring_setup_status_page(dev, ring); + intel_ring_setup_status_page(ring); DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n", ring->name, ring->status_page.gfx_addr); @@ -566,7 +805,7 @@ static int init_status_page(struct drm_device *dev, err_unpin: i915_gem_object_unpin(obj); err_unref: - drm_gem_object_unreference(obj); + drm_gem_object_unreference(&obj->base); err: return ret; } @@ -574,9 +813,7 @@ err: int intel_init_ring_buffer(struct drm_device *dev, struct intel_ring_buffer *ring) { - struct drm_i915_private *dev_priv = dev->dev_private; - struct drm_i915_gem_object *obj_priv; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; int ret; ring->dev = dev; @@ -584,8 +821,11 @@ int intel_init_ring_buffer(struct drm_device *dev, INIT_LIST_HEAD(&ring->request_list); INIT_LIST_HEAD(&ring->gpu_write_list); + spin_lock_init(&ring->irq_lock); + ring->irq_mask = ~0; + if (I915_NEED_GFX_HWS(dev)) { - ret = init_status_page(dev, ring); + ret = init_status_page(ring); if (ret) return ret; } @@ -597,15 +837,14 @@ int intel_init_ring_buffer(struct drm_device *dev, goto err_hws; } - ring->gem_object = obj; + ring->obj = obj; - ret = i915_gem_object_pin(obj, PAGE_SIZE); + ret = i915_gem_object_pin(obj, PAGE_SIZE, true); if (ret) goto err_unref; - obj_priv = to_intel_bo(obj); ring->map.size = ring->size; - ring->map.offset = dev->agp->base + obj_priv->gtt_offset; + ring->map.offset = dev->agp->base + obj->gtt_offset; ring->map.type = 0; ring->map.flags = 0; ring->map.mtrr = 0; @@ -618,60 +857,64 @@ int intel_init_ring_buffer(struct drm_device *dev, } ring->virtual_start = ring->map.handle; - ret = ring->init(dev, ring); + ret = ring->init(ring); if (ret) goto err_unmap; - if (!drm_core_check_feature(dev, DRIVER_MODESET)) - i915_kernel_lost_context(dev); - else { - ring->head = I915_READ_HEAD(ring) & HEAD_ADDR; - ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR; - ring->space = ring->head - (ring->tail + 8); - if (ring->space < 0) - ring->space += ring->size; - } - return ret; + /* Workaround an erratum on the i830 which causes a hang if + * the TAIL pointer points to within the last 2 cachelines + * of the buffer. + */ + ring->effective_size = ring->size; + if (IS_I830(ring->dev)) + ring->effective_size -= 128; + + return 0; err_unmap: drm_core_ioremapfree(&ring->map, dev); err_unpin: i915_gem_object_unpin(obj); err_unref: - drm_gem_object_unreference(obj); - ring->gem_object = NULL; + drm_gem_object_unreference(&obj->base); + ring->obj = NULL; err_hws: - cleanup_status_page(dev, ring); + cleanup_status_page(ring); return ret; } -void intel_cleanup_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring) +void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring) { - if (ring->gem_object == NULL) + struct drm_i915_private *dev_priv; + int ret; + + if (ring->obj == NULL) return; - drm_core_ioremapfree(&ring->map, dev); + /* Disable the ring buffer. The ring must be idle at this point */ + dev_priv = ring->dev->dev_private; + ret = intel_wait_ring_buffer(ring, ring->size - 8); + I915_WRITE_CTL(ring, 0); + + drm_core_ioremapfree(&ring->map, ring->dev); - i915_gem_object_unpin(ring->gem_object); - drm_gem_object_unreference(ring->gem_object); - ring->gem_object = NULL; + i915_gem_object_unpin(ring->obj); + drm_gem_object_unreference(&ring->obj->base); + ring->obj = NULL; if (ring->cleanup) ring->cleanup(ring); - cleanup_status_page(dev, ring); + cleanup_status_page(ring); } -static int intel_wrap_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring) +static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring) { unsigned int *virt; - int rem; - rem = ring->size - ring->tail; + int rem = ring->size - ring->tail; if (ring->space < rem) { - int ret = intel_wait_ring_buffer(dev, ring, rem); + int ret = intel_wait_ring_buffer(ring, rem); if (ret) return ret; } @@ -684,34 +927,36 @@ static int intel_wrap_ring_buffer(struct drm_device *dev, } ring->tail = 0; - ring->space = ring->head - 8; + ring->space = ring_space(ring); return 0; } -int intel_wait_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring, int n) +int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n) { + struct drm_device *dev = ring->dev; + struct drm_i915_private *dev_priv = dev->dev_private; unsigned long end; - drm_i915_private_t *dev_priv = dev->dev_private; u32 head; + /* If the reported head position has wrapped or hasn't advanced, + * fallback to the slow and accurate path. + */ + head = intel_read_status_page(ring, 4); + if (head > ring->head) { + ring->head = head; + ring->space = ring_space(ring); + if (ring->space >= n) + return 0; + } + trace_i915_ring_wait_begin (dev); end = jiffies + 3 * HZ; do { - /* If the reported head position has wrapped or hasn't advanced, - * fallback to the slow and accurate path. - */ - head = intel_read_status_page(ring, 4); - if (head < ring->actual_head) - head = I915_READ_HEAD(ring); - ring->actual_head = head; - ring->head = head & HEAD_ADDR; - ring->space = ring->head - (ring->tail + 8); - if (ring->space < 0) - ring->space += ring->size; + ring->head = I915_READ_HEAD(ring); + ring->space = ring_space(ring); if (ring->space >= n) { - trace_i915_ring_wait_end (dev); + trace_i915_ring_wait_end(dev); return 0; } @@ -722,29 +967,39 @@ int intel_wait_ring_buffer(struct drm_device *dev, } msleep(1); + if (atomic_read(&dev_priv->mm.wedged)) + return -EAGAIN; } while (!time_after(jiffies, end)); trace_i915_ring_wait_end (dev); return -EBUSY; } -void intel_ring_begin(struct drm_device *dev, - struct intel_ring_buffer *ring, - int num_dwords) +int intel_ring_begin(struct intel_ring_buffer *ring, + int num_dwords) { int n = 4*num_dwords; - if (unlikely(ring->tail + n > ring->size)) - intel_wrap_ring_buffer(dev, ring); - if (unlikely(ring->space < n)) - intel_wait_ring_buffer(dev, ring, n); + int ret; + + if (unlikely(ring->tail + n > ring->effective_size)) { + ret = intel_wrap_ring_buffer(ring); + if (unlikely(ret)) + return ret; + } + + if (unlikely(ring->space < n)) { + ret = intel_wait_ring_buffer(ring, n); + if (unlikely(ret)) + return ret; + } ring->space -= n; + return 0; } -void intel_ring_advance(struct drm_device *dev, - struct intel_ring_buffer *ring) +void intel_ring_advance(struct intel_ring_buffer *ring) { ring->tail &= ring->size - 1; - ring->write_tail(dev, ring, ring->tail); + ring->write_tail(ring, ring->tail); } static const struct intel_ring_buffer render_ring = { @@ -756,10 +1011,11 @@ static const struct intel_ring_buffer render_ring = { .write_tail = ring_write_tail, .flush = render_ring_flush, .add_request = render_ring_add_request, - .get_seqno = render_ring_get_seqno, - .user_irq_get = render_ring_get_user_irq, - .user_irq_put = render_ring_put_user_irq, - .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer, + .get_seqno = ring_get_seqno, + .irq_get = render_ring_get_irq, + .irq_put = render_ring_put_irq, + .dispatch_execbuffer = render_ring_dispatch_execbuffer, + .cleanup = render_ring_cleanup, }; /* ring buffer for bit-stream decoder */ @@ -769,22 +1025,21 @@ static const struct intel_ring_buffer bsd_ring = { .id = RING_BSD, .mmio_base = BSD_RING_BASE, .size = 32 * PAGE_SIZE, - .init = init_bsd_ring, + .init = init_ring_common, .write_tail = ring_write_tail, .flush = bsd_ring_flush, .add_request = ring_add_request, - .get_seqno = ring_status_page_get_seqno, - .user_irq_get = bsd_ring_get_user_irq, - .user_irq_put = bsd_ring_put_user_irq, - .dispatch_gem_execbuffer = ring_dispatch_gem_execbuffer, + .get_seqno = ring_get_seqno, + .irq_get = bsd_ring_get_irq, + .irq_put = bsd_ring_put_irq, + .dispatch_execbuffer = ring_dispatch_execbuffer, }; -static void gen6_bsd_ring_write_tail(struct drm_device *dev, - struct intel_ring_buffer *ring, +static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring, u32 value) { - drm_i915_private_t *dev_priv = dev->dev_private; + drm_i915_private_t *dev_priv = ring->dev->dev_private; /* Every tail move must follow the sequence below */ I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL, @@ -803,69 +1058,112 @@ static void gen6_bsd_ring_write_tail(struct drm_device *dev, GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE); } -static void gen6_ring_flush(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 invalidate_domains, - u32 flush_domains) +static int gen6_ring_flush(struct intel_ring_buffer *ring, + u32 invalidate, u32 flush) { - intel_ring_begin(dev, ring, 4); - intel_ring_emit(dev, ring, MI_FLUSH_DW); - intel_ring_emit(dev, ring, 0); - intel_ring_emit(dev, ring, 0); - intel_ring_emit(dev, ring, 0); - intel_ring_advance(dev, ring); + uint32_t cmd; + int ret; + + if (((invalidate | flush) & I915_GEM_GPU_DOMAINS) == 0) + return 0; + + ret = intel_ring_begin(ring, 4); + if (ret) + return ret; + + cmd = MI_FLUSH_DW; + if (invalidate & I915_GEM_GPU_DOMAINS) + cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD; + intel_ring_emit(ring, cmd); + intel_ring_emit(ring, 0); + intel_ring_emit(ring, 0); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); + return 0; } static int -gen6_ring_dispatch_gem_execbuffer(struct drm_device *dev, - struct intel_ring_buffer *ring, - struct drm_i915_gem_execbuffer2 *exec, - struct drm_clip_rect *cliprects, - uint64_t exec_offset) +gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring, + u32 offset, u32 len) { - uint32_t exec_start; + int ret; - exec_start = (uint32_t) exec_offset + exec->batch_start_offset; + ret = intel_ring_begin(ring, 2); + if (ret) + return ret; - intel_ring_begin(dev, ring, 2); - intel_ring_emit(dev, ring, - MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965); + intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965); /* bit0-7 is the length on GEN6+ */ - intel_ring_emit(dev, ring, exec_start); - intel_ring_advance(dev, ring); + intel_ring_emit(ring, offset); + intel_ring_advance(ring); return 0; } +static bool +gen6_render_ring_get_irq(struct intel_ring_buffer *ring) +{ + return gen6_ring_get_irq(ring, + GT_USER_INTERRUPT, + GEN6_RENDER_USER_INTERRUPT); +} + +static void +gen6_render_ring_put_irq(struct intel_ring_buffer *ring) +{ + return gen6_ring_put_irq(ring, + GT_USER_INTERRUPT, + GEN6_RENDER_USER_INTERRUPT); +} + +static bool +gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring) +{ + return gen6_ring_get_irq(ring, + GT_GEN6_BSD_USER_INTERRUPT, + GEN6_BSD_USER_INTERRUPT); +} + +static void +gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring) +{ + return gen6_ring_put_irq(ring, + GT_GEN6_BSD_USER_INTERRUPT, + GEN6_BSD_USER_INTERRUPT); +} + /* ring buffer for Video Codec for Gen6+ */ static const struct intel_ring_buffer gen6_bsd_ring = { - .name = "gen6 bsd ring", - .id = RING_BSD, - .mmio_base = GEN6_BSD_RING_BASE, - .size = 32 * PAGE_SIZE, - .init = init_bsd_ring, - .write_tail = gen6_bsd_ring_write_tail, - .flush = gen6_ring_flush, - .add_request = ring_add_request, - .get_seqno = ring_status_page_get_seqno, - .user_irq_get = bsd_ring_get_user_irq, - .user_irq_put = bsd_ring_put_user_irq, - .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, + .name = "gen6 bsd ring", + .id = RING_BSD, + .mmio_base = GEN6_BSD_RING_BASE, + .size = 32 * PAGE_SIZE, + .init = init_ring_common, + .write_tail = gen6_bsd_ring_write_tail, + .flush = gen6_ring_flush, + .add_request = gen6_add_request, + .get_seqno = ring_get_seqno, + .irq_get = gen6_bsd_ring_get_irq, + .irq_put = gen6_bsd_ring_put_irq, + .dispatch_execbuffer = gen6_ring_dispatch_execbuffer, }; /* Blitter support (SandyBridge+) */ -static void -blt_ring_get_user_irq(struct drm_device *dev, - struct intel_ring_buffer *ring) +static bool +blt_ring_get_irq(struct intel_ring_buffer *ring) { - /* do nothing */ + return gen6_ring_get_irq(ring, + GT_BLT_USER_INTERRUPT, + GEN6_BLITTER_USER_INTERRUPT); } + static void -blt_ring_put_user_irq(struct drm_device *dev, - struct intel_ring_buffer *ring) +blt_ring_put_irq(struct intel_ring_buffer *ring) { - /* do nothing */ + gen6_ring_put_irq(ring, + GT_BLT_USER_INTERRUPT, + GEN6_BLITTER_USER_INTERRUPT); } @@ -883,32 +1181,31 @@ to_blt_workaround(struct intel_ring_buffer *ring) return ring->private; } -static int blt_ring_init(struct drm_device *dev, - struct intel_ring_buffer *ring) +static int blt_ring_init(struct intel_ring_buffer *ring) { - if (NEED_BLT_WORKAROUND(dev)) { + if (NEED_BLT_WORKAROUND(ring->dev)) { struct drm_i915_gem_object *obj; - u32 __iomem *ptr; + u32 *ptr; int ret; - obj = to_intel_bo(i915_gem_alloc_object(dev, 4096)); + obj = i915_gem_alloc_object(ring->dev, 4096); if (obj == NULL) return -ENOMEM; - ret = i915_gem_object_pin(&obj->base, 4096); + ret = i915_gem_object_pin(obj, 4096, true); if (ret) { drm_gem_object_unreference(&obj->base); return ret; } ptr = kmap(obj->pages[0]); - iowrite32(MI_BATCH_BUFFER_END, ptr); - iowrite32(MI_NOOP, ptr+1); + *ptr++ = MI_BATCH_BUFFER_END; + *ptr++ = MI_NOOP; kunmap(obj->pages[0]); - ret = i915_gem_object_set_to_gtt_domain(&obj->base, false); + ret = i915_gem_object_set_to_gtt_domain(obj, false); if (ret) { - i915_gem_object_unpin(&obj->base); + i915_gem_object_unpin(obj); drm_gem_object_unreference(&obj->base); return ret; } @@ -916,51 +1213,47 @@ static int blt_ring_init(struct drm_device *dev, ring->private = obj; } - return init_ring_common(dev, ring); + return init_ring_common(ring); } -static void blt_ring_begin(struct drm_device *dev, - struct intel_ring_buffer *ring, +static int blt_ring_begin(struct intel_ring_buffer *ring, int num_dwords) { if (ring->private) { - intel_ring_begin(dev, ring, num_dwords+2); - intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START); - intel_ring_emit(dev, ring, to_blt_workaround(ring)->gtt_offset); - } else - intel_ring_begin(dev, ring, 4); -} + int ret = intel_ring_begin(ring, num_dwords+2); + if (ret) + return ret; -static void blt_ring_flush(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 invalidate_domains, - u32 flush_domains) -{ - blt_ring_begin(dev, ring, 4); - intel_ring_emit(dev, ring, MI_FLUSH_DW); - intel_ring_emit(dev, ring, 0); - intel_ring_emit(dev, ring, 0); - intel_ring_emit(dev, ring, 0); - intel_ring_advance(dev, ring); + intel_ring_emit(ring, MI_BATCH_BUFFER_START); + intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset); + + return 0; + } else + return intel_ring_begin(ring, 4); } -static u32 -blt_ring_add_request(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 flush_domains) +static int blt_ring_flush(struct intel_ring_buffer *ring, + u32 invalidate, u32 flush) { - u32 seqno = i915_gem_get_seqno(dev); + uint32_t cmd; + int ret; - blt_ring_begin(dev, ring, 4); - intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX); - intel_ring_emit(dev, ring, - I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT); - intel_ring_emit(dev, ring, seqno); - intel_ring_emit(dev, ring, MI_USER_INTERRUPT); - intel_ring_advance(dev, ring); + if (((invalidate | flush) & I915_GEM_DOMAIN_RENDER) == 0) + return 0; - DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno); - return seqno; + ret = blt_ring_begin(ring, 4); + if (ret) + return ret; + + cmd = MI_FLUSH_DW; + if (invalidate & I915_GEM_DOMAIN_RENDER) + cmd |= MI_INVALIDATE_TLB; + intel_ring_emit(ring, cmd); + intel_ring_emit(ring, 0); + intel_ring_emit(ring, 0); + intel_ring_emit(ring, MI_NOOP); + intel_ring_advance(ring); + return 0; } static void blt_ring_cleanup(struct intel_ring_buffer *ring) @@ -981,47 +1274,98 @@ static const struct intel_ring_buffer gen6_blt_ring = { .init = blt_ring_init, .write_tail = ring_write_tail, .flush = blt_ring_flush, - .add_request = blt_ring_add_request, - .get_seqno = ring_status_page_get_seqno, - .user_irq_get = blt_ring_get_user_irq, - .user_irq_put = blt_ring_put_user_irq, - .dispatch_gem_execbuffer = gen6_ring_dispatch_gem_execbuffer, + .add_request = gen6_add_request, + .get_seqno = ring_get_seqno, + .irq_get = blt_ring_get_irq, + .irq_put = blt_ring_put_irq, + .dispatch_execbuffer = gen6_ring_dispatch_execbuffer, .cleanup = blt_ring_cleanup, }; int intel_init_render_ring_buffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; - - dev_priv->render_ring = render_ring; + struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + + *ring = render_ring; + if (INTEL_INFO(dev)->gen >= 6) { + ring->add_request = gen6_add_request; + ring->irq_get = gen6_render_ring_get_irq; + ring->irq_put = gen6_render_ring_put_irq; + } else if (IS_GEN5(dev)) { + ring->add_request = pc_render_add_request; + ring->get_seqno = pc_render_get_seqno; + } if (!I915_NEED_GFX_HWS(dev)) { - dev_priv->render_ring.status_page.page_addr - = dev_priv->status_page_dmah->vaddr; - memset(dev_priv->render_ring.status_page.page_addr, - 0, PAGE_SIZE); + ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr; + memset(ring->status_page.page_addr, 0, PAGE_SIZE); + } + + return intel_init_ring_buffer(dev, ring); +} + +int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size) +{ + drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring = &dev_priv->ring[RCS]; + + *ring = render_ring; + if (INTEL_INFO(dev)->gen >= 6) { + ring->add_request = gen6_add_request; + ring->irq_get = gen6_render_ring_get_irq; + ring->irq_put = gen6_render_ring_put_irq; + } else if (IS_GEN5(dev)) { + ring->add_request = pc_render_add_request; + ring->get_seqno = pc_render_get_seqno; + } + + ring->dev = dev; + INIT_LIST_HEAD(&ring->active_list); + INIT_LIST_HEAD(&ring->request_list); + INIT_LIST_HEAD(&ring->gpu_write_list); + + ring->size = size; + ring->effective_size = ring->size; + if (IS_I830(ring->dev)) + ring->effective_size -= 128; + + ring->map.offset = start; + ring->map.size = size; + ring->map.type = 0; + ring->map.flags = 0; + ring->map.mtrr = 0; + + drm_core_ioremap_wc(&ring->map, dev); + if (ring->map.handle == NULL) { + DRM_ERROR("can not ioremap virtual address for" + " ring buffer\n"); + return -ENOMEM; } - return intel_init_ring_buffer(dev, &dev_priv->render_ring); + ring->virtual_start = (void __force __iomem *)ring->map.handle; + return 0; } int intel_init_bsd_ring_buffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring = &dev_priv->ring[VCS]; if (IS_GEN6(dev)) - dev_priv->bsd_ring = gen6_bsd_ring; + *ring = gen6_bsd_ring; else - dev_priv->bsd_ring = bsd_ring; + *ring = bsd_ring; - return intel_init_ring_buffer(dev, &dev_priv->bsd_ring); + return intel_init_ring_buffer(dev, ring); } int intel_init_blt_ring_buffer(struct drm_device *dev) { drm_i915_private_t *dev_priv = dev->dev_private; + struct intel_ring_buffer *ring = &dev_priv->ring[BCS]; - dev_priv->blt_ring = gen6_blt_ring; + *ring = gen6_blt_ring; - return intel_init_ring_buffer(dev, &dev_priv->blt_ring); + return intel_init_ring_buffer(dev, ring); } diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h index d2cd0f1efeed..6d6fde85a636 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.h +++ b/drivers/gpu/drm/i915/intel_ringbuffer.h @@ -1,22 +1,40 @@ #ifndef _INTEL_RINGBUFFER_H_ #define _INTEL_RINGBUFFER_H_ +enum { + RCS = 0x0, + VCS, + BCS, + I915_NUM_RINGS, +}; + struct intel_hw_status_page { - void *page_addr; + u32 __iomem *page_addr; unsigned int gfx_addr; - struct drm_gem_object *obj; + struct drm_i915_gem_object *obj; }; -#define I915_READ_TAIL(ring) I915_READ(RING_TAIL(ring->mmio_base)) -#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL(ring->mmio_base), val) -#define I915_READ_START(ring) I915_READ(RING_START(ring->mmio_base)) -#define I915_WRITE_START(ring, val) I915_WRITE(RING_START(ring->mmio_base), val) -#define I915_READ_HEAD(ring) I915_READ(RING_HEAD(ring->mmio_base)) -#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD(ring->mmio_base), val) -#define I915_READ_CTL(ring) I915_READ(RING_CTL(ring->mmio_base)) -#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL(ring->mmio_base), val) +#define I915_RING_READ(reg) i915_safe_read(dev_priv, reg) + +#define I915_READ_TAIL(ring) I915_RING_READ(RING_TAIL((ring)->mmio_base)) +#define I915_WRITE_TAIL(ring, val) I915_WRITE(RING_TAIL((ring)->mmio_base), val) + +#define I915_READ_START(ring) I915_RING_READ(RING_START((ring)->mmio_base)) +#define I915_WRITE_START(ring, val) I915_WRITE(RING_START((ring)->mmio_base), val) + +#define I915_READ_HEAD(ring) I915_RING_READ(RING_HEAD((ring)->mmio_base)) +#define I915_WRITE_HEAD(ring, val) I915_WRITE(RING_HEAD((ring)->mmio_base), val) + +#define I915_READ_CTL(ring) I915_RING_READ(RING_CTL((ring)->mmio_base)) +#define I915_WRITE_CTL(ring, val) I915_WRITE(RING_CTL((ring)->mmio_base), val) + +#define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val) +#define I915_READ_IMR(ring) I915_RING_READ(RING_IMR((ring)->mmio_base)) + +#define I915_READ_NOPID(ring) I915_RING_READ(RING_NOPID((ring)->mmio_base)) +#define I915_READ_SYNC_0(ring) I915_RING_READ(RING_SYNC_0((ring)->mmio_base)) +#define I915_READ_SYNC_1(ring) I915_RING_READ(RING_SYNC_1((ring)->mmio_base)) -struct drm_i915_gem_execbuffer2; struct intel_ring_buffer { const char *name; enum intel_ring_id { @@ -25,45 +43,38 @@ struct intel_ring_buffer { RING_BLT = 0x4, } id; u32 mmio_base; - unsigned long size; void *virtual_start; struct drm_device *dev; - struct drm_gem_object *gem_object; + struct drm_i915_gem_object *obj; - u32 actual_head; u32 head; u32 tail; int space; + int size; + int effective_size; struct intel_hw_status_page status_page; - u32 irq_gem_seqno; /* last seq seem at irq time */ - u32 waiting_gem_seqno; - int user_irq_refcount; - void (*user_irq_get)(struct drm_device *dev, - struct intel_ring_buffer *ring); - void (*user_irq_put)(struct drm_device *dev, - struct intel_ring_buffer *ring); + spinlock_t irq_lock; + u32 irq_refcount; + u32 irq_mask; + u32 irq_seqno; /* last seq seem at irq time */ + u32 waiting_seqno; + u32 sync_seqno[I915_NUM_RINGS-1]; + bool __must_check (*irq_get)(struct intel_ring_buffer *ring); + void (*irq_put)(struct intel_ring_buffer *ring); - int (*init)(struct drm_device *dev, - struct intel_ring_buffer *ring); + int (*init)(struct intel_ring_buffer *ring); - void (*write_tail)(struct drm_device *dev, - struct intel_ring_buffer *ring, + void (*write_tail)(struct intel_ring_buffer *ring, u32 value); - void (*flush)(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 invalidate_domains, - u32 flush_domains); - u32 (*add_request)(struct drm_device *dev, - struct intel_ring_buffer *ring, - u32 flush_domains); - u32 (*get_seqno)(struct drm_device *dev, - struct intel_ring_buffer *ring); - int (*dispatch_gem_execbuffer)(struct drm_device *dev, - struct intel_ring_buffer *ring, - struct drm_i915_gem_execbuffer2 *exec, - struct drm_clip_rect *cliprects, - uint64_t exec_offset); + int __must_check (*flush)(struct intel_ring_buffer *ring, + u32 invalidate_domains, + u32 flush_domains); + int (*add_request)(struct intel_ring_buffer *ring, + u32 *seqno); + u32 (*get_seqno)(struct intel_ring_buffer *ring); + int (*dispatch_execbuffer)(struct intel_ring_buffer *ring, + u32 offset, u32 length); void (*cleanup)(struct intel_ring_buffer *ring); /** @@ -96,7 +107,7 @@ struct intel_ring_buffer { /** * Do we have some not yet emitted requests outstanding? */ - bool outstanding_lazy_request; + u32 outstanding_lazy_request; wait_queue_head_t irq_queue; drm_local_map_t map; @@ -105,44 +116,57 @@ struct intel_ring_buffer { }; static inline u32 +intel_ring_sync_index(struct intel_ring_buffer *ring, + struct intel_ring_buffer *other) +{ + int idx; + + /* + * cs -> 0 = vcs, 1 = bcs + * vcs -> 0 = bcs, 1 = cs, + * bcs -> 0 = cs, 1 = vcs. + */ + + idx = (other - ring) - 1; + if (idx < 0) + idx += I915_NUM_RINGS; + + return idx; +} + +static inline u32 intel_read_status_page(struct intel_ring_buffer *ring, - int reg) + int reg) { - u32 *regs = ring->status_page.page_addr; - return regs[reg]; + return ioread32(ring->status_page.page_addr + reg); } -int intel_init_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring); -void intel_cleanup_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring); -int intel_wait_ring_buffer(struct drm_device *dev, - struct intel_ring_buffer *ring, int n); -void intel_ring_begin(struct drm_device *dev, - struct intel_ring_buffer *ring, int n); - -static inline void intel_ring_emit(struct drm_device *dev, - struct intel_ring_buffer *ring, - unsigned int data) +void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring); +int __must_check intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n); +int __must_check intel_ring_begin(struct intel_ring_buffer *ring, int n); + +static inline void intel_ring_emit(struct intel_ring_buffer *ring, + u32 data) { - unsigned int *virt = ring->virtual_start + ring->tail; - *virt = data; + iowrite32(data, ring->virtual_start + ring->tail); ring->tail += 4; } -void intel_ring_advance(struct drm_device *dev, - struct intel_ring_buffer *ring); +void intel_ring_advance(struct intel_ring_buffer *ring); -u32 intel_ring_get_seqno(struct drm_device *dev, - struct intel_ring_buffer *ring); +u32 intel_ring_get_seqno(struct intel_ring_buffer *ring); +int intel_ring_sync(struct intel_ring_buffer *ring, + struct intel_ring_buffer *to, + u32 seqno); int intel_init_render_ring_buffer(struct drm_device *dev); int intel_init_bsd_ring_buffer(struct drm_device *dev); int intel_init_blt_ring_buffer(struct drm_device *dev); -u32 intel_ring_get_active_head(struct drm_device *dev, - struct intel_ring_buffer *ring); -void intel_ring_setup_status_page(struct drm_device *dev, - struct intel_ring_buffer *ring); +u32 intel_ring_get_active_head(struct intel_ring_buffer *ring); +void intel_ring_setup_status_page(struct intel_ring_buffer *ring); + +/* DRI warts */ +int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size); #endif /* _INTEL_RINGBUFFER_H_ */ diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c index 6bc42fa2a6ec..7c50cdce84f0 100644 --- a/drivers/gpu/drm/i915/intel_sdvo.c +++ b/drivers/gpu/drm/i915/intel_sdvo.c @@ -46,6 +46,7 @@ SDVO_TV_MASK) #define IS_TV(c) (c->output_flag & SDVO_TV_MASK) +#define IS_TMDS(c) (c->output_flag & SDVO_TMDS_MASK) #define IS_LVDS(c) (c->output_flag & SDVO_LVDS_MASK) #define IS_TV_OR_LVDS(c) (c->output_flag & (SDVO_TV_MASK | SDVO_LVDS_MASK)) @@ -473,20 +474,6 @@ static bool intel_sdvo_write_cmd(struct intel_sdvo *intel_sdvo, u8 cmd, return false; } - i = 3; - while (status == SDVO_CMD_STATUS_PENDING && i--) { - if (!intel_sdvo_read_byte(intel_sdvo, - SDVO_I2C_CMD_STATUS, - &status)) - return false; - } - if (status != SDVO_CMD_STATUS_SUCCESS) { - DRM_DEBUG_KMS("command returns response %s [%d]\n", - status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP ? cmd_status_names[status] : "???", - status); - return false; - } - return true; } @@ -497,6 +484,8 @@ static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo, u8 status; int i; + DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(intel_sdvo)); + /* * The documentation states that all commands will be * processed within 15µs, and that we need only poll @@ -505,14 +494,19 @@ static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo, * * Check 5 times in case the hardware failed to read the docs. */ - do { + if (!intel_sdvo_read_byte(intel_sdvo, + SDVO_I2C_CMD_STATUS, + &status)) + goto log_fail; + + while (status == SDVO_CMD_STATUS_PENDING && retry--) { + udelay(15); if (!intel_sdvo_read_byte(intel_sdvo, SDVO_I2C_CMD_STATUS, &status)) - return false; - } while (status == SDVO_CMD_STATUS_PENDING && --retry); + goto log_fail; + } - DRM_DEBUG_KMS("%s: R: ", SDVO_NAME(intel_sdvo)); if (status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP) DRM_LOG_KMS("(%s)", cmd_status_names[status]); else @@ -533,7 +527,7 @@ static bool intel_sdvo_read_response(struct intel_sdvo *intel_sdvo, return true; log_fail: - DRM_LOG_KMS("\n"); + DRM_LOG_KMS("... failed\n"); return false; } @@ -550,6 +544,7 @@ static int intel_sdvo_get_pixel_multiplier(struct drm_display_mode *mode) static bool intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo, u8 ddc_bus) { + /* This must be the immediately preceding write before the i2c xfer */ return intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_CONTROL_BUS_SWITCH, &ddc_bus, 1); @@ -557,7 +552,10 @@ static bool intel_sdvo_set_control_bus_switch(struct intel_sdvo *intel_sdvo, static bool intel_sdvo_set_value(struct intel_sdvo *intel_sdvo, u8 cmd, const void *data, int len) { - return intel_sdvo_write_cmd(intel_sdvo, cmd, data, len); + if (!intel_sdvo_write_cmd(intel_sdvo, cmd, data, len)) + return false; + + return intel_sdvo_read_response(intel_sdvo, NULL, 0); } static bool @@ -859,18 +857,21 @@ static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo) intel_dip_infoframe_csum(&avi_if); - if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_INDEX, + if (!intel_sdvo_set_value(intel_sdvo, + SDVO_CMD_SET_HBUF_INDEX, set_buf_index, 2)) return false; for (i = 0; i < sizeof(avi_if); i += 8) { - if (!intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_DATA, + if (!intel_sdvo_set_value(intel_sdvo, + SDVO_CMD_SET_HBUF_DATA, data, 8)) return false; data++; } - return intel_sdvo_write_cmd(intel_sdvo, SDVO_CMD_SET_HBUF_TXRATE, + return intel_sdvo_set_value(intel_sdvo, + SDVO_CMD_SET_HBUF_TXRATE, &tx_rate, 1); } @@ -1024,9 +1025,13 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder, if (!intel_sdvo_set_target_input(intel_sdvo)) return; - if (intel_sdvo->has_hdmi_monitor && - !intel_sdvo_set_avi_infoframe(intel_sdvo)) - return; + if (intel_sdvo->has_hdmi_monitor) { + intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI); + intel_sdvo_set_colorimetry(intel_sdvo, + SDVO_COLORIMETRY_RGB256); + intel_sdvo_set_avi_infoframe(intel_sdvo); + } else + intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_DVI); if (intel_sdvo->is_tv && !intel_sdvo_set_tv_format(intel_sdvo)) @@ -1045,7 +1050,9 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder, /* Set the SDVO control regs. */ if (INTEL_INFO(dev)->gen >= 4) { - sdvox = SDVO_BORDER_ENABLE; + sdvox = 0; + if (INTEL_INFO(dev)->gen < 5) + sdvox |= SDVO_BORDER_ENABLE; if (adjusted_mode->flags & DRM_MODE_FLAG_PVSYNC) sdvox |= SDVO_VSYNC_ACTIVE_HIGH; if (adjusted_mode->flags & DRM_MODE_FLAG_PHSYNC) @@ -1075,7 +1082,8 @@ static void intel_sdvo_mode_set(struct drm_encoder *encoder, sdvox |= (pixel_multiplier - 1) << SDVO_PORT_MULTIPLY_SHIFT; } - if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL) + if (input_dtd.part2.sdvo_flags & SDVO_NEED_TO_STALL && + INTEL_INFO(dev)->gen < 5) sdvox |= SDVO_STALL_SELECT; intel_sdvo_write_sdvox(intel_sdvo, sdvox); } @@ -1352,7 +1360,8 @@ intel_sdvo_hdmi_sink_detect(struct drm_connector *connector) intel_sdvo->has_hdmi_monitor = drm_detect_hdmi_monitor(edid); intel_sdvo->has_hdmi_audio = drm_detect_monitor_audio(edid); } - } + } else + status = connector_status_disconnected; connector->display_info.raw_edid = NULL; kfree(edid); } @@ -1395,12 +1404,30 @@ intel_sdvo_detect(struct drm_connector *connector, bool force) intel_sdvo->attached_output = response; + intel_sdvo->has_hdmi_monitor = false; + intel_sdvo->has_hdmi_audio = false; + if ((intel_sdvo_connector->output_flag & response) == 0) ret = connector_status_disconnected; - else if (response & SDVO_TMDS_MASK) + else if (IS_TMDS(intel_sdvo_connector)) ret = intel_sdvo_hdmi_sink_detect(connector); - else - ret = connector_status_connected; + else { + struct edid *edid; + + /* if we have an edid check it matches the connection */ + edid = intel_sdvo_get_edid(connector); + if (edid == NULL) + edid = intel_sdvo_get_analog_edid(connector); + if (edid != NULL) { + if (edid->input & DRM_EDID_INPUT_DIGITAL) + ret = connector_status_disconnected; + else + ret = connector_status_connected; + connector->display_info.raw_edid = NULL; + kfree(edid); + } else + ret = connector_status_connected; + } /* May update encoder flag for like clock for SDVO TV, etc.*/ if (ret == connector_status_connected) { @@ -1436,10 +1463,15 @@ static void intel_sdvo_get_ddc_modes(struct drm_connector *connector) edid = intel_sdvo_get_analog_edid(connector); if (edid != NULL) { - if (edid->input & DRM_EDID_INPUT_DIGITAL) { + struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector); + bool monitor_is_digital = !!(edid->input & DRM_EDID_INPUT_DIGITAL); + bool connector_is_digital = !!IS_TMDS(intel_sdvo_connector); + + if (connector_is_digital == monitor_is_digital) { drm_mode_connector_update_edid_property(connector, edid); drm_add_edid_modes(connector, edid); } + connector->display_info.raw_edid = NULL; kfree(edid); } @@ -1658,6 +1690,22 @@ static void intel_sdvo_destroy(struct drm_connector *connector) kfree(connector); } +static bool intel_sdvo_detect_hdmi_audio(struct drm_connector *connector) +{ + struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector); + struct edid *edid; + bool has_audio = false; + + if (!intel_sdvo->is_hdmi) + return false; + + edid = intel_sdvo_get_edid(connector); + if (edid != NULL && edid->input & DRM_EDID_INPUT_DIGITAL) + has_audio = drm_detect_monitor_audio(edid); + + return has_audio; +} + static int intel_sdvo_set_property(struct drm_connector *connector, struct drm_property *property, @@ -1674,17 +1722,23 @@ intel_sdvo_set_property(struct drm_connector *connector, return ret; if (property == intel_sdvo_connector->force_audio_property) { - if (val == intel_sdvo_connector->force_audio) + int i = val; + bool has_audio; + + if (i == intel_sdvo_connector->force_audio) return 0; - intel_sdvo_connector->force_audio = val; + intel_sdvo_connector->force_audio = i; - if (val > 0 && intel_sdvo->has_hdmi_audio) - return 0; - if (val < 0 && !intel_sdvo->has_hdmi_audio) + if (i == 0) + has_audio = intel_sdvo_detect_hdmi_audio(connector); + else + has_audio = i > 0; + + if (has_audio == intel_sdvo->has_hdmi_audio) return 0; - intel_sdvo->has_hdmi_audio = val > 0; + intel_sdvo->has_hdmi_audio = has_audio; goto done; } @@ -1919,20 +1973,7 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv, static bool intel_sdvo_is_hdmi_connector(struct intel_sdvo *intel_sdvo, int device) { - int is_hdmi; - - if (!intel_sdvo_check_supp_encode(intel_sdvo)) - return false; - - if (!intel_sdvo_set_target_output(intel_sdvo, - device == 0 ? SDVO_OUTPUT_TMDS0 : SDVO_OUTPUT_TMDS1)) - return false; - - is_hdmi = 0; - if (!intel_sdvo_get_value(intel_sdvo, SDVO_CMD_GET_ENCODE, &is_hdmi, 1)) - return false; - - return !!is_hdmi; + return intel_sdvo_check_supp_encode(intel_sdvo); } static u8 @@ -2034,12 +2075,7 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device) connector->connector_type = DRM_MODE_CONNECTOR_DVID; if (intel_sdvo_is_hdmi_connector(intel_sdvo, device)) { - /* enable hdmi encoding mode if supported */ - intel_sdvo_set_encode(intel_sdvo, SDVO_ENCODE_HDMI); - intel_sdvo_set_colorimetry(intel_sdvo, - SDVO_COLORIMETRY_RGB256); connector->connector_type = DRM_MODE_CONNECTOR_HDMIA; - intel_sdvo->is_hdmi = true; } intel_sdvo->base.clone_mask = ((1 << INTEL_SDVO_NON_TV_CLONE_BIT) | diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c index 2f7681989316..fe4a53a50b83 100644 --- a/drivers/gpu/drm/i915/intel_tv.c +++ b/drivers/gpu/drm/i915/intel_tv.c @@ -1234,7 +1234,8 @@ static const struct drm_display_mode reported_modes[] = { * \return false if TV is disconnected. */ static int -intel_tv_detect_type (struct intel_tv *intel_tv) +intel_tv_detect_type (struct intel_tv *intel_tv, + struct drm_connector *connector) { struct drm_encoder *encoder = &intel_tv->base.base; struct drm_device *dev = encoder->dev; @@ -1245,10 +1246,13 @@ intel_tv_detect_type (struct intel_tv *intel_tv) int type; /* Disable TV interrupts around load detect or we'll recurse */ - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); - i915_disable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE | - PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + if (connector->polled & DRM_CONNECTOR_POLL_HPD) { + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + i915_disable_pipestat(dev_priv, 0, + PIPE_HOTPLUG_INTERRUPT_ENABLE | + PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + } save_tv_dac = tv_dac = I915_READ(TV_DAC); save_tv_ctl = tv_ctl = I915_READ(TV_CTL); @@ -1301,10 +1305,13 @@ intel_tv_detect_type (struct intel_tv *intel_tv) I915_WRITE(TV_CTL, save_tv_ctl); /* Restore interrupt config */ - spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags); - i915_enable_pipestat(dev_priv, 0, PIPE_HOTPLUG_INTERRUPT_ENABLE | - PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); - spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags); + if (connector->polled & DRM_CONNECTOR_POLL_HPD) { + spin_lock_irqsave(&dev_priv->irq_lock, irqflags); + i915_enable_pipestat(dev_priv, 0, + PIPE_HOTPLUG_INTERRUPT_ENABLE | + PIPE_HOTPLUG_TV_INTERRUPT_ENABLE); + spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags); + } return type; } @@ -1354,7 +1361,7 @@ intel_tv_detect(struct drm_connector *connector, bool force) drm_mode_set_crtcinfo(&mode, CRTC_INTERLACE_HALVE_V); if (intel_tv->base.base.crtc && intel_tv->base.base.crtc->enabled) { - type = intel_tv_detect_type(intel_tv); + type = intel_tv_detect_type(intel_tv, connector); } else if (force) { struct drm_crtc *crtc; int dpms_mode; @@ -1362,7 +1369,7 @@ intel_tv_detect(struct drm_connector *connector, bool force) crtc = intel_get_load_detect_pipe(&intel_tv->base, connector, &mode, &dpms_mode); if (crtc) { - type = intel_tv_detect_type(intel_tv); + type = intel_tv_detect_type(intel_tv, connector); intel_release_load_detect_pipe(&intel_tv->base, connector, dpms_mode); } else @@ -1656,6 +1663,18 @@ intel_tv_init(struct drm_device *dev) intel_encoder = &intel_tv->base; connector = &intel_connector->base; + /* The documentation, for the older chipsets at least, recommend + * using a polling method rather than hotplug detection for TVs. + * This is because in order to perform the hotplug detection, the PLLs + * for the TV must be kept alive increasing power drain and starving + * bandwidth from other encoders. Notably for instance, it causes + * pipe underruns on Crestline when this encoder is supposedly idle. + * + * More recent chipsets favour HDMI rather than integrated S-Video. + */ + connector->polled = + DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT; + drm_connector_init(dev, connector, &intel_tv_connector_funcs, DRM_MODE_CONNECTOR_SVIDEO); |