diff options
Diffstat (limited to 'drivers/gpu/drm/i915/i915_drv.h')
| -rw-r--r-- | drivers/gpu/drm/i915/i915_drv.h | 1149 |
1 files changed, 506 insertions, 643 deletions
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 685e9e065287..243224aeabf8 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -41,6 +41,7 @@ #include <linux/intel-iommu.h> #include <linux/kref.h> #include <linux/pm_qos.h> +#include <linux/reservation.h> #include <linux/shmem_fs.h> #include <drm/drmP.h> @@ -59,9 +60,14 @@ #include "intel_ringbuffer.h" #include "i915_gem.h" +#include "i915_gem_fence_reg.h" +#include "i915_gem_object.h" #include "i915_gem_gtt.h" #include "i915_gem_render_state.h" #include "i915_gem_request.h" +#include "i915_gem_timeline.h" + +#include "i915_vma.h" #include "intel_gvt.h" @@ -70,7 +76,8 @@ #define DRIVER_NAME "i915" #define DRIVER_DESC "Intel Graphics" -#define DRIVER_DATE "20160919" +#define DRIVER_DATE "20161121" +#define DRIVER_TIMESTAMP 1479717903 #undef WARN_ON /* Many gcc seem to no see through this and fall over :( */ @@ -122,6 +129,11 @@ static inline const char *onoff(bool v) return v ? "on" : "off"; } +static inline const char *enableddisabled(bool v) +{ + return v ? "enabled" : "disabled"; +} + enum pipe { INVALID_PIPE = -1, PIPE_A = 0, @@ -182,9 +194,10 @@ enum plane { }; #define plane_name(p) ((p) + 'A') -#define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A') +#define sprite_name(p, s) ((p) * INTEL_INFO(dev_priv)->num_sprites[(p)] + (s) + 'A') enum port { + PORT_NONE = -1, PORT_A = 0, PORT_B, PORT_C, @@ -310,7 +323,7 @@ struct i915_hotplug { #define for_each_pipe_masked(__dev_priv, __p, __mask) \ for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++) \ for_each_if ((__mask) & (1 << (__p))) -#define for_each_plane(__dev_priv, __pipe, __p) \ +#define for_each_universal_plane(__dev_priv, __pipe, __p) \ for ((__p) = 0; \ (__p) < INTEL_INFO(__dev_priv)->num_sprites[(__pipe)] + 1; \ (__p)++) @@ -455,23 +468,6 @@ struct intel_opregion { struct intel_overlay; struct intel_overlay_error_state; -struct drm_i915_fence_reg { - struct list_head link; - struct drm_i915_private *i915; - struct i915_vma *vma; - int pin_count; - int id; - /** - * Whether the tiling parameters for the currently - * associated fence register have changed. Note that - * for the purposes of tracking tiling changes we also - * treat the unfenced register, the register slot that - * the object occupies whilst it executes a fenced - * command (such as BLT on gen2/3), as a "fence". - */ - bool dirty; -}; - struct sdvo_device_mapping { u8 initialized; u8 dvo_port; @@ -483,6 +479,7 @@ struct sdvo_device_mapping { struct intel_connector; struct intel_encoder; +struct intel_atomic_state; struct intel_crtc_state; struct intel_initial_plane_config; struct intel_crtc; @@ -490,16 +487,20 @@ struct intel_limit; struct dpll; struct drm_i915_display_funcs { - int (*get_display_clock_speed)(struct drm_device *dev); - int (*get_fifo_size)(struct drm_device *dev, int plane); + int (*get_display_clock_speed)(struct drm_i915_private *dev_priv); + int (*get_fifo_size)(struct drm_i915_private *dev_priv, int plane); int (*compute_pipe_wm)(struct intel_crtc_state *cstate); int (*compute_intermediate_wm)(struct drm_device *dev, struct intel_crtc *intel_crtc, struct intel_crtc_state *newstate); - void (*initial_watermarks)(struct intel_crtc_state *cstate); - void (*optimize_watermarks)(struct intel_crtc_state *cstate); + void (*initial_watermarks)(struct intel_atomic_state *state, + struct intel_crtc_state *cstate); + void (*atomic_update_watermarks)(struct intel_atomic_state *state, + struct intel_crtc_state *cstate); + void (*optimize_watermarks)(struct intel_atomic_state *state, + struct intel_crtc_state *cstate); int (*compute_global_watermarks)(struct drm_atomic_state *state); - void (*update_wm)(struct drm_crtc *crtc); + void (*update_wm)(struct intel_crtc *crtc); int (*modeset_calc_cdclk)(struct drm_atomic_state *state); void (*modeset_commit_cdclk)(struct drm_atomic_state *state); /* Returns the active state of the crtc, and if the crtc is active, @@ -521,7 +522,7 @@ struct drm_i915_display_funcs { const struct drm_display_mode *adjusted_mode); void (*audio_codec_disable)(struct intel_encoder *encoder); void (*fdi_link_train)(struct drm_crtc *crtc); - void (*init_clock_gating)(struct drm_device *dev); + void (*init_clock_gating)(struct drm_i915_private *dev_priv); int (*queue_flip)(struct drm_device *dev, struct drm_crtc *crtc, struct drm_framebuffer *fb, struct drm_i915_gem_object *obj, @@ -558,6 +559,18 @@ enum forcewake_domains { #define FW_REG_READ (1) #define FW_REG_WRITE (2) +enum decoupled_power_domain { + GEN9_DECOUPLED_PD_BLITTER = 0, + GEN9_DECOUPLED_PD_RENDER, + GEN9_DECOUPLED_PD_MEDIA, + GEN9_DECOUPLED_PD_ALL +}; + +enum decoupled_ops { + GEN9_DECOUPLED_OP_WRITE = 0, + GEN9_DECOUPLED_OP_READ +}; + enum forcewake_domains intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv, i915_reg_t reg, unsigned int op); @@ -581,13 +594,25 @@ struct intel_uncore_funcs { uint32_t val, bool trace); }; +struct intel_forcewake_range { + u32 start; + u32 end; + + enum forcewake_domains domains; +}; + struct intel_uncore { spinlock_t lock; /** lock is also taken in irq contexts. */ + const struct intel_forcewake_range *fw_domains_table; + unsigned int fw_domains_table_entries; + struct intel_uncore_funcs funcs; unsigned fifo_count; + enum forcewake_domains fw_domains; + enum forcewake_domains fw_domains_active; struct intel_uncore_forcewake_domain { struct drm_i915_private *i915; @@ -633,54 +658,55 @@ struct intel_csr { uint32_t allowed_dc_mask; }; -#define DEV_INFO_FOR_EACH_FLAG(func, sep) \ - func(is_mobile) sep \ - func(is_i85x) sep \ - func(is_i915g) sep \ - func(is_i945gm) sep \ - func(is_g33) sep \ - func(hws_needs_physical) sep \ - func(is_g4x) sep \ - func(is_pineview) sep \ - func(is_broadwater) sep \ - func(is_crestline) sep \ - func(is_ivybridge) sep \ - func(is_valleyview) sep \ - func(is_cherryview) sep \ - func(is_haswell) sep \ - func(is_broadwell) sep \ - func(is_skylake) sep \ - func(is_broxton) sep \ - func(is_kabylake) sep \ - func(is_preliminary) sep \ - func(has_fbc) sep \ - func(has_psr) sep \ - func(has_runtime_pm) sep \ - func(has_csr) sep \ - func(has_resource_streamer) sep \ - func(has_rc6) sep \ - func(has_rc6p) sep \ - func(has_dp_mst) sep \ - func(has_gmbus_irq) sep \ - func(has_hw_contexts) sep \ - func(has_logical_ring_contexts) sep \ - func(has_l3_dpf) sep \ - func(has_gmch_display) sep \ - func(has_guc) sep \ - func(has_pipe_cxsr) sep \ - func(has_hotplug) sep \ - func(cursor_needs_physical) sep \ - func(has_overlay) sep \ - func(overlay_needs_physical) sep \ - func(supports_tv) sep \ - func(has_llc) sep \ - func(has_snoop) sep \ - func(has_ddi) sep \ - func(has_fpga_dbg) sep \ - func(has_pooled_eu) - -#define DEFINE_FLAG(name) u8 name:1 -#define SEP_SEMICOLON ; +#define DEV_INFO_FOR_EACH_FLAG(func) \ + /* Keep is_* in chronological order */ \ + func(is_mobile); \ + func(is_i85x); \ + func(is_i915g); \ + func(is_i945gm); \ + func(is_g33); \ + func(is_g4x); \ + func(is_pineview); \ + func(is_broadwater); \ + func(is_crestline); \ + func(is_ivybridge); \ + func(is_valleyview); \ + func(is_cherryview); \ + func(is_haswell); \ + func(is_broadwell); \ + func(is_skylake); \ + func(is_broxton); \ + func(is_kabylake); \ + func(is_alpha_support); \ + /* Keep has_* in alphabetical order */ \ + func(has_64bit_reloc); \ + func(has_csr); \ + func(has_ddi); \ + func(has_dp_mst); \ + func(has_fbc); \ + func(has_fpga_dbg); \ + func(has_gmbus_irq); \ + func(has_gmch_display); \ + func(has_guc); \ + func(has_hotplug); \ + func(has_hw_contexts); \ + func(has_l3_dpf); \ + func(has_llc); \ + func(has_logical_ring_contexts); \ + func(has_overlay); \ + func(has_pipe_cxsr); \ + func(has_pooled_eu); \ + func(has_psr); \ + func(has_rc6); \ + func(has_rc6p); \ + func(has_resource_streamer); \ + func(has_runtime_pm); \ + func(has_snoop); \ + func(cursor_needs_physical); \ + func(hws_needs_physical); \ + func(overlay_needs_physical); \ + func(supports_tv); \ + func(has_decoupled_mmio) struct sseu_dev_info { u8 slice_mask; @@ -709,7 +735,9 @@ struct intel_device_info { u16 gen_mask; u8 ring_mask; /* Rings supported by the HW */ u8 num_rings; - DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON); +#define DEFINE_FLAG(name) u8 name:1 + DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); +#undef DEFINE_FLAG u16 ddb_size; /* in blocks */ /* Register offsets for the various display pipes and transcoders */ int pipe_offsets[I915_MAX_TRANSCODERS]; @@ -726,14 +754,15 @@ struct intel_device_info { } color; }; -#undef DEFINE_FLAG -#undef SEP_SEMICOLON - struct intel_display_error_state; struct drm_i915_error_state { struct kref ref; struct timeval time; + struct timeval boottime; + struct timeval uptime; + + struct drm_i915_private *i915; char error_msg[128]; bool simulated; @@ -759,11 +788,12 @@ struct drm_i915_error_state { u32 gam_ecochk; u32 gab_ctl; u32 gfx_mode; - u32 extra_instdone[I915_NUM_INSTDONE_REG]; + u64 fence[I915_MAX_NUM_FENCES]; struct intel_overlay_error_state *overlay; struct intel_display_error_state *display; struct drm_i915_error_object *semaphore; + struct drm_i915_error_object *guc_log; struct drm_i915_error_engine { int engine_id; @@ -775,12 +805,14 @@ struct drm_i915_error_state { struct i915_address_space *vm; int num_requests; + /* position of active request inside the ring */ + u32 rq_head, rq_post, rq_tail; + /* our own tracking of ring head and tail */ u32 cpu_ring_head; u32 cpu_ring_tail; u32 last_seqno; - u32 semaphore_seqno[I915_NUM_ENGINES - 1]; /* Register state */ u32 start; @@ -791,7 +823,6 @@ struct drm_i915_error_state { u32 hws; u32 ipeir; u32 ipehr; - u32 instdone; u32 bbstate; u32 instpm; u32 instps; @@ -802,11 +833,13 @@ struct drm_i915_error_state { u64 faddr; u32 rc_psmi; /* sleep state */ u32 semaphore_mboxes[I915_NUM_ENGINES - 1]; + struct intel_instdone instdone; struct drm_i915_error_object { - int page_count; u64 gtt_offset; u64 gtt_size; + int page_count; + int unused; u32 *pages[0]; } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page; @@ -815,10 +848,11 @@ struct drm_i915_error_state { struct drm_i915_error_request { long jiffies; pid_t pid; + u32 context; u32 seqno; u32 head; u32 tail; - } *requests; + } *requests, execlist[2]; struct drm_i915_error_waiter { char comm[TASK_COMM_LEN]; @@ -914,6 +948,7 @@ struct i915_gem_context { struct drm_i915_file_private *file_priv; struct i915_hw_ppgtt *ppgtt; struct pid *pid; + const char *name; struct i915_ctx_hang_stats hang_stats; @@ -924,6 +959,7 @@ struct i915_gem_context { /* Unique identifier for this context, used by the hw for tracking */ unsigned int hw_id; u32 user_handle; + int priority; /* greater priorities are serviced first */ u32 ggtt_alignment; @@ -972,6 +1008,9 @@ struct intel_fbc { bool enabled; bool active; + bool underrun_detected; + struct work_struct underrun_work; + struct intel_fbc_state_cache { struct { unsigned int mode_flags; @@ -1297,6 +1336,12 @@ struct i915_power_well { /* cached hw enabled state */ bool hw_enabled; unsigned long domains; + /* unique identifier for this power well */ + unsigned long id; + /* + * Arbitraty data associated with this power well. Platform and power + * well specific. + */ unsigned long data; const struct i915_power_well_ops *ops; }; @@ -1334,11 +1379,22 @@ struct i915_gem_mm { struct list_head bound_list; /** * List of objects which are not bound to the GTT (thus - * are idle and not used by the GPU) but still have - * (presumably uncached) pages still attached. + * are idle and not used by the GPU). These objects may or may + * not actually have any pages attached. */ struct list_head unbound_list; + /** List of all objects in gtt_space, currently mmaped by userspace. + * All objects within this list must also be on bound_list. + */ + struct list_head userfault_list; + + /** + * List of objects which are pending destruction. + */ + struct llist_head free_list; + struct work_struct free_work; + /** Usable portion of the GTT for GEM */ unsigned long stolen_base; /* limited to low memory (32-bit) */ @@ -1368,7 +1424,7 @@ struct i915_gem_mm { /* accounting, useful for userland debugging */ spinlock_t object_stat_lock; - size_t object_memory; + u64 object_memory; u32 object_count; }; @@ -1387,6 +1443,9 @@ struct i915_error_state_file_priv { struct drm_i915_error_state *error; }; +#define I915_RESET_TIMEOUT (10 * HZ) /* 10s */ +#define I915_FENCE_TIMEOUT (10 * HZ) /* 10s */ + struct i915_gpu_error { /* For hangcheck timer */ #define DRM_I915_HANGCHECK_PERIOD 1500 /* in ms */ @@ -1620,7 +1679,6 @@ static inline bool skl_ddb_entry_equal(const struct skl_ddb_entry *e1, } struct skl_ddb_allocation { - struct skl_ddb_entry pipe[I915_MAX_PIPES]; struct skl_ddb_entry plane[I915_MAX_PIPES][I915_MAX_PLANES]; /* packed/uv */ struct skl_ddb_entry y_plane[I915_MAX_PIPES][I915_MAX_PLANES]; }; @@ -1628,15 +1686,12 @@ struct skl_ddb_allocation { struct skl_wm_values { unsigned dirty_pipes; struct skl_ddb_allocation ddb; - uint32_t wm_linetime[I915_MAX_PIPES]; - uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8]; - uint32_t plane_trans[I915_MAX_PIPES][I915_MAX_PLANES]; }; struct skl_wm_level { - bool plane_en[I915_MAX_PLANES]; - uint16_t plane_res_b[I915_MAX_PLANES]; - uint8_t plane_res_l[I915_MAX_PLANES]; + bool plane_en; + uint16_t plane_res_b; + uint8_t plane_res_l; }; /* @@ -1664,7 +1719,6 @@ struct skl_wm_level { */ struct i915_runtime_pm { atomic_t wakeref_count; - atomic_t atomic_seq; bool suspended; bool irqs_enabled; }; @@ -1748,6 +1802,7 @@ struct drm_i915_private { struct kmem_cache *objects; struct kmem_cache *vmas; struct kmem_cache *requests; + struct kmem_cache *dependencies; const struct intel_device_info info; @@ -1759,7 +1814,7 @@ struct drm_i915_private { struct i915_virtual_gpu vgpu; - struct intel_gvt gvt; + struct intel_gvt *gvt; struct intel_guc guc; @@ -1787,9 +1842,8 @@ struct drm_i915_private { struct pci_dev *bridge_dev; struct i915_gem_context *kernel_context; - struct intel_engine_cs engine[I915_NUM_ENGINES]; + struct intel_engine_cs *engine[I915_NUM_ENGINES]; struct i915_vma *semaphore; - u32 next_seqno; struct drm_dma_handle *status_page_dmah; struct resource mch_res; @@ -1814,8 +1868,10 @@ struct drm_i915_private { u32 de_irq_mask[I915_MAX_PIPES]; }; u32 gt_irq_mask; - u32 pm_irq_mask; + u32 pm_imr; + u32 pm_ier; u32 pm_rps_events; + u32 pm_guc_events; u32 pipestat_irq_mask[I915_MAX_PIPES]; struct i915_hotplug hotplug; @@ -1892,8 +1948,8 @@ struct drm_i915_private { /* Kernel Modesetting */ - struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; - struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *plane_to_crtc_mapping[I915_MAX_PIPES]; + struct intel_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES]; wait_queue_head_t pending_flip_queue; #ifdef CONFIG_DEBUG_FS @@ -2009,13 +2065,6 @@ struct drm_i915_private { */ uint16_t skl_latency[8]; - /* - * The skl_wm_values structure is a bit too big for stack - * allocation, so we keep the staging struct where we store - * intermediate results here instead. - */ - struct skl_wm_values skl_results; - /* current hardware state */ union { struct ilk_wm_values hw; @@ -2047,6 +2096,10 @@ struct drm_i915_private { void (*resume)(struct drm_i915_private *); void (*cleanup_engine)(struct intel_engine_cs *engine); + struct list_head timelines; + struct i915_gem_timeline global_timeline; + u32 active_requests; + /** * Is the GPU currently considered idle, or busy executing * userspace requests? Whilst idle, we allow runtime power @@ -2054,7 +2107,6 @@ struct drm_i915_private { * In order to reduce the effect on performance, there * is a slight delay before we do so. */ - unsigned int active_engines; bool awake; /** @@ -2074,12 +2126,15 @@ struct drm_i915_private { * off the idle_work. */ struct delayed_work idle_work; + + ktime_t last_init_time; } gt; /* perform PHY state sanity checks? */ bool chv_phy_assert[2]; - struct intel_encoder *dig_port_map[I915_MAX_PORTS]; + /* Used to save the pipe-to-encoder mapping for audio */ + struct intel_encoder *av_enc_map[I915_MAX_PIPES]; /* * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch @@ -2103,19 +2158,11 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc) } /* Simple iterator over all initialised engines */ -#define for_each_engine(engine__, dev_priv__) \ - for ((engine__) = &(dev_priv__)->engine[0]; \ - (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \ - (engine__)++) \ - for_each_if (intel_engine_initialized(engine__)) - -/* Iterator with engine_id */ -#define for_each_engine_id(engine__, dev_priv__, id__) \ - for ((engine__) = &(dev_priv__)->engine[0], (id__) = 0; \ - (engine__) < &(dev_priv__)->engine[I915_NUM_ENGINES]; \ - (engine__)++) \ - for_each_if (((id__) = (engine__)->id, \ - intel_engine_initialized(engine__))) +#define for_each_engine(engine__, dev_priv__, id__) \ + for ((id__) = 0; \ + (id__) < I915_NUM_ENGINES; \ + (id__)++) \ + for_each_if ((engine__) = (dev_priv__)->engine[(id__)]) #define __mask_next_bit(mask) ({ \ int __idx = ffs(mask) - 1; \ @@ -2126,7 +2173,7 @@ static inline struct drm_i915_private *guc_to_i915(struct intel_guc *guc) /* Iterator over subset of engines selected by mask */ #define for_each_engine_masked(engine__, dev_priv__, mask__, tmp__) \ for (tmp__ = mask__ & INTEL_INFO(dev_priv__)->ring_mask; \ - tmp__ ? (engine__ = &(dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; ) + tmp__ ? (engine__ = (dev_priv__)->engine[__mask_next_bit(tmp__)]), 1 : 0; ) enum hdmi_force_audio { HDMI_AUDIO_OFF_DVI = -2, /* no aux data for HDMI-DVI converter */ @@ -2137,30 +2184,6 @@ enum hdmi_force_audio { #define I915_GTT_OFFSET_NONE ((u32)-1) -struct drm_i915_gem_object_ops { - unsigned int flags; -#define I915_GEM_OBJECT_HAS_STRUCT_PAGE 0x1 - - /* Interface between the GEM object and its backing storage. - * get_pages() is called once prior to the use of the associated set - * of pages before to binding them into the GTT, and put_pages() is - * called after we no longer need them. As we expect there to be - * associated cost with migrating pages between the backing storage - * and making them available for the GPU (e.g. clflush), we may hold - * onto the pages after they are no longer referenced by the GPU - * in case they may be used again shortly (for example migrating the - * pages to a different memory domain within the GTT). put_pages() - * will therefore most likely be called when the object itself is - * being released or under memory pressure (where we attempt to - * reap pages for the shrinker). - */ - int (*get_pages)(struct drm_i915_gem_object *); - void (*put_pages)(struct drm_i915_gem_object *); - - int (*dmabuf_export)(struct drm_i915_gem_object *); - void (*release)(struct drm_i915_gem_object *); -}; - /* * Frontbuffer tracking bits. Set in obj->frontbuffer_bits while a gem bo is * considered to be the frontbuffer for the given plane interface-wise. This @@ -2182,232 +2205,6 @@ struct drm_i915_gem_object_ops { #define INTEL_FRONTBUFFER_ALL_MASK(pipe) \ (0xff << (INTEL_FRONTBUFFER_BITS_PER_PIPE * (pipe))) -struct drm_i915_gem_object { - struct drm_gem_object base; - - const struct drm_i915_gem_object_ops *ops; - - /** List of VMAs backed by this object */ - struct list_head vma_list; - - /** Stolen memory for this object, instead of being backed by shmem. */ - struct drm_mm_node *stolen; - struct list_head global_list; - - /** Used in execbuf to temporarily hold a ref */ - struct list_head obj_exec_link; - - struct list_head batch_pool_link; - - unsigned long flags; - /** - * This is set if the object is on the active lists (has pending - * rendering and so a non-zero seqno), and is not set if it i s on - * inactive (ready to be unbound) list. - */ -#define I915_BO_ACTIVE_SHIFT 0 -#define I915_BO_ACTIVE_MASK ((1 << I915_NUM_ENGINES) - 1) -#define __I915_BO_ACTIVE(bo) \ - ((READ_ONCE((bo)->flags) >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK) - - /** - * This is set if the object has been written to since last bound - * to the GTT - */ - unsigned int dirty:1; - - /** - * Advice: are the backing pages purgeable? - */ - unsigned int madv:2; - - /** - * Whether the current gtt mapping needs to be mappable (and isn't just - * mappable by accident). Track pin and fault separate for a more - * accurate mappable working set. - */ - unsigned int fault_mappable:1; - - /* - * Is the object to be mapped as read-only to the GPU - * Only honoured if hardware has relevant pte bit - */ - unsigned long gt_ro:1; - unsigned int cache_level:3; - unsigned int cache_dirty:1; - - atomic_t frontbuffer_bits; - unsigned int frontbuffer_ggtt_origin; /* write once */ - - /** Current tiling stride for the object, if it's tiled. */ - unsigned int tiling_and_stride; -#define FENCE_MINIMUM_STRIDE 128 /* See i915_tiling_ok() */ -#define TILING_MASK (FENCE_MINIMUM_STRIDE-1) -#define STRIDE_MASK (~TILING_MASK) - - /** Count of VMA actually bound by this object */ - unsigned int bind_count; - unsigned int pin_display; - - struct sg_table *pages; - int pages_pin_count; - struct get_page { - struct scatterlist *sg; - int last; - } get_page; - void *mapping; - - /** Breadcrumb of last rendering to the buffer. - * There can only be one writer, but we allow for multiple readers. - * If there is a writer that necessarily implies that all other - * read requests are complete - but we may only be lazily clearing - * the read requests. A read request is naturally the most recent - * request on a ring, so we may have two different write and read - * requests on one ring where the write request is older than the - * read request. This allows for the CPU to read from an active - * buffer by only waiting for the write to complete. - */ - struct i915_gem_active last_read[I915_NUM_ENGINES]; - struct i915_gem_active last_write; - - /** References from framebuffers, locks out tiling changes. */ - unsigned long framebuffer_references; - - /** Record of address bit 17 of each page at last unbind. */ - unsigned long *bit_17; - - struct i915_gem_userptr { - uintptr_t ptr; - unsigned read_only :1; - unsigned workers :4; -#define I915_GEM_USERPTR_MAX_WORKERS 15 - - struct i915_mm_struct *mm; - struct i915_mmu_object *mmu_object; - struct work_struct *work; - } userptr; - - /** for phys allocated objects */ - struct drm_dma_handle *phys_handle; -}; - -static inline struct drm_i915_gem_object * -to_intel_bo(struct drm_gem_object *gem) -{ - /* Assert that to_intel_bo(NULL) == NULL */ - BUILD_BUG_ON(offsetof(struct drm_i915_gem_object, base)); - - return container_of(gem, struct drm_i915_gem_object, base); -} - -static inline struct drm_i915_gem_object * -i915_gem_object_lookup(struct drm_file *file, u32 handle) -{ - return to_intel_bo(drm_gem_object_lookup(file, handle)); -} - -__deprecated -extern struct drm_gem_object * -drm_gem_object_lookup(struct drm_file *file, u32 handle); - -__attribute__((nonnull)) -static inline struct drm_i915_gem_object * -i915_gem_object_get(struct drm_i915_gem_object *obj) -{ - drm_gem_object_reference(&obj->base); - return obj; -} - -__deprecated -extern void drm_gem_object_reference(struct drm_gem_object *); - -__attribute__((nonnull)) -static inline void -i915_gem_object_put(struct drm_i915_gem_object *obj) -{ - drm_gem_object_unreference(&obj->base); -} - -__deprecated -extern void drm_gem_object_unreference(struct drm_gem_object *); - -__attribute__((nonnull)) -static inline void -i915_gem_object_put_unlocked(struct drm_i915_gem_object *obj) -{ - drm_gem_object_unreference_unlocked(&obj->base); -} - -__deprecated -extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *); - -static inline bool -i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj) -{ - return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE; -} - -static inline unsigned long -i915_gem_object_get_active(const struct drm_i915_gem_object *obj) -{ - return (obj->flags >> I915_BO_ACTIVE_SHIFT) & I915_BO_ACTIVE_MASK; -} - -static inline bool -i915_gem_object_is_active(const struct drm_i915_gem_object *obj) -{ - return i915_gem_object_get_active(obj); -} - -static inline void -i915_gem_object_set_active(struct drm_i915_gem_object *obj, int engine) -{ - obj->flags |= BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline void -i915_gem_object_clear_active(struct drm_i915_gem_object *obj, int engine) -{ - obj->flags &= ~BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline bool -i915_gem_object_has_active_engine(const struct drm_i915_gem_object *obj, - int engine) -{ - return obj->flags & BIT(engine + I915_BO_ACTIVE_SHIFT); -} - -static inline unsigned int -i915_gem_object_get_tiling(struct drm_i915_gem_object *obj) -{ - return obj->tiling_and_stride & TILING_MASK; -} - -static inline bool -i915_gem_object_is_tiled(struct drm_i915_gem_object *obj) -{ - return i915_gem_object_get_tiling(obj) != I915_TILING_NONE; -} - -static inline unsigned int -i915_gem_object_get_stride(struct drm_i915_gem_object *obj) -{ - return obj->tiling_and_stride & STRIDE_MASK; -} - -static inline struct i915_vma *i915_vma_get(struct i915_vma *vma) -{ - i915_gem_object_get(vma->obj); - return vma; -} - -static inline void i915_vma_put(struct i915_vma *vma) -{ - lockdep_assert_held(&vma->vm->dev->struct_mutex); - i915_gem_object_put(vma->obj); -} - /* * Optimised SGL iterator for GEM objects */ @@ -2434,6 +2231,14 @@ static __always_inline struct sgt_iter { return s; } +static inline struct scatterlist *____sg_next(struct scatterlist *sg) +{ + ++sg; + if (unlikely(sg_is_chain(sg))) + sg = sg_chain_ptr(sg); + return sg; +} + /** * __sg_next - return the next scatterlist entry in a list * @sg: The current sg entry @@ -2448,9 +2253,7 @@ static inline struct scatterlist *__sg_next(struct scatterlist *sg) #ifdef CONFIG_DEBUG_SG BUG_ON(sg->sg_magic != SG_MAGIC); #endif - return sg_is_last(sg) ? NULL : - likely(!sg_is_chain(++sg)) ? sg : - sg_chain_ptr(sg); + return sg_is_last(sg) ? NULL : ____sg_next(sg); } /** @@ -2574,23 +2377,19 @@ struct drm_i915_cmd_table { int count; }; -/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */ -#define __I915__(p) ({ \ - struct drm_i915_private *__p; \ - if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \ - __p = (struct drm_i915_private *)p; \ - else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \ - __p = to_i915((struct drm_device *)p); \ - else \ - BUILD_BUG(); \ - __p; \ -}) -#define INTEL_INFO(p) (&__I915__(p)->info) -#define INTEL_GEN(p) (INTEL_INFO(p)->gen) -#define INTEL_DEVID(p) (INTEL_INFO(p)->device_id) +static inline const struct intel_device_info * +intel_info(const struct drm_i915_private *dev_priv) +{ + return &dev_priv->info; +} + +#define INTEL_INFO(dev_priv) intel_info((dev_priv)) + +#define INTEL_GEN(dev_priv) ((dev_priv)->info.gen) +#define INTEL_DEVID(dev_priv) ((dev_priv)->info.device_id) #define REVID_FOREVER 0xff -#define INTEL_REVID(p) (__I915__(p)->drm.pdev->revision) +#define INTEL_REVID(dev_priv) ((dev_priv)->drm.pdev->revision) #define GEN_FOREVER (0) /* @@ -2598,7 +2397,7 @@ struct drm_i915_cmd_table { * * Use GEN_FOREVER for unbound start and or end. */ -#define IS_GEN(p, s, e) ({ \ +#define IS_GEN(dev_priv, s, e) ({ \ unsigned int __s = (s), __e = (e); \ BUILD_BUG_ON(!__builtin_constant_p(s)); \ BUILD_BUG_ON(!__builtin_constant_p(e)); \ @@ -2608,7 +2407,7 @@ struct drm_i915_cmd_table { __e = BITS_PER_LONG - 1; \ else \ __e = (e) - 1; \ - !!(INTEL_INFO(p)->gen_mask & GENMASK((__e), (__s))); \ + !!((dev_priv)->info.gen_mask & GENMASK((__e), (__s))); \ }) /* @@ -2619,75 +2418,75 @@ struct drm_i915_cmd_table { #define IS_REVID(p, since, until) \ (INTEL_REVID(p) >= (since) && INTEL_REVID(p) <= (until)) -#define IS_I830(dev) (INTEL_DEVID(dev) == 0x3577) -#define IS_845G(dev) (INTEL_DEVID(dev) == 0x2562) -#define IS_I85X(dev) (INTEL_INFO(dev)->is_i85x) -#define IS_I865G(dev) (INTEL_DEVID(dev) == 0x2572) -#define IS_I915G(dev) (INTEL_INFO(dev)->is_i915g) -#define IS_I915GM(dev) (INTEL_DEVID(dev) == 0x2592) -#define IS_I945G(dev) (INTEL_DEVID(dev) == 0x2772) -#define IS_I945GM(dev) (INTEL_INFO(dev)->is_i945gm) -#define IS_BROADWATER(dev) (INTEL_INFO(dev)->is_broadwater) -#define IS_CRESTLINE(dev) (INTEL_INFO(dev)->is_crestline) -#define IS_GM45(dev) (INTEL_DEVID(dev) == 0x2A42) -#define IS_G4X(dev) (INTEL_INFO(dev)->is_g4x) -#define IS_PINEVIEW_G(dev) (INTEL_DEVID(dev) == 0xa001) -#define IS_PINEVIEW_M(dev) (INTEL_DEVID(dev) == 0xa011) -#define IS_PINEVIEW(dev) (INTEL_INFO(dev)->is_pineview) -#define IS_G33(dev) (INTEL_INFO(dev)->is_g33) -#define IS_IRONLAKE_M(dev) (INTEL_DEVID(dev) == 0x0046) -#define IS_IVYBRIDGE(dev) (INTEL_INFO(dev)->is_ivybridge) -#define IS_IVB_GT1(dev) (INTEL_DEVID(dev) == 0x0156 || \ - INTEL_DEVID(dev) == 0x0152 || \ - INTEL_DEVID(dev) == 0x015a) -#define IS_VALLEYVIEW(dev) (INTEL_INFO(dev)->is_valleyview) -#define IS_CHERRYVIEW(dev) (INTEL_INFO(dev)->is_cherryview) -#define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell) -#define IS_BROADWELL(dev) (INTEL_INFO(dev)->is_broadwell) -#define IS_SKYLAKE(dev) (INTEL_INFO(dev)->is_skylake) -#define IS_BROXTON(dev) (INTEL_INFO(dev)->is_broxton) -#define IS_KABYLAKE(dev) (INTEL_INFO(dev)->is_kabylake) -#define IS_MOBILE(dev) (INTEL_INFO(dev)->is_mobile) -#define IS_HSW_EARLY_SDV(dev) (IS_HASWELL(dev) && \ - (INTEL_DEVID(dev) & 0xFF00) == 0x0C00) -#define IS_BDW_ULT(dev) (IS_BROADWELL(dev) && \ - ((INTEL_DEVID(dev) & 0xf) == 0x6 || \ - (INTEL_DEVID(dev) & 0xf) == 0xb || \ - (INTEL_DEVID(dev) & 0xf) == 0xe)) +#define IS_I830(dev_priv) (INTEL_DEVID(dev_priv) == 0x3577) +#define IS_845G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2562) +#define IS_I85X(dev_priv) ((dev_priv)->info.is_i85x) +#define IS_I865G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2572) +#define IS_I915G(dev_priv) ((dev_priv)->info.is_i915g) +#define IS_I915GM(dev_priv) (INTEL_DEVID(dev_priv) == 0x2592) +#define IS_I945G(dev_priv) (INTEL_DEVID(dev_priv) == 0x2772) +#define IS_I945GM(dev_priv) ((dev_priv)->info.is_i945gm) +#define IS_BROADWATER(dev_priv) ((dev_priv)->info.is_broadwater) +#define IS_CRESTLINE(dev_priv) ((dev_priv)->info.is_crestline) +#define IS_GM45(dev_priv) (INTEL_DEVID(dev_priv) == 0x2A42) +#define IS_G4X(dev_priv) ((dev_priv)->info.is_g4x) +#define IS_PINEVIEW_G(dev_priv) (INTEL_DEVID(dev_priv) == 0xa001) +#define IS_PINEVIEW_M(dev_priv) (INTEL_DEVID(dev_priv) == 0xa011) +#define IS_PINEVIEW(dev_priv) ((dev_priv)->info.is_pineview) +#define IS_G33(dev_priv) ((dev_priv)->info.is_g33) +#define IS_IRONLAKE_M(dev_priv) (INTEL_DEVID(dev_priv) == 0x0046) +#define IS_IVYBRIDGE(dev_priv) ((dev_priv)->info.is_ivybridge) +#define IS_IVB_GT1(dev_priv) (INTEL_DEVID(dev_priv) == 0x0156 || \ + INTEL_DEVID(dev_priv) == 0x0152 || \ + INTEL_DEVID(dev_priv) == 0x015a) +#define IS_VALLEYVIEW(dev_priv) ((dev_priv)->info.is_valleyview) +#define IS_CHERRYVIEW(dev_priv) ((dev_priv)->info.is_cherryview) +#define IS_HASWELL(dev_priv) ((dev_priv)->info.is_haswell) +#define IS_BROADWELL(dev_priv) ((dev_priv)->info.is_broadwell) +#define IS_SKYLAKE(dev_priv) ((dev_priv)->info.is_skylake) +#define IS_BROXTON(dev_priv) ((dev_priv)->info.is_broxton) +#define IS_KABYLAKE(dev_priv) ((dev_priv)->info.is_kabylake) +#define IS_MOBILE(dev_priv) ((dev_priv)->info.is_mobile) +#define IS_HSW_EARLY_SDV(dev_priv) (IS_HASWELL(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0C00) +#define IS_BDW_ULT(dev_priv) (IS_BROADWELL(dev_priv) && \ + ((INTEL_DEVID(dev_priv) & 0xf) == 0x6 || \ + (INTEL_DEVID(dev_priv) & 0xf) == 0xb || \ + (INTEL_DEVID(dev_priv) & 0xf) == 0xe)) /* ULX machines are also considered ULT. */ -#define IS_BDW_ULX(dev) (IS_BROADWELL(dev) && \ - (INTEL_DEVID(dev) & 0xf) == 0xe) -#define IS_BDW_GT3(dev) (IS_BROADWELL(dev) && \ - (INTEL_DEVID(dev) & 0x00F0) == 0x0020) -#define IS_HSW_ULT(dev) (IS_HASWELL(dev) && \ - (INTEL_DEVID(dev) & 0xFF00) == 0x0A00) -#define IS_HSW_GT3(dev) (IS_HASWELL(dev) && \ - (INTEL_DEVID(dev) & 0x00F0) == 0x0020) +#define IS_BDW_ULX(dev_priv) (IS_BROADWELL(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0xf) == 0xe) +#define IS_BDW_GT3(dev_priv) (IS_BROADWELL(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) +#define IS_HSW_ULT(dev_priv) (IS_HASWELL(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0xFF00) == 0x0A00) +#define IS_HSW_GT3(dev_priv) (IS_HASWELL(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) /* ULX machines are also considered ULT. */ -#define IS_HSW_ULX(dev) (INTEL_DEVID(dev) == 0x0A0E || \ - INTEL_DEVID(dev) == 0x0A1E) -#define IS_SKL_ULT(dev) (INTEL_DEVID(dev) == 0x1906 || \ - INTEL_DEVID(dev) == 0x1913 || \ - INTEL_DEVID(dev) == 0x1916 || \ - INTEL_DEVID(dev) == 0x1921 || \ - INTEL_DEVID(dev) == 0x1926) -#define IS_SKL_ULX(dev) (INTEL_DEVID(dev) == 0x190E || \ - INTEL_DEVID(dev) == 0x1915 || \ - INTEL_DEVID(dev) == 0x191E) -#define IS_KBL_ULT(dev) (INTEL_DEVID(dev) == 0x5906 || \ - INTEL_DEVID(dev) == 0x5913 || \ - INTEL_DEVID(dev) == 0x5916 || \ - INTEL_DEVID(dev) == 0x5921 || \ - INTEL_DEVID(dev) == 0x5926) -#define IS_KBL_ULX(dev) (INTEL_DEVID(dev) == 0x590E || \ - INTEL_DEVID(dev) == 0x5915 || \ - INTEL_DEVID(dev) == 0x591E) -#define IS_SKL_GT3(dev) (IS_SKYLAKE(dev) && \ - (INTEL_DEVID(dev) & 0x00F0) == 0x0020) -#define IS_SKL_GT4(dev) (IS_SKYLAKE(dev) && \ - (INTEL_DEVID(dev) & 0x00F0) == 0x0030) - -#define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary) +#define IS_HSW_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x0A0E || \ + INTEL_DEVID(dev_priv) == 0x0A1E) +#define IS_SKL_ULT(dev_priv) (INTEL_DEVID(dev_priv) == 0x1906 || \ + INTEL_DEVID(dev_priv) == 0x1913 || \ + INTEL_DEVID(dev_priv) == 0x1916 || \ + INTEL_DEVID(dev_priv) == 0x1921 || \ + INTEL_DEVID(dev_priv) == 0x1926) +#define IS_SKL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x190E || \ + INTEL_DEVID(dev_priv) == 0x1915 || \ + INTEL_DEVID(dev_priv) == 0x191E) +#define IS_KBL_ULT(dev_priv) (INTEL_DEVID(dev_priv) == 0x5906 || \ + INTEL_DEVID(dev_priv) == 0x5913 || \ + INTEL_DEVID(dev_priv) == 0x5916 || \ + INTEL_DEVID(dev_priv) == 0x5921 || \ + INTEL_DEVID(dev_priv) == 0x5926) +#define IS_KBL_ULX(dev_priv) (INTEL_DEVID(dev_priv) == 0x590E || \ + INTEL_DEVID(dev_priv) == 0x5915 || \ + INTEL_DEVID(dev_priv) == 0x591E) +#define IS_SKL_GT3(dev_priv) (IS_SKYLAKE(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0020) +#define IS_SKL_GT4(dev_priv) (IS_SKYLAKE(dev_priv) && \ + (INTEL_DEVID(dev_priv) & 0x00F0) == 0x0030) + +#define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support) #define SKL_REVID_A0 0x0 #define SKL_REVID_B0 0x1 @@ -2705,7 +2504,8 @@ struct drm_i915_cmd_table { #define BXT_REVID_B0 0x3 #define BXT_REVID_C0 0x9 -#define IS_BXT_REVID(p, since, until) (IS_BROXTON(p) && IS_REVID(p, since, until)) +#define IS_BXT_REVID(dev_priv, since, until) \ + (IS_BROXTON(dev_priv) && IS_REVID(dev_priv, since, until)) #define KBL_REVID_A0 0x0 #define KBL_REVID_B0 0x1 @@ -2713,8 +2513,8 @@ struct drm_i915_cmd_table { #define KBL_REVID_D0 0x3 #define KBL_REVID_E0 0x4 -#define IS_KBL_REVID(p, since, until) \ - (IS_KABYLAKE(p) && IS_REVID(p, since, until)) +#define IS_KBL_REVID(dev_priv, since, until) \ + (IS_KABYLAKE(dev_priv) && IS_REVID(dev_priv, since, until)) /* * The genX designation typically refers to the render engine, so render @@ -2722,14 +2522,14 @@ struct drm_i915_cmd_table { * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular * chips, etc.). */ -#define IS_GEN2(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(1))) -#define IS_GEN3(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(2))) -#define IS_GEN4(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(3))) -#define IS_GEN5(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(4))) -#define IS_GEN6(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(5))) -#define IS_GEN7(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(6))) -#define IS_GEN8(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(7))) -#define IS_GEN9(dev) (!!(INTEL_INFO(dev)->gen_mask & BIT(8))) +#define IS_GEN2(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(1))) +#define IS_GEN3(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(2))) +#define IS_GEN4(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(3))) +#define IS_GEN5(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(4))) +#define IS_GEN6(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(5))) +#define IS_GEN7(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(6))) +#define IS_GEN8(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(7))) +#define IS_GEN9(dev_priv) (!!((dev_priv)->info.gen_mask & BIT(8))) #define ENGINE_MASK(id) BIT(id) #define RENDER_RING ENGINE_MASK(RCS) @@ -2740,31 +2540,34 @@ struct drm_i915_cmd_table { #define ALL_ENGINES (~0) #define HAS_ENGINE(dev_priv, id) \ - (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id))) + (!!((dev_priv)->info.ring_mask & ENGINE_MASK(id))) #define HAS_BSD(dev_priv) HAS_ENGINE(dev_priv, VCS) #define HAS_BSD2(dev_priv) HAS_ENGINE(dev_priv, VCS2) #define HAS_BLT(dev_priv) HAS_ENGINE(dev_priv, BCS) #define HAS_VEBOX(dev_priv) HAS_ENGINE(dev_priv, VECS) -#define HAS_LLC(dev) (INTEL_INFO(dev)->has_llc) -#define HAS_SNOOP(dev) (INTEL_INFO(dev)->has_snoop) -#define HAS_EDRAM(dev) (!!(__I915__(dev)->edram_cap & EDRAM_ENABLED)) -#define HAS_WT(dev) ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \ - HAS_EDRAM(dev)) -#define HWS_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->hws_needs_physical) +#define HAS_LLC(dev_priv) ((dev_priv)->info.has_llc) +#define HAS_SNOOP(dev_priv) ((dev_priv)->info.has_snoop) +#define HAS_EDRAM(dev_priv) (!!((dev_priv)->edram_cap & EDRAM_ENABLED)) +#define HAS_WT(dev_priv) ((IS_HASWELL(dev_priv) || \ + IS_BROADWELL(dev_priv)) && HAS_EDRAM(dev_priv)) -#define HAS_HW_CONTEXTS(dev) (INTEL_INFO(dev)->has_hw_contexts) -#define HAS_LOGICAL_RING_CONTEXTS(dev) (INTEL_INFO(dev)->has_logical_ring_contexts) -#define USES_PPGTT(dev) (i915.enable_ppgtt) -#define USES_FULL_PPGTT(dev) (i915.enable_ppgtt >= 2) -#define USES_FULL_48BIT_PPGTT(dev) (i915.enable_ppgtt == 3) +#define HWS_NEEDS_PHYSICAL(dev_priv) ((dev_priv)->info.hws_needs_physical) -#define HAS_OVERLAY(dev) (INTEL_INFO(dev)->has_overlay) -#define OVERLAY_NEEDS_PHYSICAL(dev) (INTEL_INFO(dev)->overlay_needs_physical) +#define HAS_HW_CONTEXTS(dev_priv) ((dev_priv)->info.has_hw_contexts) +#define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \ + ((dev_priv)->info.has_logical_ring_contexts) +#define USES_PPGTT(dev_priv) (i915.enable_ppgtt) +#define USES_FULL_PPGTT(dev_priv) (i915.enable_ppgtt >= 2) +#define USES_FULL_48BIT_PPGTT(dev_priv) (i915.enable_ppgtt == 3) + +#define HAS_OVERLAY(dev_priv) ((dev_priv)->info.has_overlay) +#define OVERLAY_NEEDS_PHYSICAL(dev_priv) \ + ((dev_priv)->info.overlay_needs_physical) /* Early gen2 have a totally busted CS tlb and require pinned batches. */ -#define HAS_BROKEN_CS_TLB(dev) (IS_I830(dev) || IS_845G(dev)) +#define HAS_BROKEN_CS_TLB(dev_priv) (IS_I830(dev_priv) || IS_845G(dev_priv)) /* WaRsDisableCoarsePowerGating:skl,bxt */ #define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \ @@ -2778,46 +2581,49 @@ struct drm_i915_cmd_table { * legacy irq no. is shared with another device. The kernel then disables that * interrupt source and so prevents the other device from working properly. */ -#define HAS_AUX_IRQ(dev) (INTEL_INFO(dev)->gen >= 5) -#define HAS_GMBUS_IRQ(dev) (INTEL_INFO(dev)->has_gmbus_irq) +#define HAS_AUX_IRQ(dev_priv) ((dev_priv)->info.gen >= 5) +#define HAS_GMBUS_IRQ(dev_priv) ((dev_priv)->info.has_gmbus_irq) /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte * rows, which changed the alignment requirements and fence programming. */ -#define HAS_128_BYTE_Y_TILING(dev) (!IS_GEN2(dev) && !(IS_I915G(dev) || \ - IS_I915GM(dev))) -#define SUPPORTS_TV(dev) (INTEL_INFO(dev)->supports_tv) -#define I915_HAS_HOTPLUG(dev) (INTEL_INFO(dev)->has_hotplug) +#define HAS_128_BYTE_Y_TILING(dev_priv) (!IS_GEN2(dev_priv) && \ + !(IS_I915G(dev_priv) || \ + IS_I915GM(dev_priv))) +#define SUPPORTS_TV(dev_priv) ((dev_priv)->info.supports_tv) +#define I915_HAS_HOTPLUG(dev_priv) ((dev_priv)->info.has_hotplug) + +#define HAS_FW_BLC(dev_priv) (INTEL_GEN(dev_priv) > 2) +#define HAS_PIPE_CXSR(dev_priv) ((dev_priv)->info.has_pipe_cxsr) +#define HAS_FBC(dev_priv) ((dev_priv)->info.has_fbc) -#define HAS_FW_BLC(dev) (INTEL_INFO(dev)->gen > 2) -#define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr) -#define HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc) +#define HAS_IPS(dev_priv) (IS_HSW_ULT(dev_priv) || IS_BROADWELL(dev_priv)) -#define HAS_IPS(dev) (IS_HSW_ULT(dev) || IS_BROADWELL(dev)) +#define HAS_DP_MST(dev_priv) ((dev_priv)->info.has_dp_mst) -#define HAS_DP_MST(dev) (INTEL_INFO(dev)->has_dp_mst) +#define HAS_DDI(dev_priv) ((dev_priv)->info.has_ddi) +#define HAS_FPGA_DBG_UNCLAIMED(dev_priv) ((dev_priv)->info.has_fpga_dbg) +#define HAS_PSR(dev_priv) ((dev_priv)->info.has_psr) +#define HAS_RC6(dev_priv) ((dev_priv)->info.has_rc6) +#define HAS_RC6p(dev_priv) ((dev_priv)->info.has_rc6p) -#define HAS_DDI(dev) (INTEL_INFO(dev)->has_ddi) -#define HAS_FPGA_DBG_UNCLAIMED(dev) (INTEL_INFO(dev)->has_fpga_dbg) -#define HAS_PSR(dev) (INTEL_INFO(dev)->has_psr) -#define HAS_RUNTIME_PM(dev) (INTEL_INFO(dev)->has_runtime_pm) -#define HAS_RC6(dev) (INTEL_INFO(dev)->has_rc6) -#define HAS_RC6p(dev) (INTEL_INFO(dev)->has_rc6p) +#define HAS_CSR(dev_priv) ((dev_priv)->info.has_csr) -#define HAS_CSR(dev) (INTEL_INFO(dev)->has_csr) +#define HAS_RUNTIME_PM(dev_priv) ((dev_priv)->info.has_runtime_pm) +#define HAS_64BIT_RELOC(dev_priv) ((dev_priv)->info.has_64bit_reloc) /* * For now, anything with a GuC requires uCode loading, and then supports * command submission once loaded. But these are logically independent * properties, so we have separate macros to test them. */ -#define HAS_GUC(dev) (INTEL_INFO(dev)->has_guc) -#define HAS_GUC_UCODE(dev) (HAS_GUC(dev)) -#define HAS_GUC_SCHED(dev) (HAS_GUC(dev)) +#define HAS_GUC(dev_priv) ((dev_priv)->info.has_guc) +#define HAS_GUC_UCODE(dev_priv) (HAS_GUC(dev_priv)) +#define HAS_GUC_SCHED(dev_priv) (HAS_GUC(dev_priv)) -#define HAS_RESOURCE_STREAMER(dev) (INTEL_INFO(dev)->has_resource_streamer) +#define HAS_RESOURCE_STREAMER(dev_priv) ((dev_priv)->info.has_resource_streamer) -#define HAS_POOLED_EU(dev) (INTEL_INFO(dev)->has_pooled_eu) +#define HAS_POOLED_EU(dev_priv) ((dev_priv)->info.has_pooled_eu) #define INTEL_PCH_DEVICE_ID_MASK 0xff00 #define INTEL_PCH_IBX_DEVICE_ID_TYPE 0x3b00 @@ -2832,26 +2638,33 @@ struct drm_i915_cmd_table { #define INTEL_PCH_P3X_DEVICE_ID_TYPE 0x7000 #define INTEL_PCH_QEMU_DEVICE_ID_TYPE 0x2900 /* qemu q35 has 2918 */ -#define INTEL_PCH_TYPE(dev) (__I915__(dev)->pch_type) -#define HAS_PCH_KBP(dev) (INTEL_PCH_TYPE(dev) == PCH_KBP) -#define HAS_PCH_SPT(dev) (INTEL_PCH_TYPE(dev) == PCH_SPT) -#define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT) -#define HAS_PCH_LPT_LP(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) -#define HAS_PCH_LPT_H(dev) (__I915__(dev)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) -#define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT) -#define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX) -#define HAS_PCH_NOP(dev) (INTEL_PCH_TYPE(dev) == PCH_NOP) -#define HAS_PCH_SPLIT(dev) (INTEL_PCH_TYPE(dev) != PCH_NONE) +#define INTEL_PCH_TYPE(dev_priv) ((dev_priv)->pch_type) +#define HAS_PCH_KBP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_KBP) +#define HAS_PCH_SPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_SPT) +#define HAS_PCH_LPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_LPT) +#define HAS_PCH_LPT_LP(dev_priv) \ + ((dev_priv)->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) +#define HAS_PCH_LPT_H(dev_priv) \ + ((dev_priv)->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) +#define HAS_PCH_CPT(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_CPT) +#define HAS_PCH_IBX(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_IBX) +#define HAS_PCH_NOP(dev_priv) (INTEL_PCH_TYPE(dev_priv) == PCH_NOP) +#define HAS_PCH_SPLIT(dev_priv) (INTEL_PCH_TYPE(dev_priv) != PCH_NONE) -#define HAS_GMCH_DISPLAY(dev) (INTEL_INFO(dev)->has_gmch_display) +#define HAS_GMCH_DISPLAY(dev_priv) ((dev_priv)->info.has_gmch_display) + +#define HAS_LSPCON(dev_priv) (IS_GEN9(dev_priv)) /* DPF == dynamic parity feature */ -#define HAS_L3_DPF(dev) (INTEL_INFO(dev)->has_l3_dpf) -#define NUM_L3_SLICES(dev) (IS_HSW_GT3(dev) ? 2 : HAS_L3_DPF(dev)) +#define HAS_L3_DPF(dev_priv) ((dev_priv)->info.has_l3_dpf) +#define NUM_L3_SLICES(dev_priv) (IS_HSW_GT3(dev_priv) ? \ + 2 : HAS_L3_DPF(dev_priv)) #define GT_FREQUENCY_MULTIPLIER 50 #define GEN9_FREQ_SCALER 3 +#define HAS_DECOUPLED_MMIO(dev_priv) (INTEL_INFO(dev_priv)->has_decoupled_mmio) + #include "i915_trace.h" static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv) @@ -2882,6 +2695,8 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level, #ifdef CONFIG_COMPAT extern long i915_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg); +#else +#define i915_compat_ioctl NULL #endif extern const struct dev_pm_ops i915_pm_ops; @@ -2893,6 +2708,7 @@ extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv); extern void i915_reset(struct drm_i915_private *dev_priv); extern int intel_guc_reset(struct drm_i915_private *dev_priv); extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine); +extern void intel_hangcheck_init(struct drm_i915_private *dev_priv); extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv); extern unsigned long i915_mch_val(struct drm_i915_private *dev_priv); extern unsigned long i915_gfx_val(struct drm_i915_private *dev_priv); @@ -2974,7 +2790,7 @@ int intel_wait_for_register_fw(struct drm_i915_private *dev_priv, static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) { - return dev_priv->gvt.initialized; + return dev_priv->gvt; } static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv) @@ -3076,7 +2892,7 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); -void i915_gem_load_init(struct drm_device *dev); +int i915_gem_load_init(struct drm_device *dev); void i915_gem_load_cleanup(struct drm_device *dev); void i915_gem_load_init_fences(struct drm_i915_private *dev_priv); int i915_gem_freeze(struct drm_i915_private *dev_priv); @@ -3087,7 +2903,7 @@ void i915_gem_object_free(struct drm_i915_gem_object *obj); void i915_gem_object_init(struct drm_i915_gem_object *obj, const struct drm_i915_gem_object_ops *ops); struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev, - size_t size); + u64 size); struct drm_i915_gem_object *i915_gem_object_create_from_data( struct drm_device *dev, const void *data, size_t size); void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file); @@ -3100,77 +2916,86 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj, u64 alignment, u64 flags); -int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level, - u32 flags); -void __i915_vma_set_map_and_fenceable(struct i915_vma *vma); -int __must_check i915_vma_unbind(struct i915_vma *vma); -void i915_vma_close(struct i915_vma *vma); -void i915_vma_destroy(struct i915_vma *vma); - int i915_gem_object_unbind(struct drm_i915_gem_object *obj); -int i915_gem_object_put_pages(struct drm_i915_gem_object *obj); -void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv); void i915_gem_release_mmap(struct drm_i915_gem_object *obj); -int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj); +void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv); -static inline int __sg_page_count(struct scatterlist *sg) +static inline int __sg_page_count(const struct scatterlist *sg) { return sg->length >> PAGE_SHIFT; } +struct scatterlist * +i915_gem_object_get_sg(struct drm_i915_gem_object *obj, + unsigned int n, unsigned int *offset); + +struct page * +i915_gem_object_get_page(struct drm_i915_gem_object *obj, + unsigned int n); + struct page * -i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n); +i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, + unsigned int n); -static inline dma_addr_t -i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n) +dma_addr_t +i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, + unsigned long n); + +void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, + struct sg_table *pages); +int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj); + +static inline int __must_check +i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - } + might_lock(&obj->mm.lock); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { - obj->get_page.last += __sg_page_count(obj->get_page.sg++); - if (unlikely(sg_is_chain(obj->get_page.sg))) - obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); - } + if (atomic_inc_not_zero(&obj->mm.pages_pin_count)) + return 0; - return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT); + return __i915_gem_object_get_pages(obj); } -static inline struct page * -i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n) +static inline void +__i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) { - if (WARN_ON(n >= obj->base.size >> PAGE_SHIFT)) - return NULL; - - if (n < obj->get_page.last) { - obj->get_page.sg = obj->pages->sgl; - obj->get_page.last = 0; - } + GEM_BUG_ON(!obj->mm.pages); - while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) { - obj->get_page.last += __sg_page_count(obj->get_page.sg++); - if (unlikely(sg_is_chain(obj->get_page.sg))) - obj->get_page.sg = sg_chain_ptr(obj->get_page.sg); - } + atomic_inc(&obj->mm.pages_pin_count); +} - return nth_page(sg_page(obj->get_page.sg), n - obj->get_page.last); +static inline bool +i915_gem_object_has_pinned_pages(struct drm_i915_gem_object *obj) +{ + return atomic_read(&obj->mm.pages_pin_count); } -static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) +static inline void +__i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - BUG_ON(obj->pages == NULL); - obj->pages_pin_count++; + GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); + GEM_BUG_ON(!obj->mm.pages); + + atomic_dec(&obj->mm.pages_pin_count); + GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count); } -static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) +static inline void +i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj) { - BUG_ON(obj->pages_pin_count == 0); - obj->pages_pin_count--; + __i915_gem_object_unpin_pages(obj); } +enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock */ + I915_MM_NORMAL = 0, + I915_MM_SHRINKER +}; + +void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj, + enum i915_mm_subclass subclass); +void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj); + enum i915_map_type { I915_MAP_WB = 0, I915_MAP_WC, @@ -3186,8 +3011,8 @@ enum i915_map_type { * the kernel address space. Based on the @type of mapping, the PTE will be * set to either WriteBack or WriteCombine (via pgprot_t). * - * The caller must hold the struct_mutex, and is responsible for calling - * i915_gem_object_unpin_map() when the mapping is no longer required. + * The caller is responsible for calling i915_gem_object_unpin_map() when the + * mapping is no longer required. * * Returns the pointer through which to access the mapped object, or an * ERR_PTR() on error. @@ -3203,12 +3028,9 @@ void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj, * with your access, call i915_gem_object_unpin_map() to release the pin * upon the mapping. Once the pin count reaches zero, that mapping may be * removed. - * - * The caller must hold the struct_mutex. */ static inline void i915_gem_object_unpin_map(struct drm_i915_gem_object *obj) { - lockdep_assert_held(&obj->base.dev->struct_mutex); i915_gem_object_unpin_pages(obj); } @@ -3241,7 +3063,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old, struct drm_i915_gem_object *new, unsigned frontbuffer_bits); -int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno); +int __must_check i915_gem_set_global_seqno(struct drm_device *dev, u32 seqno); struct drm_i915_gem_request * i915_gem_find_active_request(struct intel_engine_cs *engine); @@ -3270,19 +3092,25 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error) void i915_gem_reset(struct drm_i915_private *dev_priv); void i915_gem_set_wedged(struct drm_i915_private *dev_priv); -bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); +void i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force); int __must_check i915_gem_init(struct drm_device *dev); int __must_check i915_gem_init_hw(struct drm_device *dev); -void i915_gem_init_swizzling(struct drm_device *dev); +void i915_gem_init_swizzling(struct drm_i915_private *dev_priv); void i915_gem_cleanup_engines(struct drm_device *dev); int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv, unsigned int flags); int __must_check i915_gem_suspend(struct drm_device *dev); void i915_gem_resume(struct drm_device *dev); int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf); -int __must_check -i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj, - bool readonly); +int i915_gem_object_wait(struct drm_i915_gem_object *obj, + unsigned int flags, + long timeout, + struct intel_rps_client *rps); +int i915_gem_object_wait_priority(struct drm_i915_gem_object *obj, + unsigned int flags, + int priority); +#define I915_PRIORITY_DISPLAY I915_PRIORITY_MAX + int __must_check i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write); @@ -3342,57 +3170,17 @@ i915_gem_object_ggtt_offset(struct drm_i915_gem_object *o, return i915_ggtt_offset(i915_gem_object_to_ggtt(o, view)); } -/* i915_gem_fence.c */ +/* i915_gem_fence_reg.c */ int __must_check i915_vma_get_fence(struct i915_vma *vma); int __must_check i915_vma_put_fence(struct i915_vma *vma); -/** - * i915_vma_pin_fence - pin fencing state - * @vma: vma to pin fencing for - * - * This pins the fencing state (whether tiled or untiled) to make sure the - * vma (and its object) is ready to be used as a scanout target. Fencing - * status must be synchronize first by calling i915_vma_get_fence(): - * - * The resulting fence pin reference must be released again with - * i915_vma_unpin_fence(). - * - * Returns: - * - * True if the vma has a fence, false otherwise. - */ -static inline bool -i915_vma_pin_fence(struct i915_vma *vma) -{ - if (vma->fence) { - vma->fence->pin_count++; - return true; - } else - return false; -} - -/** - * i915_vma_unpin_fence - unpin fencing state - * @vma: vma to unpin fencing for - * - * This releases the fence pin reference acquired through - * i915_vma_pin_fence. It will handle both objects with and without an - * attached fence correctly, callers do not need to distinguish this. - */ -static inline void -i915_vma_unpin_fence(struct i915_vma *vma) -{ - if (vma->fence) { - GEM_BUG_ON(vma->fence->pin_count <= 0); - vma->fence->pin_count--; - } -} - -void i915_gem_restore_fences(struct drm_device *dev); +void i915_gem_restore_fences(struct drm_i915_private *dev_priv); -void i915_gem_detect_bit_6_swizzle(struct drm_device *dev); -void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj); -void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj); +void i915_gem_detect_bit_6_swizzle(struct drm_i915_private *dev_priv); +void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages); +void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj, + struct sg_table *pages); /* i915_gem_context.c */ int __must_check i915_gem_context_init(struct drm_device *dev); @@ -3402,6 +3190,9 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file); void i915_gem_context_close(struct drm_device *dev, struct drm_file *file); int i915_switch_context(struct drm_i915_gem_request *req); int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv); +struct i915_vma * +i915_gem_context_pin_legacy(struct i915_gem_context *ctx, + unsigned int flags); void i915_gem_context_free(struct kref *ctx_ref); struct drm_i915_gem_object * i915_gem_alloc_context_obj(struct drm_device *dev, size_t size); @@ -3435,6 +3226,16 @@ static inline void i915_gem_context_put(struct i915_gem_context *ctx) kref_put(&ctx->ref, i915_gem_context_free); } +static inline struct intel_timeline * +i915_gem_context_lookup_timeline(struct i915_gem_context *ctx, + struct intel_engine_cs *engine) +{ + struct i915_address_space *vm; + + vm = ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base; + return &vm->timeline.engine[engine->id]; +} + static inline bool i915_gem_context_is_default(const struct i915_gem_context *c) { return c->user_handle == DEFAULT_CONTEXT_HANDLE; @@ -3478,7 +3279,7 @@ int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *dev_priv, u64 end); void i915_gem_stolen_remove_node(struct drm_i915_private *dev_priv, struct drm_mm_node *node); -int i915_gem_init_stolen(struct drm_device *dev); +int i915_gem_init_stolen(struct drm_i915_private *dev_priv); void i915_gem_cleanup_stolen(struct drm_device *dev); struct drm_i915_gem_object * i915_gem_object_create_stolen(struct drm_device *dev, u32 size); @@ -3488,6 +3289,11 @@ i915_gem_object_create_stolen_for_preallocated(struct drm_device *dev, u32 gtt_offset, u32 size); +/* i915_gem_internal.c */ +struct drm_i915_gem_object * +i915_gem_object_create_internal(struct drm_i915_private *dev_priv, + unsigned int size); + /* i915_gem_shrinker.c */ unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv, unsigned long target, @@ -3526,6 +3332,8 @@ static inline void intel_display_crc_init(struct drm_i915_private *dev_priv) {} #endif /* i915_gpu_error.c */ +#if IS_ENABLED(CONFIG_DRM_I915_CAPTURE_ERROR) + __printf(2, 3) void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...); int i915_error_state_to_str(struct drm_i915_error_state_buf *estr, @@ -3546,7 +3354,20 @@ void i915_error_state_get(struct drm_device *dev, void i915_error_state_put(struct i915_error_state_file_priv *error_priv); void i915_destroy_error_state(struct drm_device *dev); -void i915_get_extra_instdone(struct drm_i915_private *dev_priv, uint32_t *instdone); +#else + +static inline void i915_capture_error_state(struct drm_i915_private *dev_priv, + u32 engine_mask, + const char *error_msg) +{ +} + +static inline void i915_destroy_error_state(struct drm_device *dev) +{ +} + +#endif + const char *i915_cache_level_str(struct drm_i915_private *i915, int type); /* i915_cmd_parser.c */ @@ -3596,6 +3417,9 @@ bool intel_bios_is_port_dp_dual_mode(struct drm_i915_private *dev_priv, enum por bool intel_bios_is_dsi_present(struct drm_i915_private *dev_priv, enum port *port); bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv, enum port port); +bool intel_bios_is_lspcon_present(struct drm_i915_private *dev_priv, + enum port port); + /* intel_opregion.c */ #ifdef CONFIG_ACPI @@ -3652,15 +3476,16 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv); /* modesetting */ extern void intel_modeset_init_hw(struct drm_device *dev); -extern void intel_modeset_init(struct drm_device *dev); +extern int intel_modeset_init(struct drm_device *dev); extern void intel_modeset_gem_init(struct drm_device *dev); extern void intel_modeset_cleanup(struct drm_device *dev); extern int intel_connector_register(struct drm_connector *); extern void intel_connector_unregister(struct drm_connector *); -extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state); +extern int intel_modeset_vga_set_state(struct drm_i915_private *dev_priv, + bool state); extern void intel_display_resume(struct drm_device *dev); -extern void i915_redisable_vga(struct drm_device *dev); -extern void i915_redisable_vga_power_on(struct drm_device *dev); +extern void i915_redisable_vga(struct drm_i915_private *dev_priv); +extern void i915_redisable_vga_power_on(struct drm_i915_private *dev_priv); extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val); extern void intel_init_pch_refclk(struct drm_device *dev); extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val); @@ -3679,11 +3504,13 @@ extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e, extern struct intel_display_error_state * intel_display_capture_error_state(struct drm_i915_private *dev_priv); extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e, - struct drm_device *dev, + struct drm_i915_private *dev_priv, struct intel_display_error_state *error); int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val); int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val); +int skl_pcode_request(struct drm_i915_private *dev_priv, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms); /* intel_sideband.c */ u32 vlv_punit_read(struct drm_i915_private *dev_priv, u32 addr); @@ -3707,6 +3534,23 @@ u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg); void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val); /* intel_dpio_phy.c */ +void bxt_port_to_phy_channel(enum port port, + enum dpio_phy *phy, enum dpio_channel *ch); +void bxt_ddi_phy_set_signal_level(struct drm_i915_private *dev_priv, + enum port port, u32 margin, u32 scale, + u32 enable, u32 deemphasis); +void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy); +void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy); +bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv, + enum dpio_phy phy); +uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder, + uint8_t lane_count); +void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder, + uint8_t lane_lat_optim_mask); +uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder); + void chv_set_phy_signal_level(struct intel_encoder *encoder, u32 deemph_reg_value, u32 margin_reg_value, bool uniq_trans_scale); @@ -3796,11 +3640,30 @@ __raw_write(64, q) #undef __raw_write /* These are untraced mmio-accessors that are only valid to be used inside - * critical sections inside IRQ handlers where forcewake is explicitly + * critical sections, such as inside IRQ handlers, where forcewake is explicitly * controlled. + * * Think twice, and think again, before using these. - * Note: Should only be used between intel_uncore_forcewake_irqlock() and - * intel_uncore_forcewake_irqunlock(). + * + * As an example, these accessors can possibly be used between: + * + * spin_lock_irq(&dev_priv->uncore.lock); + * intel_uncore_forcewake_get__locked(); + * + * and + * + * intel_uncore_forcewake_put__locked(); + * spin_unlock_irq(&dev_priv->uncore.lock); + * + * + * Note: some registers may not need forcewake held, so + * intel_uncore_forcewake_{get,put} can be omitted, see + * intel_uncore_forcewake_for_reg(). + * + * Certain architectures will die if the same cacheline is concurrently accessed + * by different clients (e.g. on Ivybridge). Access to registers should + * therefore generally be serialised, by either the dev_priv->uncore.lock or + * a more localised lock guarding all access to that bank of registers. */ #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__)) #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__)) @@ -3812,11 +3675,11 @@ __raw_write(64, q) #define INTEL_BROADCAST_RGB_FULL 1 #define INTEL_BROADCAST_RGB_LIMITED 2 -static inline i915_reg_t i915_vgacntrl_reg(struct drm_device *dev) +static inline i915_reg_t i915_vgacntrl_reg(struct drm_i915_private *dev_priv) { - if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) + if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) return VLV_VGACNTRL; - else if (INTEL_INFO(dev)->gen >= 5) + else if (INTEL_GEN(dev_priv) >= 5) return CPU_VGACNTRL; else return VGACNTRL; @@ -3877,7 +3740,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) /* Before we do the heavier coherent read of the seqno, * check the value (hopefully) in the CPU cacheline. */ - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; /* Ensure our read of the seqno is coherent so that we @@ -3928,7 +3791,7 @@ __i915_request_irq_complete(struct drm_i915_gem_request *req) wake_up_process(tsk); rcu_read_unlock(); - if (i915_gem_request_completed(req)) + if (__i915_gem_request_completed(req)) return true; } |