From d223f98f02099b002903b9b22b56febae16ef80d Mon Sep 17 00:00:00 2001 From: Eric Anholt Date: Tue, 16 Apr 2019 15:58:54 -0700 Subject: drm/v3d: Add support for compute shader dispatch. The compute shader dispatch interface is pretty simple -- just pass in the regs that userspace has passed us, with no CLs to run. However, with no CL to run it means that we need to do manual cache flushing of the L2 after the HW execution completes (for SSBO, atomic, and image_load_store writes that are the output of compute shaders). This doesn't yet expose the L2 cache's ability to have a region of the address space not write back to memory (which could be used for shared_var storage). So far, the Mesa side has been tested on V3D v4.2 simpenrose (passing the ES31 tests), and on the kernel side on 7278 (failing atomic compswap tests in a way that doesn't reproduce on simpenrose). v2: Fix excessive allocation for the clean_job (reported by Dan Carpenter). Keep refs on jobs until clean_job is finished, to avoid spurious MMU errors if the output BOs are freed by userspace before L2 cleaning is finished. Signed-off-by: Eric Anholt Link: https://patchwork.freedesktop.org/patch/msgid/20190416225856.20264-4-eric@anholt.net Acked-by: Rob Clark --- include/uapi/drm/v3d_drm.h | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index ea70669d2138..58fbe48c91e9 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -37,6 +37,7 @@ extern "C" { #define DRM_V3D_GET_PARAM 0x04 #define DRM_V3D_GET_BO_OFFSET 0x05 #define DRM_V3D_SUBMIT_TFU 0x06 +#define DRM_V3D_SUBMIT_CSD 0x07 #define DRM_IOCTL_V3D_SUBMIT_CL DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CL, struct drm_v3d_submit_cl) #define DRM_IOCTL_V3D_WAIT_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_WAIT_BO, struct drm_v3d_wait_bo) @@ -45,6 +46,7 @@ extern "C" { #define DRM_IOCTL_V3D_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_GET_PARAM, struct drm_v3d_get_param) #define DRM_IOCTL_V3D_GET_BO_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_GET_BO_OFFSET, struct drm_v3d_get_bo_offset) #define DRM_IOCTL_V3D_SUBMIT_TFU DRM_IOW(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_TFU, struct drm_v3d_submit_tfu) +#define DRM_IOCTL_V3D_SUBMIT_CSD DRM_IOW(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CSD, struct drm_v3d_submit_csd) /** * struct drm_v3d_submit_cl - ioctl argument for submitting commands to the 3D @@ -190,6 +192,7 @@ enum drm_v3d_param { DRM_V3D_PARAM_V3D_CORE0_IDENT1, DRM_V3D_PARAM_V3D_CORE0_IDENT2, DRM_V3D_PARAM_SUPPORTS_TFU, + DRM_V3D_PARAM_SUPPORTS_CSD, }; struct drm_v3d_get_param { @@ -230,6 +233,31 @@ struct drm_v3d_submit_tfu { __u32 out_sync; }; +/* Submits a compute shader for dispatch. This job will block on any + * previous compute shaders submitted on this fd, and any other + * synchronization must be performed with in_sync/out_sync. + */ +struct drm_v3d_submit_csd { + __u32 cfg[7]; + __u32 coef[4]; + + /* Pointer to a u32 array of the BOs that are referenced by the job. + */ + __u64 bo_handles; + + /* Number of BO handles passed in (size is that times 4). */ + __u32 bo_handle_count; + + /* sync object to block on before running the CSD job. Each + * CSD job will execute in the order submitted to its FD. + * Synchronization against rendering/TFU jobs or CSD from + * other fds requires using sync objects. + */ + __u32 in_sync; + /* Sync object to signal when the CSD job is done. */ + __u32 out_sync; +}; + #if defined(__cplusplus) } #endif -- cgit v1.2.3-59-g8ed1b From 75b3f1cb50bdbdc7fd557ca3ed63b3eb87c2bab3 Mon Sep 17 00:00:00 2001 From: James Clarke Date: Tue, 15 Jan 2019 15:04:18 +0000 Subject: drm: Fix drm.h uapi header for GNU/kFreeBSD Like GNU/Linux, GNU/kFreeBSD's sys/types.h does not define the uintX_t types, which differs from the BSDs' headers. Thus we should include stdint.h to ensure we have all the required integer types. Signed-off-by: James Clarke Signed-off-by: Eric Anholt Link: https://patchwork.freedesktop.org/patch/msgid/20190115150418.68080-1-jrtc27@jrtc27.com Reviewed-by: Eric Anholt --- include/uapi/drm/drm.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h index 661d73f9a919..8a5b2f8f8eb9 100644 --- a/include/uapi/drm/drm.h +++ b/include/uapi/drm/drm.h @@ -50,6 +50,7 @@ typedef unsigned int drm_handle_t; #else /* One of the BSDs */ +#include #include #include typedef int8_t __s8; -- cgit v1.2.3-59-g8ed1b From 7f3f317a66cac307f6fbc1b5dd74902fb1b48860 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:25 +0100 Subject: drm/i915: Restore control over ppgtt for context creation ABI Having hid the partially exposed new ABI from the PR, put it back again for completion of context recovery. A significant part of context recovery is the ability to reuse as much of the old context as is feasible (to avoid expensive reconstruction). The biggest chunk kept hidden at the moment is fine-control over the ctx->ppgtt (the GPU page tables and associated translation tables and kernel maps), so make control over the ctx->ppgtt explicit. This allows userspace to create and share virtual memory address spaces (within the limits of a single fd) between contexts they own, along with the ability to query the contexts for the vm state. Signed-off-by: Chris Wilson Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-1-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_drv.c | 2 ++ drivers/gpu/drm/i915/i915_gem_context.c | 5 ----- include/uapi/drm/i915_drm.h | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 2c7a4318d13c..5061cb32856b 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -3164,6 +3164,8 @@ static const struct drm_ioctl_desc i915_ioctls[] = { DRM_IOCTL_DEF_DRV(I915_PERF_ADD_CONFIG, i915_perf_add_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_PERF_REMOVE_CONFIG, i915_perf_remove_config_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(I915_QUERY, i915_query_ioctl, DRM_UNLOCKED|DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_VM_CREATE, i915_gem_vm_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(I915_GEM_VM_DESTROY, i915_gem_vm_destroy_ioctl, DRM_RENDER_ALLOW), }; static struct drm_driver driver = { diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 65cefc520e79..413c4529191d 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -98,7 +98,6 @@ #include "i915_user_extensions.h" #define I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE (1 << 1) -#define I915_CONTEXT_PARAM_VM 0x9 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 @@ -966,8 +965,6 @@ static int get_ppgtt(struct drm_i915_file_private *file_priv, struct i915_hw_ppgtt *ppgtt; int ret; - return -EINVAL; /* nothing to see here; please move along */ - if (!ctx->ppgtt) return -ENODEV; @@ -1066,8 +1063,6 @@ static int set_ppgtt(struct drm_i915_file_private *file_priv, struct i915_hw_ppgtt *ppgtt, *old; int err; - return -EINVAL; /* nothing to see here; please move along */ - if (args->size) return -EINVAL; diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 3a73f5316766..d6ad4a15b2b9 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -355,6 +355,8 @@ typedef struct _drm_i915_sarea { #define DRM_I915_PERF_ADD_CONFIG 0x37 #define DRM_I915_PERF_REMOVE_CONFIG 0x38 #define DRM_I915_QUERY 0x39 +#define DRM_I915_GEM_VM_CREATE 0x3a +#define DRM_I915_GEM_VM_DESTROY 0x3b /* Must be kept compact -- no holes */ #define DRM_IOCTL_I915_INIT DRM_IOW( DRM_COMMAND_BASE + DRM_I915_INIT, drm_i915_init_t) @@ -415,6 +417,8 @@ typedef struct _drm_i915_sarea { #define DRM_IOCTL_I915_PERF_ADD_CONFIG DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_ADD_CONFIG, struct drm_i915_perf_oa_config) #define DRM_IOCTL_I915_PERF_REMOVE_CONFIG DRM_IOW(DRM_COMMAND_BASE + DRM_I915_PERF_REMOVE_CONFIG, __u64) #define DRM_IOCTL_I915_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_QUERY, struct drm_i915_query) +#define DRM_IOCTL_I915_GEM_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_I915_GEM_VM_CREATE, struct drm_i915_gem_vm_control) +#define DRM_IOCTL_I915_GEM_VM_DESTROY DRM_IOW (DRM_COMMAND_BASE + DRM_I915_GEM_VM_DESTROY, struct drm_i915_gem_vm_control) /* Allow drivers to submit batchbuffers directly to hardware, relying * on the security mechanisms provided by hardware. @@ -1507,6 +1511,17 @@ struct drm_i915_gem_context_param { * On creation, all new contexts are marked as recoverable. */ #define I915_CONTEXT_PARAM_RECOVERABLE 0x8 + + /* + * The id of the associated virtual memory address space (ppGTT) of + * this context. Can be retrieved and passed to another context + * (on the same fd) for both to use the same ppGTT and so share + * address layouts, and avoid reloading the page tables on context + * switches between themselves. + * + * See DRM_I915_GEM_VM_CREATE and DRM_I915_GEM_VM_DESTROY. + */ +#define I915_CONTEXT_PARAM_VM 0x9 /* Must be kept compact -- no holes and well documented */ __u64 value; -- cgit v1.2.3-59-g8ed1b From 976b55f0e1db5cb8fccb0a42f68ea77ae42604a6 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:26 +0100 Subject: drm/i915: Allow a context to define its set of engines Over the last few years, we have debated how to extend the user API to support an increase in the number of engines, that may be sparse and even be heterogeneous within a class (not all video decoders created equal). We settled on using (class, instance) tuples to identify a specific engine, with an API for the user to construct a map of engines to capabilities. Into this picture, we then add a challenge of virtual engines; one user engine that maps behind the scenes to any number of physical engines. To keep it general, we want the user to have full control over that mapping. To that end, we allow the user to constrain a context to define the set of engines that it can access, order fully controlled by the user via (class, instance). With such precise control in context setup, we can continue to use the existing execbuf uABI of specifying a single index; only now it doesn't automagically map onto the engines, it uses the user defined engine map from the context. v2: Fixup freeing of local on success of get_engines() v3: Allow empty engines[] v4: s/nengine/num_engines/ v5: Replace 64 limit on num_engines with a note that execbuf is currently limited to only using the first 64 engines. v6: Actually use the engines_mutex to guard the ctx->engines. Testcase: igt/gem_ctx_engines Signed-off-by: Chris Wilson Cc: Tvrtko Ursulin Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-2-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_gem_context.c | 265 ++++++++++++++++++++++++-- drivers/gpu/drm/i915/i915_gem_context.h | 18 ++ drivers/gpu/drm/i915/i915_gem_context_types.h | 1 + drivers/gpu/drm/i915/i915_gem_execbuffer.c | 5 +- drivers/gpu/drm/i915/i915_utils.h | 34 ++++ include/uapi/drm/i915_drm.h | 31 +++ 6 files changed, 341 insertions(+), 13 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 413c4529191d..c501e2848ca0 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -90,7 +90,6 @@ #include #include "gt/intel_lrc_reg.h" -#include "gt/intel_workarounds.h" #include "i915_drv.h" #include "i915_globals.h" @@ -141,15 +140,31 @@ static void lut_close(struct i915_gem_context *ctx) } static struct intel_context * -lookup_user_engine(struct i915_gem_context *ctx, u16 class, u16 instance) +lookup_user_engine(struct i915_gem_context *ctx, + unsigned long flags, + const struct i915_engine_class_instance *ci) +#define LOOKUP_USER_INDEX BIT(0) { - struct intel_engine_cs *engine; + int idx; - engine = intel_engine_lookup_user(ctx->i915, class, instance); - if (!engine) + if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) return ERR_PTR(-EINVAL); - return i915_gem_context_get_engine(ctx, engine->id); + if (!i915_gem_context_user_engines(ctx)) { + struct intel_engine_cs *engine; + + engine = intel_engine_lookup_user(ctx->i915, + ci->engine_class, + ci->engine_instance); + if (!engine) + return ERR_PTR(-EINVAL); + + idx = engine->id; + } else { + idx = ci->engine_instance; + } + + return i915_gem_context_get_engine(ctx, idx); } static inline int new_hw_id(struct drm_i915_private *i915, gfp_t gfp) @@ -257,6 +272,17 @@ static void free_engines(struct i915_gem_engines *e) __free_engines(e, e->num_engines); } +static void free_engines_rcu(struct work_struct *wrk) +{ + struct i915_gem_engines *e = + container_of(wrk, struct i915_gem_engines, rcu.work); + struct drm_i915_private *i915 = e->i915; + + mutex_lock(&i915->drm.struct_mutex); + free_engines(e); + mutex_unlock(&i915->drm.struct_mutex); +} + static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) { struct intel_engine_cs *engine; @@ -1352,9 +1378,7 @@ static int set_sseu(struct i915_gem_context *ctx, if (user_sseu.flags || user_sseu.rsvd) return -EINVAL; - ce = lookup_user_engine(ctx, - user_sseu.engine.engine_class, - user_sseu.engine.engine_instance); + ce = lookup_user_engine(ctx, 0, &user_sseu.engine); if (IS_ERR(ce)) return PTR_ERR(ce); @@ -1379,6 +1403,217 @@ out_ce: return ret; } +struct set_engines { + struct i915_gem_context *ctx; + struct i915_gem_engines *engines; +}; + +static const i915_user_extension_fn set_engines__extensions[] = { +}; + +static int +set_engines(struct i915_gem_context *ctx, + const struct drm_i915_gem_context_param *args) +{ + struct i915_context_param_engines __user *user = + u64_to_user_ptr(args->value); + struct set_engines set = { .ctx = ctx }; + unsigned int num_engines, n; + u64 extensions; + int err; + + if (!args->size) { /* switch back to legacy user_ring_map */ + if (!i915_gem_context_user_engines(ctx)) + return 0; + + set.engines = default_engines(ctx); + if (IS_ERR(set.engines)) + return PTR_ERR(set.engines); + + goto replace; + } + + BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); + if (args->size < sizeof(*user) || + !IS_ALIGNED(args->size, sizeof(*user->engines))) { + DRM_DEBUG("Invalid size for engine array: %d\n", + args->size); + return -EINVAL; + } + + /* + * Note that I915_EXEC_RING_MASK limits execbuf to only using the + * first 64 engines defined here. + */ + num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); + + set.engines = kmalloc(struct_size(set.engines, engines, num_engines), + GFP_KERNEL); + if (!set.engines) + return -ENOMEM; + + set.engines->i915 = ctx->i915; + for (n = 0; n < num_engines; n++) { + struct i915_engine_class_instance ci; + struct intel_engine_cs *engine; + + if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { + __free_engines(set.engines, n); + return -EFAULT; + } + + if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && + ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { + set.engines->engines[n] = NULL; + continue; + } + + engine = intel_engine_lookup_user(ctx->i915, + ci.engine_class, + ci.engine_instance); + if (!engine) { + DRM_DEBUG("Invalid engine[%d]: { class:%d, instance:%d }\n", + n, ci.engine_class, ci.engine_instance); + __free_engines(set.engines, n); + return -ENOENT; + } + + set.engines->engines[n] = intel_context_create(ctx, engine); + if (!set.engines->engines[n]) { + __free_engines(set.engines, n); + return -ENOMEM; + } + } + set.engines->num_engines = num_engines; + + err = -EFAULT; + if (!get_user(extensions, &user->extensions)) + err = i915_user_extensions(u64_to_user_ptr(extensions), + set_engines__extensions, + ARRAY_SIZE(set_engines__extensions), + &set); + if (err) { + free_engines(set.engines); + return err; + } + +replace: + mutex_lock(&ctx->engines_mutex); + if (args->size) + i915_gem_context_set_user_engines(ctx); + else + i915_gem_context_clear_user_engines(ctx); + rcu_swap_protected(ctx->engines, set.engines, 1); + mutex_unlock(&ctx->engines_mutex); + + INIT_RCU_WORK(&set.engines->rcu, free_engines_rcu); + queue_rcu_work(system_wq, &set.engines->rcu); + + return 0; +} + +static struct i915_gem_engines * +__copy_engines(struct i915_gem_engines *e) +{ + struct i915_gem_engines *copy; + unsigned int n; + + copy = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); + if (!copy) + return ERR_PTR(-ENOMEM); + + copy->i915 = e->i915; + for (n = 0; n < e->num_engines; n++) { + if (e->engines[n]) + copy->engines[n] = intel_context_get(e->engines[n]); + else + copy->engines[n] = NULL; + } + copy->num_engines = n; + + return copy; +} + +static int +get_engines(struct i915_gem_context *ctx, + struct drm_i915_gem_context_param *args) +{ + struct i915_context_param_engines __user *user; + struct i915_gem_engines *e; + size_t n, count, size; + int err = 0; + + err = mutex_lock_interruptible(&ctx->engines_mutex); + if (err) + return err; + + e = NULL; + if (i915_gem_context_user_engines(ctx)) + e = __copy_engines(i915_gem_context_engines(ctx)); + mutex_unlock(&ctx->engines_mutex); + if (IS_ERR_OR_NULL(e)) { + args->size = 0; + return PTR_ERR_OR_ZERO(e); + } + + count = e->num_engines; + + /* Be paranoid in case we have an impedance mismatch */ + if (!check_struct_size(user, engines, count, &size)) { + err = -EINVAL; + goto err_free; + } + if (overflows_type(size, args->size)) { + err = -EINVAL; + goto err_free; + } + + if (!args->size) { + args->size = size; + goto err_free; + } + + if (args->size < size) { + err = -EINVAL; + goto err_free; + } + + user = u64_to_user_ptr(args->value); + if (!access_ok(user, size)) { + err = -EFAULT; + goto err_free; + } + + if (put_user(0, &user->extensions)) { + err = -EFAULT; + goto err_free; + } + + for (n = 0; n < count; n++) { + struct i915_engine_class_instance ci = { + .engine_class = I915_ENGINE_CLASS_INVALID, + .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, + }; + + if (e->engines[n]) { + ci.engine_class = e->engines[n]->engine->uabi_class; + ci.engine_instance = e->engines[n]->engine->instance; + } + + if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { + err = -EFAULT; + goto err_free; + } + } + + args->size = size; + +err_free: + INIT_RCU_WORK(&e->rcu, free_engines_rcu); + queue_rcu_work(system_wq, &e->rcu); + return err; +} + static int ctx_setparam(struct drm_i915_file_private *fpriv, struct i915_gem_context *ctx, struct drm_i915_gem_context_param *args) @@ -1452,6 +1687,10 @@ static int ctx_setparam(struct drm_i915_file_private *fpriv, ret = set_ppgtt(fpriv, ctx, args); break; + case I915_CONTEXT_PARAM_ENGINES: + ret = set_engines(ctx, args); + break; + case I915_CONTEXT_PARAM_BAN_PERIOD: default: ret = -EINVAL; @@ -1596,9 +1835,7 @@ static int get_sseu(struct i915_gem_context *ctx, if (user_sseu.flags || user_sseu.rsvd) return -EINVAL; - ce = lookup_user_engine(ctx, - user_sseu.engine.engine_class, - user_sseu.engine.engine_instance); + ce = lookup_user_engine(ctx, 0, &user_sseu.engine); if (IS_ERR(ce)) return PTR_ERR(ce); @@ -1682,6 +1919,10 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, ret = get_ppgtt(file_priv, ctx, args); break; + case I915_CONTEXT_PARAM_ENGINES: + ret = get_engines(ctx, args); + break; + case I915_CONTEXT_PARAM_BAN_PERIOD: default: ret = -EINVAL; diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h index 272e183ebc0c..9ad4a6362438 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.h +++ b/drivers/gpu/drm/i915/i915_gem_context.h @@ -112,6 +112,24 @@ static inline void i915_gem_context_set_force_single_submission(struct i915_gem_ __set_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ctx->flags); } +static inline bool +i915_gem_context_user_engines(const struct i915_gem_context *ctx) +{ + return test_bit(CONTEXT_USER_ENGINES, &ctx->flags); +} + +static inline void +i915_gem_context_set_user_engines(struct i915_gem_context *ctx) +{ + set_bit(CONTEXT_USER_ENGINES, &ctx->flags); +} + +static inline void +i915_gem_context_clear_user_engines(struct i915_gem_context *ctx) +{ + clear_bit(CONTEXT_USER_ENGINES, &ctx->flags); +} + int __i915_gem_context_pin_hw_id(struct i915_gem_context *ctx); static inline int i915_gem_context_pin_hw_id(struct i915_gem_context *ctx) { diff --git a/drivers/gpu/drm/i915/i915_gem_context_types.h b/drivers/gpu/drm/i915/i915_gem_context_types.h index d5cb4f121aad..fb965ded2508 100644 --- a/drivers/gpu/drm/i915/i915_gem_context_types.h +++ b/drivers/gpu/drm/i915/i915_gem_context_types.h @@ -146,6 +146,7 @@ struct i915_gem_context { #define CONTEXT_BANNED 0 #define CONTEXT_CLOSED 1 #define CONTEXT_FORCE_SINGLE_SUBMISSION 2 +#define CONTEXT_USER_ENGINES 3 /** * @hw_id: - unique identifier for the context diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index 679f7c1561ba..d6c5220addd0 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -2165,7 +2165,10 @@ eb_select_engine(struct i915_execbuffer *eb, unsigned int idx; int err; - idx = eb_select_legacy_ring(eb, file, args); + if (i915_gem_context_user_engines(eb->gem_context)) + idx = args->flags & I915_EXEC_RING_MASK; + else + idx = eb_select_legacy_ring(eb, file, args); ce = i915_gem_context_get_engine(eb->gem_context, idx); if (IS_ERR(ce)) diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h index 5c94c7ab4607..e52866084891 100644 --- a/drivers/gpu/drm/i915/i915_utils.h +++ b/drivers/gpu/drm/i915/i915_utils.h @@ -26,6 +26,7 @@ #define __I915_UTILS_H #include +#include #include #include #include @@ -78,6 +79,39 @@ #define overflows_type(x, T) \ (sizeof(x) > sizeof(T) && (x) >> BITS_PER_TYPE(T)) +static inline bool +__check_struct_size(size_t base, size_t arr, size_t count, size_t *size) +{ + size_t sz; + + if (check_mul_overflow(count, arr, &sz)) + return false; + + if (check_add_overflow(sz, base, &sz)) + return false; + + *size = sz; + return true; +} + +/** + * check_struct_size() - Calculate size of structure with trailing array. + * @p: Pointer to the structure. + * @member: Name of the array member. + * @n: Number of elements in the array. + * @sz: Total size of structure and array + * + * Calculates size of memory needed for structure @p followed by an + * array of @n @member elements, like struct_size() but reports + * whether it overflowed, and the resultant size in @sz + * + * Return: false if the calculation overflowed. + */ +#define check_struct_size(p, member, n, sz) \ + likely(__check_struct_size(sizeof(*(p)), \ + sizeof(*(p)->member) + __must_be_array((p)->member), \ + n, sz)) + #define ptr_mask_bits(ptr, n) ({ \ unsigned long __v = (unsigned long)(ptr); \ (typeof(ptr))(__v & -BIT(n)); \ diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index d6ad4a15b2b9..8e1bb22926e4 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -136,6 +136,7 @@ enum drm_i915_gem_engine_class { struct i915_engine_class_instance { __u16 engine_class; /* see enum drm_i915_gem_engine_class */ __u16 engine_instance; +#define I915_ENGINE_CLASS_INVALID_NONE -1 }; /** @@ -1522,6 +1523,26 @@ struct drm_i915_gem_context_param { * See DRM_I915_GEM_VM_CREATE and DRM_I915_GEM_VM_DESTROY. */ #define I915_CONTEXT_PARAM_VM 0x9 + +/* + * I915_CONTEXT_PARAM_ENGINES: + * + * Bind this context to operate on this subset of available engines. Henceforth, + * the I915_EXEC_RING selector for DRM_IOCTL_I915_GEM_EXECBUFFER2 operates as + * an index into this array of engines; I915_EXEC_DEFAULT selecting engine[0] + * and upwards. Slots 0...N are filled in using the specified (class, instance). + * Use + * engine_class: I915_ENGINE_CLASS_INVALID, + * engine_instance: I915_ENGINE_CLASS_INVALID_NONE + * to specify a gap in the array that can be filled in later, e.g. by a + * virtual engine used for load balancing. + * + * Setting the number of engines bound to the context to 0, by passing a zero + * sized argument, will revert back to default settings. + * + * See struct i915_context_param_engines. + */ +#define I915_CONTEXT_PARAM_ENGINES 0xa /* Must be kept compact -- no holes and well documented */ __u64 value; @@ -1585,6 +1606,16 @@ struct drm_i915_gem_context_param_sseu { __u32 rsvd; }; +struct i915_context_param_engines { + __u64 extensions; /* linked chain of extension blocks, 0 terminates */ + struct i915_engine_class_instance engines[0]; +} __attribute__((packed)); + +#define I915_DEFINE_CONTEXT_PARAM_ENGINES(name__, N__) struct { \ + __u64 extensions; \ + struct i915_engine_class_instance engines[N__]; \ +} __attribute__((packed)) name__ + struct drm_i915_gem_context_create_ext_setparam { #define I915_CONTEXT_CREATE_EXT_SETPARAM 0 struct i915_user_extension base; -- cgit v1.2.3-59-g8ed1b From e620f7b3a26389dfce2663ad4e64c2271ad1a815 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:27 +0100 Subject: drm/i915: Extend I915_CONTEXT_PARAM_SSEU to support local ctx->engine[] Allow the user to specify a local engine index (as opposed to class:index) that they can use to refer to a preset engine inside the ctx->engine[] array defined by an earlier I915_CONTEXT_PARAM_ENGINES. This will be useful for setting SSEU parameters on virtual engines that are local to the context and do not have a valid global class:instance lookup. Note that due to the ambiguity in using class:instance with ctx->engines[], if a user supplied engine map is active the user must specify the engine to alter by its index into the ctx->engines[]. Signed-off-by: Chris Wilson Cc: Tvrtko Ursulin Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-3-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_gem_context.c | 24 ++++++++++++++++++++---- include/uapi/drm/i915_drm.h | 3 ++- 2 files changed, 22 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index c501e2848ca0..d38d8a45a5cf 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -1363,6 +1363,7 @@ static int set_sseu(struct i915_gem_context *ctx, struct drm_i915_gem_context_param_sseu user_sseu; struct intel_context *ce; struct intel_sseu sseu; + unsigned long lookup; int ret; if (args->size < sizeof(user_sseu)) @@ -1375,10 +1376,17 @@ static int set_sseu(struct i915_gem_context *ctx, sizeof(user_sseu))) return -EFAULT; - if (user_sseu.flags || user_sseu.rsvd) + if (user_sseu.rsvd) return -EINVAL; - ce = lookup_user_engine(ctx, 0, &user_sseu.engine); + if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) + return -EINVAL; + + lookup = 0; + if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) + lookup |= LOOKUP_USER_INDEX; + + ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); if (IS_ERR(ce)) return PTR_ERR(ce); @@ -1821,6 +1829,7 @@ static int get_sseu(struct i915_gem_context *ctx, { struct drm_i915_gem_context_param_sseu user_sseu; struct intel_context *ce; + unsigned long lookup; int err; if (args->size == 0) @@ -1832,10 +1841,17 @@ static int get_sseu(struct i915_gem_context *ctx, sizeof(user_sseu))) return -EFAULT; - if (user_sseu.flags || user_sseu.rsvd) + if (user_sseu.rsvd) return -EINVAL; - ce = lookup_user_engine(ctx, 0, &user_sseu.engine); + if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) + return -EINVAL; + + lookup = 0; + if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) + lookup |= LOOKUP_USER_INDEX; + + ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); if (IS_ERR(ce)) return PTR_ERR(ce); diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 8e1bb22926e4..82bd488ed0d1 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -1576,9 +1576,10 @@ struct drm_i915_gem_context_param_sseu { struct i915_engine_class_instance engine; /* - * Unused for now. Must be cleared to zero. + * Unknown flags must be cleared to zero. */ __u32 flags; +#define I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX (1u << 0) /* * Mask of slices to enable for the context. Valid values are a subset -- cgit v1.2.3-59-g8ed1b From 8319f44c0525708c26ac7724da897cff3dbb0f84 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:28 +0100 Subject: drm/i915: Re-expose SINGLE_TIMELINE flags for context creation The SINGLE_TIMELINE flag can be used to create a context such that all engine instances within that context share a common timeline. This can be useful for mixing operations between real and virtual engines, or when using a composite context for a single client API context. Signed-off-by: Chris Wilson Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-4-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_gem_context.c | 4 ---- include/uapi/drm/i915_drm.h | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index d38d8a45a5cf..d391d474820f 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -96,8 +96,6 @@ #include "i915_trace.h" #include "i915_user_extensions.h" -#define I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE (1 << 1) - #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 static struct i915_global_gem_context { @@ -505,8 +503,6 @@ i915_gem_create_context(struct drm_i915_private *dev_priv, unsigned int flags) lockdep_assert_held(&dev_priv->drm.struct_mutex); - BUILD_BUG_ON(I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE & - ~I915_CONTEXT_CREATE_FLAGS_UNKNOWN); if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && !HAS_EXECLISTS(dev_priv)) return ERR_PTR(-EINVAL); diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 82bd488ed0d1..957ba8e60e02 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -1469,8 +1469,9 @@ struct drm_i915_gem_context_create_ext { __u32 ctx_id; /* output: id of new context*/ __u32 flags; #define I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS (1u << 0) +#define I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE (1u << 1) #define I915_CONTEXT_CREATE_FLAGS_UNKNOWN \ - (-(I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS << 1)) + (-(I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE << 1)) __u64 extensions; }; -- cgit v1.2.3-59-g8ed1b From b81dde719439c8f09bb61e742ed95bfc4b33946b Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:29 +0100 Subject: drm/i915: Allow userspace to clone contexts on creation A usecase arose out of handling context recovery in mesa, whereby they wish to recreate a context with fresh logical state but preserving all other details of the original. Currently, they create a new context and iterate over which bits they want to copy across, but it would much more convenient if they were able to just pass in a target context to clone during creation. This essentially extends the setparam during creation to pull the details from a target context instead of the user supplied parameters. The ideal here is that we don't expose control over anything more than can be obtained via CONTEXT_PARAM. That is userspace retains explicit control over all features, and this api is just convenience. For example, you could replace struct context_param p = { .param = CONTEXT_PARAM_VM }; param.ctx_id = old_id; gem_context_get_param(&p.param); new_id = gem_context_create(); param.ctx_id = new_id; gem_context_set_param(&p.param); gem_vm_destroy(param.value); /* drop the ref to VM_ID handle */ with struct create_ext_param p = { { .name = CONTEXT_CREATE_CLONE }, .clone_id = old_id, .flags = CLONE_FLAGS_VM } new_id = gem_context_create_ext(&p); and not have to worry about stray namespace pollution etc. Signed-off-by: Chris Wilson Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-5-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_gem_context.c | 206 ++++++++++++++++++++++++++++++++ include/uapi/drm/i915_drm.h | 15 +++ 2 files changed, 221 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index d391d474820f..24736fcd463d 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -1723,8 +1723,214 @@ static int create_setparam(struct i915_user_extension __user *ext, void *data) return ctx_setparam(arg->fpriv, arg->ctx, &local.param); } +static int clone_engines(struct i915_gem_context *dst, + struct i915_gem_context *src) +{ + struct i915_gem_engines *e = i915_gem_context_lock_engines(src); + struct i915_gem_engines *clone; + bool user_engines; + unsigned long n; + + clone = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); + if (!clone) + goto err_unlock; + + clone->i915 = dst->i915; + for (n = 0; n < e->num_engines; n++) { + if (!e->engines[n]) { + clone->engines[n] = NULL; + continue; + } + + clone->engines[n] = + intel_context_create(dst, e->engines[n]->engine); + if (!clone->engines[n]) { + __free_engines(clone, n); + goto err_unlock; + } + } + clone->num_engines = n; + + user_engines = i915_gem_context_user_engines(src); + i915_gem_context_unlock_engines(src); + + free_engines(dst->engines); + RCU_INIT_POINTER(dst->engines, clone); + if (user_engines) + i915_gem_context_set_user_engines(dst); + else + i915_gem_context_clear_user_engines(dst); + return 0; + +err_unlock: + i915_gem_context_unlock_engines(src); + return -ENOMEM; +} + +static int clone_flags(struct i915_gem_context *dst, + struct i915_gem_context *src) +{ + dst->user_flags = src->user_flags; + return 0; +} + +static int clone_schedattr(struct i915_gem_context *dst, + struct i915_gem_context *src) +{ + dst->sched = src->sched; + return 0; +} + +static int clone_sseu(struct i915_gem_context *dst, + struct i915_gem_context *src) +{ + struct i915_gem_engines *e = i915_gem_context_lock_engines(src); + struct i915_gem_engines *clone; + unsigned long n; + int err; + + clone = dst->engines; /* no locking required; sole access */ + if (e->num_engines != clone->num_engines) { + err = -EINVAL; + goto unlock; + } + + for (n = 0; n < e->num_engines; n++) { + struct intel_context *ce = e->engines[n]; + + if (clone->engines[n]->engine->class != ce->engine->class) { + /* Must have compatible engine maps! */ + err = -EINVAL; + goto unlock; + } + + /* serialises with set_sseu */ + err = intel_context_lock_pinned(ce); + if (err) + goto unlock; + + clone->engines[n]->sseu = ce->sseu; + intel_context_unlock_pinned(ce); + } + + err = 0; +unlock: + i915_gem_context_unlock_engines(src); + return err; +} + +static int clone_timeline(struct i915_gem_context *dst, + struct i915_gem_context *src) +{ + if (src->timeline) { + GEM_BUG_ON(src->timeline == dst->timeline); + + if (dst->timeline) + i915_timeline_put(dst->timeline); + dst->timeline = i915_timeline_get(src->timeline); + } + + return 0; +} + +static int clone_vm(struct i915_gem_context *dst, + struct i915_gem_context *src) +{ + struct i915_hw_ppgtt *ppgtt; + + rcu_read_lock(); + do { + ppgtt = READ_ONCE(src->ppgtt); + if (!ppgtt) + break; + + if (!kref_get_unless_zero(&ppgtt->ref)) + continue; + + /* + * This ppgtt may have be reallocated between + * the read and the kref, and reassigned to a third + * context. In order to avoid inadvertent sharing + * of this ppgtt with that third context (and not + * src), we have to confirm that we have the same + * ppgtt after passing through the strong memory + * barrier implied by a successful + * kref_get_unless_zero(). + * + * Once we have acquired the current ppgtt of src, + * we no longer care if it is released from src, as + * it cannot be reallocated elsewhere. + */ + + if (ppgtt == READ_ONCE(src->ppgtt)) + break; + + i915_ppgtt_put(ppgtt); + } while (1); + rcu_read_unlock(); + + if (ppgtt) { + __assign_ppgtt(dst, ppgtt); + i915_ppgtt_put(ppgtt); + } + + return 0; +} + +static int create_clone(struct i915_user_extension __user *ext, void *data) +{ + static int (* const fn[])(struct i915_gem_context *dst, + struct i915_gem_context *src) = { +#define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y + MAP(ENGINES, clone_engines), + MAP(FLAGS, clone_flags), + MAP(SCHEDATTR, clone_schedattr), + MAP(SSEU, clone_sseu), + MAP(TIMELINE, clone_timeline), + MAP(VM, clone_vm), +#undef MAP + }; + struct drm_i915_gem_context_create_ext_clone local; + const struct create_ext *arg = data; + struct i915_gem_context *dst = arg->ctx; + struct i915_gem_context *src; + int err, bit; + + if (copy_from_user(&local, ext, sizeof(local))) + return -EFAULT; + + BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != + I915_CONTEXT_CLONE_UNKNOWN); + + if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) + return -EINVAL; + + if (local.rsvd) + return -EINVAL; + + rcu_read_lock(); + src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); + rcu_read_unlock(); + if (!src) + return -ENOENT; + + GEM_BUG_ON(src == dst); + + for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { + if (!(local.flags & BIT(bit))) + continue; + + err = fn[bit](dst, src); + if (err) + return err; + } + + return 0; +} + static const i915_user_extension_fn create_extensions[] = { [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, + [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, }; static bool client_is_banned(struct drm_i915_file_private *file_priv) diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 957ba8e60e02..62396d575e28 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -1624,6 +1624,21 @@ struct drm_i915_gem_context_create_ext_setparam { struct drm_i915_gem_context_param param; }; +struct drm_i915_gem_context_create_ext_clone { +#define I915_CONTEXT_CREATE_EXT_CLONE 1 + struct i915_user_extension base; + __u32 clone_id; + __u32 flags; +#define I915_CONTEXT_CLONE_ENGINES (1u << 0) +#define I915_CONTEXT_CLONE_FLAGS (1u << 1) +#define I915_CONTEXT_CLONE_SCHEDATTR (1u << 2) +#define I915_CONTEXT_CLONE_SSEU (1u << 3) +#define I915_CONTEXT_CLONE_TIMELINE (1u << 4) +#define I915_CONTEXT_CLONE_VM (1u << 5) +#define I915_CONTEXT_CLONE_UNKNOWN -(I915_CONTEXT_CLONE_VM << 1) + __u64 rsvd; +}; + struct drm_i915_gem_context_destroy { __u32 ctx_id; __u32 pad; -- cgit v1.2.3-59-g8ed1b From 6d06779e86724322d79eb53b26989edd9db188f6 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:30 +0100 Subject: drm/i915: Load balancing across a virtual engine Having allowed the user to define a set of engines that they will want to only use, we go one step further and allow them to bind those engines into a single virtual instance. Submitting a batch to the virtual engine will then forward it to any one of the set in a manner as best to distribute load. The virtual engine has a single timeline across all engines (it operates as a single queue), so it is not able to concurrently run batches across multiple engines by itself; that is left up to the user to submit multiple concurrent batches to multiple queues. Multiple users will be load balanced across the system. The mechanism used for load balancing in this patch is a late greedy balancer. When a request is ready for execution, it is added to each engine's queue, and when an engine is ready for its next request it claims it from the virtual engine. The first engine to do so, wins, i.e. the request is executed at the earliest opportunity (idle moment) in the system. As not all HW is created equal, the user is still able to skip the virtual engine and execute the batch on a specific engine, all within the same queue. It will then be executed in order on the correct engine, with execution on other virtual engines being moved away due to the load detection. A couple of areas for potential improvement left! - The virtual engine always take priority over equal-priority tasks. Mostly broken up by applying FQ_CODEL rules for prioritising new clients, and hopefully the virtual and real engines are not then congested (i.e. all work is via virtual engines, or all work is to the real engine). - We require the breadcrumb irq around every virtual engine request. For normal engines, we eliminate the need for the slow round trip via interrupt by using the submit fence and queueing in order. For virtual engines, we have to allow any job to transfer to a new ring, and cannot coalesce the submissions, so require the completion fence instead, forcing the persistent use of interrupts. - We only drip feed single requests through each virtual engine and onto the physical engines, even if there was enough work to fill all ELSP, leaving small stalls with an idle CS event at the end of every request. Could we be greedy and fill both slots? Being lazy is virtuous for load distribution on less-than-full workloads though. Other areas of improvement are more general, such as reducing lock contention, reducing dispatch overhead, looking at direct submission rather than bouncing around tasklets etc. sseu: Lift the restriction to allow sseu to be reconfigured on virtual engines composed of RENDER_CLASS (rcs). v2: macroize check_user_mbz() v3: Cancel virtual engines on wedging v4: Commence commenting v5: Replace 64b sibling_mask with a list of class:instance v6: Drop the one-element array in the uabi v7: Assert it is an virtual engine in to_virtual_engine() v8: Skip over holes in [class][inst] so we can selftest with (vcs0, vcs2) Link: https://github.com/intel/media-driver/pull/283 Signed-off-by: Chris Wilson Cc: Tvrtko Ursulin Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-6-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/gt/intel_engine_types.h | 8 + drivers/gpu/drm/i915/gt/intel_lrc.c | 680 ++++++++++++++++++++++++++- drivers/gpu/drm/i915/gt/intel_lrc.h | 9 + drivers/gpu/drm/i915/gt/selftest_lrc.c | 180 +++++++ drivers/gpu/drm/i915/i915_gem.h | 5 + drivers/gpu/drm/i915/i915_gem_context.c | 116 ++++- drivers/gpu/drm/i915/i915_scheduler.c | 19 +- drivers/gpu/drm/i915/i915_timeline_types.h | 1 + include/uapi/drm/i915_drm.h | 39 ++ 9 files changed, 1029 insertions(+), 28 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index e381c1c73902..7b47e00fa082 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -227,6 +227,7 @@ struct intel_engine_execlists { * @queue: queue of requests, in priority lists */ struct rb_root_cached queue; + struct rb_root_cached virtual; /** * @csb_write: control register for Context Switch buffer @@ -445,6 +446,7 @@ struct intel_engine_cs { #define I915_ENGINE_HAS_PREEMPTION BIT(2) #define I915_ENGINE_HAS_SEMAPHORES BIT(3) #define I915_ENGINE_NEEDS_BREADCRUMB_TASKLET BIT(4) +#define I915_ENGINE_IS_VIRTUAL BIT(5) unsigned int flags; /* @@ -534,6 +536,12 @@ intel_engine_needs_breadcrumb_tasklet(const struct intel_engine_cs *engine) return engine->flags & I915_ENGINE_NEEDS_BREADCRUMB_TASKLET; } +static inline bool +intel_engine_is_virtual(const struct intel_engine_cs *engine) +{ + return engine->flags & I915_ENGINE_IS_VIRTUAL; +} + #define instdone_slice_mask(dev_priv__) \ (IS_GEN(dev_priv__, 7) ? \ 1 : RUNTIME_INFO(dev_priv__)->sseu.slice_mask) diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index f263a8374273..affa5e2dfce1 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -164,6 +164,42 @@ #define WA_TAIL_DWORDS 2 #define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS) +struct virtual_engine { + struct intel_engine_cs base; + struct intel_context context; + + /* + * We allow only a single request through the virtual engine at a time + * (each request in the timeline waits for the completion fence of + * the previous before being submitted). By restricting ourselves to + * only submitting a single request, each request is placed on to a + * physical to maximise load spreading (by virtue of the late greedy + * scheduling -- each real engine takes the next available request + * upon idling). + */ + struct i915_request *request; + + /* + * We keep a rbtree of available virtual engines inside each physical + * engine, sorted by priority. Here we preallocate the nodes we need + * for the virtual engine, indexed by physical_engine->id. + */ + struct ve_node { + struct rb_node rb; + int prio; + } nodes[I915_NUM_ENGINES]; + + /* And finally, which physical engines this virtual engine maps onto. */ + unsigned int num_siblings; + struct intel_engine_cs *siblings[0]; +}; + +static struct virtual_engine *to_virtual_engine(struct intel_engine_cs *engine) +{ + GEM_BUG_ON(!intel_engine_is_virtual(engine)); + return container_of(engine, struct virtual_engine, base); +} + static int execlists_context_deferred_alloc(struct intel_context *ce, struct intel_engine_cs *engine); static void execlists_init_reg_state(u32 *reg_state, @@ -216,7 +252,8 @@ static int queue_prio(const struct intel_engine_execlists *execlists) } static inline bool need_preempt(const struct intel_engine_cs *engine, - const struct i915_request *rq) + const struct i915_request *rq, + struct rb_node *rb) { int last_prio; @@ -251,6 +288,25 @@ static inline bool need_preempt(const struct intel_engine_cs *engine, rq_prio(list_next_entry(rq, link)) > last_prio) return true; + if (rb) { + struct virtual_engine *ve = + rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + bool preempt = false; + + if (engine == ve->siblings[0]) { /* only preempt one sibling */ + struct i915_request *next; + + rcu_read_lock(); + next = READ_ONCE(ve->request); + if (next) + preempt = rq_prio(next) > last_prio; + rcu_read_unlock(); + } + + if (preempt) + return preempt; + } + /* * If the inflight context did not trigger the preemption, then maybe * it was the set of queued requests? Pick the highest priority in @@ -369,6 +425,8 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) list_for_each_entry_safe_reverse(rq, rn, &engine->timeline.requests, link) { + struct intel_engine_cs *owner; + if (i915_request_completed(rq)) break; @@ -377,16 +435,29 @@ __unwind_incomplete_requests(struct intel_engine_cs *engine) GEM_BUG_ON(rq->hw_context->active); - GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID); - if (rq_prio(rq) != prio) { - prio = rq_prio(rq); - pl = i915_sched_lookup_priolist(engine, prio); - } - GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); - - list_add(&rq->sched.link, pl); + /* + * Push the request back into the queue for later resubmission. + * If this request is not native to this physical engine (i.e. + * it came from a virtual source), push it back onto the virtual + * engine so that it can be moved across onto another physical + * engine as load dictates. + */ + owner = rq->hw_context->engine; + if (likely(owner == engine)) { + GEM_BUG_ON(rq_prio(rq) == I915_PRIORITY_INVALID); + if (rq_prio(rq) != prio) { + prio = rq_prio(rq); + pl = i915_sched_lookup_priolist(engine, prio); + } + GEM_BUG_ON(RB_EMPTY_ROOT(&engine->execlists.queue.rb_root)); - active = rq; + list_add(&rq->sched.link, pl); + active = rq; + } else { + rq->engine = owner; + owner->submit_request(rq); + active = NULL; + } } return active; @@ -622,6 +693,90 @@ static void complete_preempt_context(struct intel_engine_execlists *execlists) execlists)); } +static void virtual_update_register_offsets(u32 *regs, + struct intel_engine_cs *engine) +{ + u32 base = engine->mmio_base; + + /* Must match execlists_init_reg_state()! */ + + regs[CTX_CONTEXT_CONTROL] = + i915_mmio_reg_offset(RING_CONTEXT_CONTROL(base)); + regs[CTX_RING_HEAD] = i915_mmio_reg_offset(RING_HEAD(base)); + regs[CTX_RING_TAIL] = i915_mmio_reg_offset(RING_TAIL(base)); + regs[CTX_RING_BUFFER_START] = i915_mmio_reg_offset(RING_START(base)); + regs[CTX_RING_BUFFER_CONTROL] = i915_mmio_reg_offset(RING_CTL(base)); + + regs[CTX_BB_HEAD_U] = i915_mmio_reg_offset(RING_BBADDR_UDW(base)); + regs[CTX_BB_HEAD_L] = i915_mmio_reg_offset(RING_BBADDR(base)); + regs[CTX_BB_STATE] = i915_mmio_reg_offset(RING_BBSTATE(base)); + regs[CTX_SECOND_BB_HEAD_U] = + i915_mmio_reg_offset(RING_SBBADDR_UDW(base)); + regs[CTX_SECOND_BB_HEAD_L] = i915_mmio_reg_offset(RING_SBBADDR(base)); + regs[CTX_SECOND_BB_STATE] = i915_mmio_reg_offset(RING_SBBSTATE(base)); + + regs[CTX_CTX_TIMESTAMP] = + i915_mmio_reg_offset(RING_CTX_TIMESTAMP(base)); + regs[CTX_PDP3_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 3)); + regs[CTX_PDP3_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 3)); + regs[CTX_PDP2_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 2)); + regs[CTX_PDP2_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 2)); + regs[CTX_PDP1_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 1)); + regs[CTX_PDP1_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 1)); + regs[CTX_PDP0_UDW] = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); + regs[CTX_PDP0_LDW] = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); + + if (engine->class == RENDER_CLASS) { + regs[CTX_RCS_INDIRECT_CTX] = + i915_mmio_reg_offset(RING_INDIRECT_CTX(base)); + regs[CTX_RCS_INDIRECT_CTX_OFFSET] = + i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(base)); + regs[CTX_BB_PER_CTX_PTR] = + i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(base)); + + regs[CTX_R_PWR_CLK_STATE] = + i915_mmio_reg_offset(GEN8_R_PWR_CLK_STATE); + } +} + +static bool virtual_matches(const struct virtual_engine *ve, + const struct i915_request *rq, + const struct intel_engine_cs *engine) +{ + const struct intel_engine_cs *active; + + /* + * We track when the HW has completed saving the context image + * (i.e. when we have seen the final CS event switching out of + * the context) and must not overwrite the context image before + * then. This restricts us to only using the active engine + * while the previous virtualized request is inflight (so + * we reuse the register offsets). This is a very small + * hystersis on the greedy seelction algorithm. + */ + active = READ_ONCE(ve->context.active); + if (active && active != engine) + return false; + + return true; +} + +static void virtual_xfer_breadcrumbs(struct virtual_engine *ve, + struct intel_engine_cs *engine) +{ + struct intel_engine_cs *old = ve->siblings[0]; + + /* All unattached (rq->engine == old) must already be completed */ + + spin_lock(&old->breadcrumbs.irq_lock); + if (!list_empty(&ve->context.signal_link)) { + list_move_tail(&ve->context.signal_link, + &engine->breadcrumbs.signalers); + intel_engine_queue_breadcrumbs(engine); + } + spin_unlock(&old->breadcrumbs.irq_lock); +} + static void execlists_dequeue(struct intel_engine_cs *engine) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -654,6 +809,26 @@ static void execlists_dequeue(struct intel_engine_cs *engine) * and context switches) submission. */ + for (rb = rb_first_cached(&execlists->virtual); rb; ) { + struct virtual_engine *ve = + rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + struct i915_request *rq = READ_ONCE(ve->request); + + if (!rq) { /* lazily cleanup after another engine handled rq */ + rb_erase_cached(rb, &execlists->virtual); + RB_CLEAR_NODE(rb); + rb = rb_first_cached(&execlists->virtual); + continue; + } + + if (!virtual_matches(ve, rq, engine)) { + rb = rb_next(rb); + continue; + } + + break; + } + if (last) { /* * Don't resubmit or switch until all outstanding @@ -675,7 +850,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK)) return; - if (need_preempt(engine, last)) { + if (need_preempt(engine, last, rb)) { inject_preempt_context(engine); return; } @@ -715,6 +890,92 @@ static void execlists_dequeue(struct intel_engine_cs *engine) last->tail = last->wa_tail; } + while (rb) { /* XXX virtual is always taking precedence */ + struct virtual_engine *ve = + rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + struct i915_request *rq; + + spin_lock(&ve->base.timeline.lock); + + rq = ve->request; + if (unlikely(!rq)) { /* lost the race to a sibling */ + spin_unlock(&ve->base.timeline.lock); + rb_erase_cached(rb, &execlists->virtual); + RB_CLEAR_NODE(rb); + rb = rb_first_cached(&execlists->virtual); + continue; + } + + GEM_BUG_ON(rq != ve->request); + GEM_BUG_ON(rq->engine != &ve->base); + GEM_BUG_ON(rq->hw_context != &ve->context); + + if (rq_prio(rq) >= queue_prio(execlists)) { + if (!virtual_matches(ve, rq, engine)) { + spin_unlock(&ve->base.timeline.lock); + rb = rb_next(rb); + continue; + } + + if (last && !can_merge_rq(last, rq)) { + spin_unlock(&ve->base.timeline.lock); + return; /* leave this rq for another engine */ + } + + GEM_TRACE("%s: virtual rq=%llx:%lld%s, new engine? %s\n", + engine->name, + rq->fence.context, + rq->fence.seqno, + i915_request_completed(rq) ? "!" : + i915_request_started(rq) ? "*" : + "", + yesno(engine != ve->siblings[0])); + + ve->request = NULL; + ve->base.execlists.queue_priority_hint = INT_MIN; + rb_erase_cached(rb, &execlists->virtual); + RB_CLEAR_NODE(rb); + + rq->engine = engine; + + if (engine != ve->siblings[0]) { + u32 *regs = ve->context.lrc_reg_state; + unsigned int n; + + GEM_BUG_ON(READ_ONCE(ve->context.active)); + virtual_update_register_offsets(regs, engine); + + if (!list_empty(&ve->context.signals)) + virtual_xfer_breadcrumbs(ve, engine); + + /* + * Move the bound engine to the top of the list + * for future execution. We then kick this + * tasklet first before checking others, so that + * we preferentially reuse this set of bound + * registers. + */ + for (n = 1; n < ve->num_siblings; n++) { + if (ve->siblings[n] == engine) { + swap(ve->siblings[n], + ve->siblings[0]); + break; + } + } + + GEM_BUG_ON(ve->siblings[0] != engine); + } + + __i915_request_submit(rq); + trace_i915_request_in(rq, port_index(port, execlists)); + submit = true; + last = rq; + } + + spin_unlock(&ve->base.timeline.lock); + break; + } + while ((rb = rb_first_cached(&execlists->queue))) { struct i915_priolist *p = to_priolist(rb); struct i915_request *rq, *rn; @@ -1838,6 +2099,25 @@ static void reset_csb_pointers(struct intel_engine_execlists *execlists) &execlists->csb_status[reset_value]); } +static struct i915_request *active_request(struct i915_request *rq) +{ + const struct list_head * const list = &rq->engine->timeline.requests; + const struct intel_context * const context = rq->hw_context; + struct i915_request *active = NULL; + + list_for_each_entry_from_reverse(rq, list, link) { + if (i915_request_completed(rq)) + break; + + if (rq->hw_context != context) + break; + + active = rq; + } + + return active; +} + static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) { struct intel_engine_execlists * const execlists = &engine->execlists; @@ -1858,7 +2138,8 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) if (!port_isset(execlists->port)) goto out_clear; - ce = port_request(execlists->port)->hw_context; + rq = port_request(execlists->port); + ce = rq->hw_context; /* * Catch up with any missed context-switch interrupts. @@ -1871,16 +2152,10 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) */ execlists_cancel_port_requests(execlists); - /* Push back any incomplete requests for replay after the reset. */ - rq = __unwind_incomplete_requests(engine); + rq = active_request(rq); if (!rq) goto out_replay; - if (rq->hw_context != ce) { /* caught just before a CS event */ - rq = NULL; - goto out_replay; - } - /* * If this request hasn't started yet, e.g. it is waiting on a * semaphore, we need to avoid skipping the request or else we @@ -1927,13 +2202,16 @@ static void __execlists_reset(struct intel_engine_cs *engine, bool stalled) } execlists_init_reg_state(regs, ce, engine, ce->ring); - /* Rerun the request; its payload has been neutered (if guilty). */ out_replay: + /* Rerun the request; its payload has been neutered (if guilty). */ ce->ring->head = rq ? intel_ring_wrap(ce->ring, rq->head) : ce->ring->tail; intel_ring_update_space(ce->ring); __execlists_update_reg_state(ce, engine); + /* Push back any incomplete requests for replay after the reset. */ + __unwind_incomplete_requests(engine); + out_clear: execlists_clear_all_active(execlists); } @@ -2007,6 +2285,26 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) i915_priolist_free(p); } + /* Cancel all attached virtual engines */ + while ((rb = rb_first_cached(&execlists->virtual))) { + struct virtual_engine *ve = + rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + + rb_erase_cached(rb, &execlists->virtual); + RB_CLEAR_NODE(rb); + + spin_lock(&ve->base.timeline.lock); + if (ve->request) { + ve->request->engine = engine; + __i915_request_submit(ve->request); + dma_fence_set_error(&ve->request->fence, -EIO); + i915_request_mark_complete(ve->request); + ve->base.execlists.queue_priority_hint = INT_MIN; + ve->request = NULL; + } + spin_unlock(&ve->base.timeline.lock); + } + /* Remaining _unready_ requests will be nop'ed when submitted */ execlists->queue_priority_hint = INT_MIN; @@ -2491,12 +2789,15 @@ static void execlists_init_reg_state(u32 *regs, bool rcs = engine->class == RENDER_CLASS; u32 base = engine->mmio_base; - /* A context is actually a big batch buffer with several + /* + * A context is actually a big batch buffer with several * MI_LOAD_REGISTER_IMM commands followed by (reg, value) pairs. The * values we are setting here are only for the first context restore: * on a subsequent save, the GPU will recreate this batchbuffer with new * values (including all the missing MI_LOAD_REGISTER_IMM commands that * we are not initializing here). + * + * Must keep consistent with virtual_update_register_offsets(). */ regs[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(rcs ? 14 : 11) | MI_LRI_FORCE_POSTED; @@ -2715,6 +3016,320 @@ error_deref_obj: return ret; } +static void virtual_context_destroy(struct kref *kref) +{ + struct virtual_engine *ve = + container_of(kref, typeof(*ve), context.ref); + unsigned int n; + + GEM_BUG_ON(ve->request); + GEM_BUG_ON(ve->context.active); + + for (n = 0; n < ve->num_siblings; n++) { + struct intel_engine_cs *sibling = ve->siblings[n]; + struct rb_node *node = &ve->nodes[sibling->id].rb; + + if (RB_EMPTY_NODE(node)) + continue; + + spin_lock_irq(&sibling->timeline.lock); + + /* Detachment is lazily performed in the execlists tasklet */ + if (!RB_EMPTY_NODE(node)) + rb_erase_cached(node, &sibling->execlists.virtual); + + spin_unlock_irq(&sibling->timeline.lock); + } + GEM_BUG_ON(__tasklet_is_scheduled(&ve->base.execlists.tasklet)); + + if (ve->context.state) + __execlists_context_fini(&ve->context); + + i915_timeline_fini(&ve->base.timeline); + kfree(ve); +} + +static void virtual_engine_initial_hint(struct virtual_engine *ve) +{ + int swp; + + /* + * Pick a random sibling on starting to help spread the load around. + * + * New contexts are typically created with exactly the same order + * of siblings, and often started in batches. Due to the way we iterate + * the array of sibling when submitting requests, sibling[0] is + * prioritised for dequeuing. If we make sure that sibling[0] is fairly + * randomised across the system, we also help spread the load by the + * first engine we inspect being different each time. + * + * NB This does not force us to execute on this engine, it will just + * typically be the first we inspect for submission. + */ + swp = prandom_u32_max(ve->num_siblings); + if (!swp) + return; + + swap(ve->siblings[swp], ve->siblings[0]); + virtual_update_register_offsets(ve->context.lrc_reg_state, + ve->siblings[0]); +} + +static int virtual_context_pin(struct intel_context *ce) +{ + struct virtual_engine *ve = container_of(ce, typeof(*ve), context); + int err; + + /* Note: we must use a real engine class for setting up reg state */ + err = __execlists_context_pin(ce, ve->siblings[0]); + if (err) + return err; + + virtual_engine_initial_hint(ve); + return 0; +} + +static void virtual_context_enter(struct intel_context *ce) +{ + struct virtual_engine *ve = container_of(ce, typeof(*ve), context); + unsigned int n; + + for (n = 0; n < ve->num_siblings; n++) + intel_engine_pm_get(ve->siblings[n]); +} + +static void virtual_context_exit(struct intel_context *ce) +{ + struct virtual_engine *ve = container_of(ce, typeof(*ve), context); + unsigned int n; + + ce->saturated = 0; + for (n = 0; n < ve->num_siblings; n++) + intel_engine_pm_put(ve->siblings[n]); +} + +static const struct intel_context_ops virtual_context_ops = { + .pin = virtual_context_pin, + .unpin = execlists_context_unpin, + + .enter = virtual_context_enter, + .exit = virtual_context_exit, + + .destroy = virtual_context_destroy, +}; + +static void virtual_submission_tasklet(unsigned long data) +{ + struct virtual_engine * const ve = (struct virtual_engine *)data; + const int prio = ve->base.execlists.queue_priority_hint; + unsigned int n; + + local_irq_disable(); + for (n = 0; READ_ONCE(ve->request) && n < ve->num_siblings; n++) { + struct intel_engine_cs *sibling = ve->siblings[n]; + struct ve_node * const node = &ve->nodes[sibling->id]; + struct rb_node **parent, *rb; + bool first; + + spin_lock(&sibling->timeline.lock); + + if (!RB_EMPTY_NODE(&node->rb)) { + /* + * Cheat and avoid rebalancing the tree if we can + * reuse this node in situ. + */ + first = rb_first_cached(&sibling->execlists.virtual) == + &node->rb; + if (prio == node->prio || (prio > node->prio && first)) + goto submit_engine; + + rb_erase_cached(&node->rb, &sibling->execlists.virtual); + } + + rb = NULL; + first = true; + parent = &sibling->execlists.virtual.rb_root.rb_node; + while (*parent) { + struct ve_node *other; + + rb = *parent; + other = rb_entry(rb, typeof(*other), rb); + if (prio > other->prio) { + parent = &rb->rb_left; + } else { + parent = &rb->rb_right; + first = false; + } + } + + rb_link_node(&node->rb, rb, parent); + rb_insert_color_cached(&node->rb, + &sibling->execlists.virtual, + first); + +submit_engine: + GEM_BUG_ON(RB_EMPTY_NODE(&node->rb)); + node->prio = prio; + if (first && prio > sibling->execlists.queue_priority_hint) { + sibling->execlists.queue_priority_hint = prio; + tasklet_hi_schedule(&sibling->execlists.tasklet); + } + + spin_unlock(&sibling->timeline.lock); + } + local_irq_enable(); +} + +static void virtual_submit_request(struct i915_request *rq) +{ + struct virtual_engine *ve = to_virtual_engine(rq->engine); + + GEM_TRACE("%s: rq=%llx:%lld\n", + ve->base.name, + rq->fence.context, + rq->fence.seqno); + + GEM_BUG_ON(ve->base.submit_request != virtual_submit_request); + + GEM_BUG_ON(ve->request); + ve->base.execlists.queue_priority_hint = rq_prio(rq); + WRITE_ONCE(ve->request, rq); + + tasklet_schedule(&ve->base.execlists.tasklet); +} + +struct intel_context * +intel_execlists_create_virtual(struct i915_gem_context *ctx, + struct intel_engine_cs **siblings, + unsigned int count) +{ + struct virtual_engine *ve; + unsigned int n; + int err; + + if (count == 0) + return ERR_PTR(-EINVAL); + + if (count == 1) + return intel_context_create(ctx, siblings[0]); + + ve = kzalloc(struct_size(ve, siblings, count), GFP_KERNEL); + if (!ve) + return ERR_PTR(-ENOMEM); + + ve->base.i915 = ctx->i915; + ve->base.id = -1; + ve->base.class = OTHER_CLASS; + ve->base.uabi_class = I915_ENGINE_CLASS_INVALID; + ve->base.instance = I915_ENGINE_CLASS_INVALID_VIRTUAL; + ve->base.flags = I915_ENGINE_IS_VIRTUAL; + + snprintf(ve->base.name, sizeof(ve->base.name), "virtual"); + + err = i915_timeline_init(ctx->i915, &ve->base.timeline, NULL); + if (err) + goto err_put; + i915_timeline_set_subclass(&ve->base.timeline, TIMELINE_VIRTUAL); + + intel_engine_init_execlists(&ve->base); + + ve->base.cops = &virtual_context_ops; + ve->base.request_alloc = execlists_request_alloc; + + ve->base.schedule = i915_schedule; + ve->base.submit_request = virtual_submit_request; + + ve->base.execlists.queue_priority_hint = INT_MIN; + tasklet_init(&ve->base.execlists.tasklet, + virtual_submission_tasklet, + (unsigned long)ve); + + intel_context_init(&ve->context, ctx, &ve->base); + + for (n = 0; n < count; n++) { + struct intel_engine_cs *sibling = siblings[n]; + + GEM_BUG_ON(!is_power_of_2(sibling->mask)); + if (sibling->mask & ve->base.mask) { + DRM_DEBUG("duplicate %s entry in load balancer\n", + sibling->name); + err = -EINVAL; + goto err_put; + } + + /* + * The virtual engine implementation is tightly coupled to + * the execlists backend -- we push out request directly + * into a tree inside each physical engine. We could support + * layering if we handle cloning of the requests and + * submitting a copy into each backend. + */ + if (sibling->execlists.tasklet.func != + execlists_submission_tasklet) { + err = -ENODEV; + goto err_put; + } + + GEM_BUG_ON(RB_EMPTY_NODE(&ve->nodes[sibling->id].rb)); + RB_CLEAR_NODE(&ve->nodes[sibling->id].rb); + + ve->siblings[ve->num_siblings++] = sibling; + ve->base.mask |= sibling->mask; + + /* + * All physical engines must be compatible for their emission + * functions (as we build the instructions during request + * construction and do not alter them before submission + * on the physical engine). We use the engine class as a guide + * here, although that could be refined. + */ + if (ve->base.class != OTHER_CLASS) { + if (ve->base.class != sibling->class) { + DRM_DEBUG("invalid mixing of engine class, sibling %d, already %d\n", + sibling->class, ve->base.class); + err = -EINVAL; + goto err_put; + } + continue; + } + + ve->base.class = sibling->class; + ve->base.uabi_class = sibling->uabi_class; + snprintf(ve->base.name, sizeof(ve->base.name), + "v%dx%d", ve->base.class, count); + ve->base.context_size = sibling->context_size; + + ve->base.emit_bb_start = sibling->emit_bb_start; + ve->base.emit_flush = sibling->emit_flush; + ve->base.emit_init_breadcrumb = sibling->emit_init_breadcrumb; + ve->base.emit_fini_breadcrumb = sibling->emit_fini_breadcrumb; + ve->base.emit_fini_breadcrumb_dw = + sibling->emit_fini_breadcrumb_dw; + } + + return &ve->context; + +err_put: + intel_context_put(&ve->context); + return ERR_PTR(err); +} + +struct intel_context * +intel_execlists_clone_virtual(struct i915_gem_context *ctx, + struct intel_engine_cs *src) +{ + struct virtual_engine *se = to_virtual_engine(src); + struct intel_context *dst; + + dst = intel_execlists_create_virtual(ctx, + se->siblings, + se->num_siblings); + if (IS_ERR(dst)) + return dst; + + return dst; +} + void intel_execlists_show_requests(struct intel_engine_cs *engine, struct drm_printer *m, void (*show_request)(struct drm_printer *m, @@ -2772,6 +3387,29 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, show_request(m, last, "\t\tQ "); } + last = NULL; + count = 0; + for (rb = rb_first_cached(&execlists->virtual); rb; rb = rb_next(rb)) { + struct virtual_engine *ve = + rb_entry(rb, typeof(*ve), nodes[engine->id].rb); + struct i915_request *rq = READ_ONCE(ve->request); + + if (rq) { + if (count++ < max - 1) + show_request(m, rq, "\t\tV "); + else + last = rq; + } + } + if (last) { + if (count > max) { + drm_printf(m, + "\t\t...skipping %d virtual requests...\n", + count - max); + } + show_request(m, last, "\t\tV "); + } + spin_unlock_irqrestore(&engine->timeline.lock, flags); } diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.h b/drivers/gpu/drm/i915/gt/intel_lrc.h index a0dc907a7249..5530606052e5 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc.h @@ -114,4 +114,13 @@ void intel_execlists_show_requests(struct intel_engine_cs *engine, const char *prefix), unsigned int max); +struct intel_context * +intel_execlists_create_virtual(struct i915_gem_context *ctx, + struct intel_engine_cs **siblings, + unsigned int count); + +struct intel_context * +intel_execlists_clone_virtual(struct i915_gem_context *ctx, + struct intel_engine_cs *src); + #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 5b3d8e33f1cf..f880271fb9ba 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -1310,6 +1310,185 @@ err_unlock: return err; } +static int nop_virtual_engine(struct drm_i915_private *i915, + struct intel_engine_cs **siblings, + unsigned int nsibling, + unsigned int nctx, + unsigned int flags) +#define CHAIN BIT(0) +{ + IGT_TIMEOUT(end_time); + struct i915_request *request[16]; + struct i915_gem_context *ctx[16]; + struct intel_context *ve[16]; + unsigned long n, prime, nc; + struct igt_live_test t; + ktime_t times[2] = {}; + int err; + + GEM_BUG_ON(!nctx || nctx > ARRAY_SIZE(ctx)); + + for (n = 0; n < nctx; n++) { + ctx[n] = kernel_context(i915); + if (!ctx[n]) { + err = -ENOMEM; + nctx = n; + goto out; + } + + ve[n] = intel_execlists_create_virtual(ctx[n], + siblings, nsibling); + if (IS_ERR(ve[n])) { + kernel_context_close(ctx[n]); + err = PTR_ERR(ve[n]); + nctx = n; + goto out; + } + + err = intel_context_pin(ve[n]); + if (err) { + intel_context_put(ve[n]); + kernel_context_close(ctx[n]); + nctx = n; + goto out; + } + } + + err = igt_live_test_begin(&t, i915, __func__, ve[0]->engine->name); + if (err) + goto out; + + for_each_prime_number_from(prime, 1, 8192) { + times[1] = ktime_get_raw(); + + if (flags & CHAIN) { + for (nc = 0; nc < nctx; nc++) { + for (n = 0; n < prime; n++) { + request[nc] = + i915_request_create(ve[nc]); + if (IS_ERR(request[nc])) { + err = PTR_ERR(request[nc]); + goto out; + } + + i915_request_add(request[nc]); + } + } + } else { + for (n = 0; n < prime; n++) { + for (nc = 0; nc < nctx; nc++) { + request[nc] = + i915_request_create(ve[nc]); + if (IS_ERR(request[nc])) { + err = PTR_ERR(request[nc]); + goto out; + } + + i915_request_add(request[nc]); + } + } + } + + for (nc = 0; nc < nctx; nc++) { + if (i915_request_wait(request[nc], + I915_WAIT_LOCKED, + HZ / 10) < 0) { + pr_err("%s(%s): wait for %llx:%lld timed out\n", + __func__, ve[0]->engine->name, + request[nc]->fence.context, + request[nc]->fence.seqno); + + GEM_TRACE("%s(%s) failed at request %llx:%lld\n", + __func__, ve[0]->engine->name, + request[nc]->fence.context, + request[nc]->fence.seqno); + GEM_TRACE_DUMP(); + i915_gem_set_wedged(i915); + break; + } + } + + times[1] = ktime_sub(ktime_get_raw(), times[1]); + if (prime == 1) + times[0] = times[1]; + + if (__igt_timeout(end_time, NULL)) + break; + } + + err = igt_live_test_end(&t); + if (err) + goto out; + + pr_info("Requestx%d latencies on %s: 1 = %lluns, %lu = %lluns\n", + nctx, ve[0]->engine->name, ktime_to_ns(times[0]), + prime, div64_u64(ktime_to_ns(times[1]), prime)); + +out: + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + + for (nc = 0; nc < nctx; nc++) { + intel_context_unpin(ve[nc]); + intel_context_put(ve[nc]); + kernel_context_close(ctx[nc]); + } + return err; +} + +static int live_virtual_engine(void *arg) +{ + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; + struct intel_engine_cs *engine; + enum intel_engine_id id; + unsigned int class, inst; + int err = -ENODEV; + + if (USES_GUC_SUBMISSION(i915)) + return 0; + + mutex_lock(&i915->drm.struct_mutex); + + for_each_engine(engine, i915, id) { + err = nop_virtual_engine(i915, &engine, 1, 1, 0); + if (err) { + pr_err("Failed to wrap engine %s: err=%d\n", + engine->name, err); + goto out_unlock; + } + } + + for (class = 0; class <= MAX_ENGINE_CLASS; class++) { + int nsibling, n; + + nsibling = 0; + for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { + if (!i915->engine_class[class][inst]) + continue; + + siblings[nsibling++] = i915->engine_class[class][inst]; + } + if (nsibling < 2) + continue; + + for (n = 1; n <= nsibling + 1; n++) { + err = nop_virtual_engine(i915, siblings, nsibling, + n, 0); + if (err) + goto out_unlock; + } + + err = nop_virtual_engine(i915, siblings, nsibling, n, CHAIN); + if (err) + goto out_unlock; + } + +out_unlock: + mutex_unlock(&i915->drm.struct_mutex); + return err; +} + int intel_execlists_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { @@ -1322,6 +1501,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_chain_preempt), SUBTEST(live_preempt_hang), SUBTEST(live_preempt_smoke), + SUBTEST(live_virtual_engine), }; if (!HAS_EXECLISTS(i915)) diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h index 67f8a4a807a0..fe82d3571072 100644 --- a/drivers/gpu/drm/i915/i915_gem.h +++ b/drivers/gpu/drm/i915/i915_gem.h @@ -91,4 +91,9 @@ static inline bool __tasklet_enable(struct tasklet_struct *t) return atomic_dec_and_test(&t->count); } +static inline bool __tasklet_is_scheduled(struct tasklet_struct *t) +{ + return test_bit(TASKLET_STATE_SCHED, &t->state); +} + #endif /* __I915_GEM_H__ */ diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 24736fcd463d..68db6b28e606 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -86,6 +86,7 @@ */ #include +#include #include @@ -1218,7 +1219,6 @@ __intel_context_reconfigure_sseu(struct intel_context *ce, int ret; GEM_BUG_ON(INTEL_GEN(ce->gem_context->i915) < 8); - GEM_BUG_ON(ce->engine->id != RCS0); ret = intel_context_lock_pinned(ce); if (ret) @@ -1412,7 +1412,100 @@ struct set_engines { struct i915_gem_engines *engines; }; +static int +set_engines__load_balance(struct i915_user_extension __user *base, void *data) +{ + struct i915_context_engines_load_balance __user *ext = + container_of_user(base, typeof(*ext), base); + const struct set_engines *set = data; + struct intel_engine_cs *stack[16]; + struct intel_engine_cs **siblings; + struct intel_context *ce; + u16 num_siblings, idx; + unsigned int n; + int err; + + if (!HAS_EXECLISTS(set->ctx->i915)) + return -ENODEV; + + if (USES_GUC_SUBMISSION(set->ctx->i915)) + return -ENODEV; /* not implement yet */ + + if (get_user(idx, &ext->engine_index)) + return -EFAULT; + + if (idx >= set->engines->num_engines) { + DRM_DEBUG("Invalid placement value, %d >= %d\n", + idx, set->engines->num_engines); + return -EINVAL; + } + + idx = array_index_nospec(idx, set->engines->num_engines); + if (set->engines->engines[idx]) { + DRM_DEBUG("Invalid placement[%d], already occupied\n", idx); + return -EEXIST; + } + + if (get_user(num_siblings, &ext->num_siblings)) + return -EFAULT; + + err = check_user_mbz(&ext->flags); + if (err) + return err; + + err = check_user_mbz(&ext->mbz64); + if (err) + return err; + + siblings = stack; + if (num_siblings > ARRAY_SIZE(stack)) { + siblings = kmalloc_array(num_siblings, + sizeof(*siblings), + GFP_KERNEL); + if (!siblings) + return -ENOMEM; + } + + for (n = 0; n < num_siblings; n++) { + struct i915_engine_class_instance ci; + + if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { + err = -EFAULT; + goto out_siblings; + } + + siblings[n] = intel_engine_lookup_user(set->ctx->i915, + ci.engine_class, + ci.engine_instance); + if (!siblings[n]) { + DRM_DEBUG("Invalid sibling[%d]: { class:%d, inst:%d }\n", + n, ci.engine_class, ci.engine_instance); + err = -EINVAL; + goto out_siblings; + } + } + + ce = intel_execlists_create_virtual(set->ctx, siblings, n); + if (IS_ERR(ce)) { + err = PTR_ERR(ce); + goto out_siblings; + } + + if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { + intel_context_put(ce); + err = -EEXIST; + goto out_siblings; + } + +out_siblings: + if (siblings != stack) + kfree(siblings); + + return err; +} + static const i915_user_extension_fn set_engines__extensions[] = { + [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, }; static int @@ -1737,14 +1830,29 @@ static int clone_engines(struct i915_gem_context *dst, clone->i915 = dst->i915; for (n = 0; n < e->num_engines; n++) { + struct intel_engine_cs *engine; + if (!e->engines[n]) { clone->engines[n] = NULL; continue; } + engine = e->engines[n]->engine; - clone->engines[n] = - intel_context_create(dst, e->engines[n]->engine); - if (!clone->engines[n]) { + /* + * Virtual engines are singletons; they can only exist + * inside a single context, because they embed their + * HW context... As each virtual context implies a single + * timeline (each engine can only dequeue a single request + * at any time), it would be surprising for two contexts + * to use the same engine. So let's create a copy of + * the virtual engine instead. + */ + if (intel_engine_is_virtual(engine)) + clone->engines[n] = + intel_execlists_clone_virtual(dst, engine); + else + clone->engines[n] = intel_context_create(dst, engine); + if (IS_ERR_OR_NULL(clone->engines[n])) { __free_engines(clone, n); goto err_unlock; } diff --git a/drivers/gpu/drm/i915/i915_scheduler.c b/drivers/gpu/drm/i915/i915_scheduler.c index d215dcdf0b1e..78ceb56d7801 100644 --- a/drivers/gpu/drm/i915/i915_scheduler.c +++ b/drivers/gpu/drm/i915/i915_scheduler.c @@ -150,17 +150,26 @@ sched_lock_engine(const struct i915_sched_node *node, struct intel_engine_cs *locked, struct sched_cache *cache) { - struct intel_engine_cs *engine = node_to_request(node)->engine; + const struct i915_request *rq = node_to_request(node); + struct intel_engine_cs *engine; GEM_BUG_ON(!locked); - if (engine != locked) { + /* + * Virtual engines complicate acquiring the engine timeline lock, + * as their rq->engine pointer is not stable until under that + * engine lock. The simple ploy we use is to take the lock then + * check that the rq still belongs to the newly locked engine. + */ + while (locked != (engine = READ_ONCE(rq->engine))) { spin_unlock(&locked->timeline.lock); memset(cache, 0, sizeof(*cache)); spin_lock(&engine->timeline.lock); + locked = engine; } - return engine; + GEM_BUG_ON(locked != engine); + return locked; } static inline int rq_prio(const struct i915_request *rq) @@ -272,6 +281,7 @@ static void __i915_schedule(struct i915_sched_node *node, spin_lock(&engine->timeline.lock); /* Fifo and depth-first replacement ensure our deps execute before us */ + engine = sched_lock_engine(node, engine, &cache); list_for_each_entry_safe_reverse(dep, p, &dfs, dfs_link) { INIT_LIST_HEAD(&dep->dfs_link); @@ -283,8 +293,11 @@ static void __i915_schedule(struct i915_sched_node *node, if (prio <= node->attr.priority || node_signaled(node)) continue; + GEM_BUG_ON(node_to_request(node)->engine != engine); + node->attr.priority = prio; if (!list_empty(&node->link)) { + GEM_BUG_ON(intel_engine_is_virtual(engine)); if (!cache.priolist) cache.priolist = i915_sched_lookup_priolist(engine, diff --git a/drivers/gpu/drm/i915/i915_timeline_types.h b/drivers/gpu/drm/i915/i915_timeline_types.h index 5256a0b5c5f7..1688705f4a2b 100644 --- a/drivers/gpu/drm/i915/i915_timeline_types.h +++ b/drivers/gpu/drm/i915/i915_timeline_types.h @@ -26,6 +26,7 @@ struct i915_timeline { spinlock_t lock; #define TIMELINE_CLIENT 0 /* default subclass */ #define TIMELINE_ENGINE 1 +#define TIMELINE_VIRTUAL 2 struct mutex mutex; /* protects the flow of requests */ unsigned int pin_count; diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 62396d575e28..f9770948161c 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -137,6 +137,7 @@ struct i915_engine_class_instance { __u16 engine_class; /* see enum drm_i915_gem_engine_class */ __u16 engine_instance; #define I915_ENGINE_CLASS_INVALID_NONE -1 +#define I915_ENGINE_CLASS_INVALID_VIRTUAL -2 }; /** @@ -1608,8 +1609,46 @@ struct drm_i915_gem_context_param_sseu { __u32 rsvd; }; +/* + * i915_context_engines_load_balance: + * + * Enable load balancing across this set of engines. + * + * Into the I915_EXEC_DEFAULT slot [0], a virtual engine is created that when + * used will proxy the execbuffer request onto one of the set of engines + * in such a way as to distribute the load evenly across the set. + * + * The set of engines must be compatible (e.g. the same HW class) as they + * will share the same logical GPU context and ring. + * + * To intermix rendering with the virtual engine and direct rendering onto + * the backing engines (bypassing the load balancing proxy), the context must + * be defined to use a single timeline for all engines. + */ +struct i915_context_engines_load_balance { + struct i915_user_extension base; + + __u16 engine_index; + __u16 num_siblings; + __u32 flags; /* all undefined flags must be zero */ + + __u64 mbz64; /* reserved for future use; must be zero */ + + struct i915_engine_class_instance engines[0]; +} __attribute__((packed)); + +#define I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE(name__, N__) struct { \ + struct i915_user_extension base; \ + __u16 engine_index; \ + __u16 num_siblings; \ + __u32 flags; \ + __u64 mbz64; \ + struct i915_engine_class_instance engines[N__]; \ +} __attribute__((packed)) name__ + struct i915_context_param_engines { __u64 extensions; /* linked chain of extension blocks, 0 terminates */ +#define I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE 0 /* see i915_context_engines_load_balance */ struct i915_engine_class_instance engines[0]; } __attribute__((packed)); -- cgit v1.2.3-59-g8ed1b From ee1136908e9b28173f9794be25465a13b2bb9b18 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:33 +0100 Subject: drm/i915/execlists: Virtual engine bonding Some users require that when a master batch is executed on one particular engine, a companion batch is run simultaneously on a specific slave engine. For this purpose, we introduce virtual engine bonding, allowing maps of master:slaves to be constructed to constrain which physical engines a virtual engine may select given a fence on a master engine. For the moment, we continue to ignore the issue of preemption deferring the master request for later. Ideally, we would like to then also remove the slave and run something else rather than have it stall the pipeline. With load balancing, we should be able to move workload around it, but there is a similar stall on the master pipeline while it may wait for the slave to be executed. At the cost of more latency for the bonded request, it may be interesting to launch both on their engines in lockstep. (Bubbles abound.) Opens: Also what about bonding an engine as its own master? It doesn't break anything internally, so allow the silliness. v2: Emancipate the bonds v3: Couple in delayed scheduling for the selftests v4: Handle invalid mutually exclusive bonding v5: Mention what the uapi does v6: s/nbond/num_bonds/ Signed-off-by: Chris Wilson Cc: Tvrtko Ursulin Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-9-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/gt/intel_engine_types.h | 7 + drivers/gpu/drm/i915/gt/intel_lrc.c | 98 +++++++++++++ drivers/gpu/drm/i915/gt/intel_lrc.h | 4 + drivers/gpu/drm/i915/gt/selftest_lrc.c | 191 ++++++++++++++++++++++++++ drivers/gpu/drm/i915/i915_gem_context.c | 84 +++++++++++ drivers/gpu/drm/i915/selftests/lib_sw_fence.c | 3 + include/uapi/drm/i915_drm.h | 44 ++++++ 7 files changed, 431 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index 7b47e00fa082..f3fc2e8acc90 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -405,6 +405,13 @@ struct intel_engine_cs { */ void (*submit_request)(struct i915_request *rq); + /* + * Called on signaling of a SUBMIT_FENCE, passing along the signaling + * request down to the bonded pairs. + */ + void (*bond_execute)(struct i915_request *rq, + struct dma_fence *signal); + /* * Call when the priority on a request has changed and it and its * dependencies may need rescheduling. Note the request itself may diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index 0b4d29d4816b..1f7bee0cae0c 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -189,6 +189,18 @@ struct virtual_engine { int prio; } nodes[I915_NUM_ENGINES]; + /* + * Keep track of bonded pairs -- restrictions upon on our selection + * of physical engines any particular request may be submitted to. + * If we receive a submit-fence from a master engine, we will only + * use one of sibling_mask physical engines. + */ + struct ve_bond { + const struct intel_engine_cs *master; + intel_engine_mask_t sibling_mask; + } *bonds; + unsigned int num_bonds; + /* And finally, which physical engines this virtual engine maps onto. */ unsigned int num_siblings; struct intel_engine_cs *siblings[0]; @@ -960,6 +972,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine) rb_erase_cached(rb, &execlists->virtual); RB_CLEAR_NODE(rb); + GEM_BUG_ON(!(rq->execution_mask & engine->mask)); rq->engine = engine; if (engine != ve->siblings[0]) { @@ -3069,6 +3082,8 @@ static void virtual_context_destroy(struct kref *kref) if (ve->context.state) __execlists_context_fini(&ve->context); + kfree(ve->bonds); + i915_timeline_fini(&ve->base.timeline); kfree(ve); } @@ -3265,6 +3280,38 @@ static void virtual_submit_request(struct i915_request *rq) tasklet_schedule(&ve->base.execlists.tasklet); } +static struct ve_bond * +virtual_find_bond(struct virtual_engine *ve, + const struct intel_engine_cs *master) +{ + int i; + + for (i = 0; i < ve->num_bonds; i++) { + if (ve->bonds[i].master == master) + return &ve->bonds[i]; + } + + return NULL; +} + +static void +virtual_bond_execute(struct i915_request *rq, struct dma_fence *signal) +{ + struct virtual_engine *ve = to_virtual_engine(rq->engine); + struct ve_bond *bond; + + bond = virtual_find_bond(ve, to_request(signal)->engine); + if (bond) { + intel_engine_mask_t old, new, cmp; + + cmp = READ_ONCE(rq->execution_mask); + do { + old = cmp; + new = cmp & bond->sibling_mask; + } while ((cmp = cmpxchg(&rq->execution_mask, old, new)) != old); + } +} + struct intel_context * intel_execlists_create_virtual(struct i915_gem_context *ctx, struct intel_engine_cs **siblings, @@ -3305,6 +3352,7 @@ intel_execlists_create_virtual(struct i915_gem_context *ctx, ve->base.schedule = i915_schedule; ve->base.submit_request = virtual_submit_request; + ve->base.bond_execute = virtual_bond_execute; ve->base.execlists.queue_priority_hint = INT_MIN; tasklet_init(&ve->base.execlists.tasklet, @@ -3394,9 +3442,59 @@ intel_execlists_clone_virtual(struct i915_gem_context *ctx, if (IS_ERR(dst)) return dst; + if (se->num_bonds) { + struct virtual_engine *de = to_virtual_engine(dst->engine); + + de->bonds = kmemdup(se->bonds, + sizeof(*se->bonds) * se->num_bonds, + GFP_KERNEL); + if (!de->bonds) { + intel_context_put(dst); + return ERR_PTR(-ENOMEM); + } + + de->num_bonds = se->num_bonds; + } + return dst; } +int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, + const struct intel_engine_cs *master, + const struct intel_engine_cs *sibling) +{ + struct virtual_engine *ve = to_virtual_engine(engine); + struct ve_bond *bond; + int n; + + /* Sanity check the sibling is part of the virtual engine */ + for (n = 0; n < ve->num_siblings; n++) + if (sibling == ve->siblings[n]) + break; + if (n == ve->num_siblings) + return -EINVAL; + + bond = virtual_find_bond(ve, master); + if (bond) { + bond->sibling_mask |= sibling->mask; + return 0; + } + + bond = krealloc(ve->bonds, + sizeof(*bond) * (ve->num_bonds + 1), + GFP_KERNEL); + if (!bond) + return -ENOMEM; + + bond[ve->num_bonds].master = master; + bond[ve->num_bonds].sibling_mask = sibling->mask; + + ve->bonds = bond; + ve->num_bonds++; + + return 0; +} + void intel_execlists_show_requests(struct intel_engine_cs *engine, struct drm_printer *m, void (*show_request)(struct drm_printer *m, diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.h b/drivers/gpu/drm/i915/gt/intel_lrc.h index 5530606052e5..e029aee87adf 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.h +++ b/drivers/gpu/drm/i915/gt/intel_lrc.h @@ -123,4 +123,8 @@ struct intel_context * intel_execlists_clone_virtual(struct i915_gem_context *ctx, struct intel_engine_cs *src); +int intel_virtual_engine_attach_bond(struct intel_engine_cs *engine, + const struct intel_engine_cs *master, + const struct intel_engine_cs *sibling); + #endif /* _INTEL_LRC_H_ */ diff --git a/drivers/gpu/drm/i915/gt/selftest_lrc.c b/drivers/gpu/drm/i915/gt/selftest_lrc.c index 61637f525690..a8c50900e2d4 100644 --- a/drivers/gpu/drm/i915/gt/selftest_lrc.c +++ b/drivers/gpu/drm/i915/gt/selftest_lrc.c @@ -13,6 +13,7 @@ #include "selftests/igt_gem_utils.h" #include "selftests/igt_live_test.h" #include "selftests/igt_spinner.h" +#include "selftests/lib_sw_fence.h" #include "selftests/mock_context.h" static int live_sanitycheck(void *arg) @@ -1619,6 +1620,195 @@ out_unlock: return err; } +static int bond_virtual_engine(struct drm_i915_private *i915, + unsigned int class, + struct intel_engine_cs **siblings, + unsigned int nsibling, + unsigned int flags) +#define BOND_SCHEDULE BIT(0) +{ + struct intel_engine_cs *master; + struct i915_gem_context *ctx; + struct i915_request *rq[16]; + enum intel_engine_id id; + unsigned long n; + int err; + + GEM_BUG_ON(nsibling >= ARRAY_SIZE(rq) - 1); + + ctx = kernel_context(i915); + if (!ctx) + return -ENOMEM; + + err = 0; + rq[0] = ERR_PTR(-ENOMEM); + for_each_engine(master, i915, id) { + struct i915_sw_fence fence = {}; + + if (master->class == class) + continue; + + memset_p((void *)rq, ERR_PTR(-EINVAL), ARRAY_SIZE(rq)); + + rq[0] = igt_request_alloc(ctx, master); + if (IS_ERR(rq[0])) { + err = PTR_ERR(rq[0]); + goto out; + } + i915_request_get(rq[0]); + + if (flags & BOND_SCHEDULE) { + onstack_fence_init(&fence); + err = i915_sw_fence_await_sw_fence_gfp(&rq[0]->submit, + &fence, + GFP_KERNEL); + } + i915_request_add(rq[0]); + if (err < 0) + goto out; + + for (n = 0; n < nsibling; n++) { + struct intel_context *ve; + + ve = intel_execlists_create_virtual(ctx, + siblings, + nsibling); + if (IS_ERR(ve)) { + err = PTR_ERR(ve); + onstack_fence_fini(&fence); + goto out; + } + + err = intel_virtual_engine_attach_bond(ve->engine, + master, + siblings[n]); + if (err) { + intel_context_put(ve); + onstack_fence_fini(&fence); + goto out; + } + + err = intel_context_pin(ve); + intel_context_put(ve); + if (err) { + onstack_fence_fini(&fence); + goto out; + } + + rq[n + 1] = i915_request_create(ve); + intel_context_unpin(ve); + if (IS_ERR(rq[n + 1])) { + err = PTR_ERR(rq[n + 1]); + onstack_fence_fini(&fence); + goto out; + } + i915_request_get(rq[n + 1]); + + err = i915_request_await_execution(rq[n + 1], + &rq[0]->fence, + ve->engine->bond_execute); + i915_request_add(rq[n + 1]); + if (err < 0) { + onstack_fence_fini(&fence); + goto out; + } + } + onstack_fence_fini(&fence); + + if (i915_request_wait(rq[0], + I915_WAIT_LOCKED, + HZ / 10) < 0) { + pr_err("Master request did not execute (on %s)!\n", + rq[0]->engine->name); + err = -EIO; + goto out; + } + + for (n = 0; n < nsibling; n++) { + if (i915_request_wait(rq[n + 1], + I915_WAIT_LOCKED, + MAX_SCHEDULE_TIMEOUT) < 0) { + err = -EIO; + goto out; + } + + if (rq[n + 1]->engine != siblings[n]) { + pr_err("Bonded request did not execute on target engine: expected %s, used %s; master was %s\n", + siblings[n]->name, + rq[n + 1]->engine->name, + rq[0]->engine->name); + err = -EINVAL; + goto out; + } + } + + for (n = 0; !IS_ERR(rq[n]); n++) + i915_request_put(rq[n]); + rq[0] = ERR_PTR(-ENOMEM); + } + +out: + for (n = 0; !IS_ERR(rq[n]); n++) + i915_request_put(rq[n]); + if (igt_flush_test(i915, I915_WAIT_LOCKED)) + err = -EIO; + + kernel_context_close(ctx); + return err; +} + +static int live_virtual_bond(void *arg) +{ + static const struct phase { + const char *name; + unsigned int flags; + } phases[] = { + { "", 0 }, + { "schedule", BOND_SCHEDULE }, + { }, + }; + struct drm_i915_private *i915 = arg; + struct intel_engine_cs *siblings[MAX_ENGINE_INSTANCE + 1]; + unsigned int class, inst; + int err = 0; + + if (USES_GUC_SUBMISSION(i915)) + return 0; + + mutex_lock(&i915->drm.struct_mutex); + + for (class = 0; class <= MAX_ENGINE_CLASS; class++) { + const struct phase *p; + int nsibling; + + nsibling = 0; + for (inst = 0; inst <= MAX_ENGINE_INSTANCE; inst++) { + if (!i915->engine_class[class][inst]) + break; + + GEM_BUG_ON(nsibling == ARRAY_SIZE(siblings)); + siblings[nsibling++] = i915->engine_class[class][inst]; + } + if (nsibling < 2) + continue; + + for (p = phases; p->name; p++) { + err = bond_virtual_engine(i915, + class, siblings, nsibling, + p->flags); + if (err) { + pr_err("%s(%s): failed class=%d, nsibling=%d, err=%d\n", + __func__, p->name, class, nsibling, err); + goto out_unlock; + } + } + } + +out_unlock: + mutex_unlock(&i915->drm.struct_mutex); + return err; +} + int intel_execlists_live_selftests(struct drm_i915_private *i915) { static const struct i915_subtest tests[] = { @@ -1633,6 +1823,7 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915) SUBTEST(live_preempt_smoke), SUBTEST(live_virtual_engine), SUBTEST(live_virtual_mask), + SUBTEST(live_virtual_bond), }; if (!HAS_EXECLISTS(i915)) diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c index 68db6b28e606..5d2f8ba92b59 100644 --- a/drivers/gpu/drm/i915/i915_gem_context.c +++ b/drivers/gpu/drm/i915/i915_gem_context.c @@ -1504,8 +1504,92 @@ out_siblings: return err; } +static int +set_engines__bond(struct i915_user_extension __user *base, void *data) +{ + struct i915_context_engines_bond __user *ext = + container_of_user(base, typeof(*ext), base); + const struct set_engines *set = data; + struct i915_engine_class_instance ci; + struct intel_engine_cs *virtual; + struct intel_engine_cs *master; + u16 idx, num_bonds; + int err, n; + + if (get_user(idx, &ext->virtual_index)) + return -EFAULT; + + if (idx >= set->engines->num_engines) { + DRM_DEBUG("Invalid index for virtual engine: %d >= %d\n", + idx, set->engines->num_engines); + return -EINVAL; + } + + idx = array_index_nospec(idx, set->engines->num_engines); + if (!set->engines->engines[idx]) { + DRM_DEBUG("Invalid engine at %d\n", idx); + return -EINVAL; + } + virtual = set->engines->engines[idx]->engine; + + err = check_user_mbz(&ext->flags); + if (err) + return err; + + for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { + err = check_user_mbz(&ext->mbz64[n]); + if (err) + return err; + } + + if (copy_from_user(&ci, &ext->master, sizeof(ci))) + return -EFAULT; + + master = intel_engine_lookup_user(set->ctx->i915, + ci.engine_class, ci.engine_instance); + if (!master) { + DRM_DEBUG("Unrecognised master engine: { class:%u, instance:%u }\n", + ci.engine_class, ci.engine_instance); + return -EINVAL; + } + + if (get_user(num_bonds, &ext->num_bonds)) + return -EFAULT; + + for (n = 0; n < num_bonds; n++) { + struct intel_engine_cs *bond; + + if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) + return -EFAULT; + + bond = intel_engine_lookup_user(set->ctx->i915, + ci.engine_class, + ci.engine_instance); + if (!bond) { + DRM_DEBUG("Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", + n, ci.engine_class, ci.engine_instance); + return -EINVAL; + } + + /* + * A non-virtual engine has no siblings to choose between; and + * a submit fence will always be directed to the one engine. + */ + if (intel_engine_is_virtual(virtual)) { + err = intel_virtual_engine_attach_bond(virtual, + master, + bond); + if (err) + return err; + } + } + + return 0; +} + static const i915_user_extension_fn set_engines__extensions[] = { [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, + [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, }; static int diff --git a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c index 2bfa72c1654b..b976c12817c5 100644 --- a/drivers/gpu/drm/i915/selftests/lib_sw_fence.c +++ b/drivers/gpu/drm/i915/selftests/lib_sw_fence.c @@ -45,6 +45,9 @@ void __onstack_fence_init(struct i915_sw_fence *fence, void onstack_fence_fini(struct i915_sw_fence *fence) { + if (!fence->flags) + return; + i915_sw_fence_commit(fence); i915_sw_fence_fini(fence); } diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index f9770948161c..e2da9027bcdf 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -1543,6 +1543,10 @@ struct drm_i915_gem_context_param { * sized argument, will revert back to default settings. * * See struct i915_context_param_engines. + * + * Extensions: + * i915_context_engines_load_balance (I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE) + * i915_context_engines_bond (I915_CONTEXT_ENGINES_EXT_BOND) */ #define I915_CONTEXT_PARAM_ENGINES 0xa /* Must be kept compact -- no holes and well documented */ @@ -1646,9 +1650,49 @@ struct i915_context_engines_load_balance { struct i915_engine_class_instance engines[N__]; \ } __attribute__((packed)) name__ +/* + * i915_context_engines_bond: + * + * Constructed bonded pairs for execution within a virtual engine. + * + * All engines are equal, but some are more equal than others. Given + * the distribution of resources in the HW, it may be preferable to run + * a request on a given subset of engines in parallel to a request on a + * specific engine. We enable this selection of engines within a virtual + * engine by specifying bonding pairs, for any given master engine we will + * only execute on one of the corresponding siblings within the virtual engine. + * + * To execute a request in parallel on the master engine and a sibling requires + * coordination with a I915_EXEC_FENCE_SUBMIT. + */ +struct i915_context_engines_bond { + struct i915_user_extension base; + + struct i915_engine_class_instance master; + + __u16 virtual_index; /* index of virtual engine in ctx->engines[] */ + __u16 num_bonds; + + __u64 flags; /* all undefined flags must be zero */ + __u64 mbz64[4]; /* reserved for future use; must be zero */ + + struct i915_engine_class_instance engines[0]; +} __attribute__((packed)); + +#define I915_DEFINE_CONTEXT_ENGINES_BOND(name__, N__) struct { \ + struct i915_user_extension base; \ + struct i915_engine_class_instance master; \ + __u16 virtual_index; \ + __u16 num_bonds; \ + __u64 flags; \ + __u64 mbz64[4]; \ + struct i915_engine_class_instance engines[N__]; \ +} __attribute__((packed)) name__ + struct i915_context_param_engines { __u64 extensions; /* linked chain of extension blocks, 0 terminates */ #define I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE 0 /* see i915_context_engines_load_balance */ +#define I915_CONTEXT_ENGINES_EXT_BOND 1 /* see i915_context_engines_bond */ struct i915_engine_class_instance engines[0]; } __attribute__((packed)); -- cgit v1.2.3-59-g8ed1b From a88b6e4cbafd6f23b3450c087acdbe23d90e7606 Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Tue, 21 May 2019 22:11:34 +0100 Subject: drm/i915: Allow specification of parallel execbuf There is a desire to split a task onto two engines and have them run at the same time, e.g. scanline interleaving to spread the workload evenly. Through the use of the out-fence from the first execbuf, we can coordinate secondary execbuf to only become ready simultaneously with the first, so that with all things idle the second execbufs are executed in parallel with the first. The key difference here between the new EXEC_FENCE_SUBMIT and the existing EXEC_FENCE_IN is that the in-fence waits for the completion of the first request (so that all of its rendering results are visible to the second execbuf, the more common userspace fence requirement). Since we only have a single input fence slot, userspace cannot mix an in-fence and a submit-fence. It has to use one or the other! This is not such a harsh requirement, since by virtue of the submit-fence, the secondary execbuf inherit all of the dependencies from the first request, and for the application the dependencies should be common between the primary and secondary execbuf. Suggested-by: Tvrtko Ursulin Testcase: igt/gem_exec_fence/parallel Link: https://github.com/intel/media-driver/pull/546 Signed-off-by: Chris Wilson Cc: Tvrtko Ursulin Reviewed-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20190521211134.16117-10-chris@chris-wilson.co.uk --- drivers/gpu/drm/i915/i915_drv.c | 1 + drivers/gpu/drm/i915/i915_gem_execbuffer.c | 25 ++++++++++++++++++++++++- include/uapi/drm/i915_drm.h | 17 ++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 5061cb32856b..83d2eb9e74cb 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -443,6 +443,7 @@ static int i915_getparam_ioctl(struct drm_device *dev, void *data, case I915_PARAM_HAS_EXEC_CAPTURE: case I915_PARAM_HAS_EXEC_BATCH_FIRST: case I915_PARAM_HAS_EXEC_FENCE_ARRAY: + case I915_PARAM_HAS_EXEC_SUBMIT_FENCE: /* For the time being all of these are always true; * if some supported hardware does not have one of these * features this value needs to be provided from diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c index d6c5220addd0..7ce25b54c57b 100644 --- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c @@ -2318,6 +2318,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, { struct i915_execbuffer eb; struct dma_fence *in_fence = NULL; + struct dma_fence *exec_fence = NULL; struct sync_file *out_fence = NULL; int out_fence_fd = -1; int err; @@ -2360,11 +2361,24 @@ i915_gem_do_execbuffer(struct drm_device *dev, return -EINVAL; } + if (args->flags & I915_EXEC_FENCE_SUBMIT) { + if (in_fence) { + err = -EINVAL; + goto err_in_fence; + } + + exec_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); + if (!exec_fence) { + err = -EINVAL; + goto err_in_fence; + } + } + if (args->flags & I915_EXEC_FENCE_OUT) { out_fence_fd = get_unused_fd_flags(O_CLOEXEC); if (out_fence_fd < 0) { err = out_fence_fd; - goto err_in_fence; + goto err_exec_fence; } } @@ -2494,6 +2508,13 @@ i915_gem_do_execbuffer(struct drm_device *dev, goto err_request; } + if (exec_fence) { + err = i915_request_await_execution(eb.request, exec_fence, + eb.engine->bond_execute); + if (err < 0) + goto err_request; + } + if (fences) { err = await_fence_array(&eb, fences); if (err) @@ -2555,6 +2576,8 @@ err_destroy: err_out_fence: if (out_fence_fd != -1) put_unused_fd(out_fence_fd); +err_exec_fence: + dma_fence_put(exec_fence); err_in_fence: dma_fence_put(in_fence); return err; diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index e2da9027bcdf..bdb00ec1f8be 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -604,6 +604,12 @@ typedef struct drm_i915_irq_wait { */ #define I915_PARAM_MMAP_GTT_COHERENT 52 +/* + * Query whether DRM_I915_GEM_EXECBUFFER2 supports coordination of parallel + * execution through use of explicit fence support. + * See I915_EXEC_FENCE_OUT and I915_EXEC_FENCE_SUBMIT. + */ +#define I915_PARAM_HAS_EXEC_SUBMIT_FENCE 53 /* Must be kept compact -- no holes and well documented */ typedef struct drm_i915_getparam { @@ -1126,7 +1132,16 @@ struct drm_i915_gem_execbuffer2 { */ #define I915_EXEC_FENCE_ARRAY (1<<19) -#define __I915_EXEC_UNKNOWN_FLAGS (-(I915_EXEC_FENCE_ARRAY<<1)) +/* + * Setting I915_EXEC_FENCE_SUBMIT implies that lower_32_bits(rsvd2) represent + * a sync_file fd to wait upon (in a nonblocking manner) prior to executing + * the batch. + * + * Returns -EINVAL if the sync_file fd cannot be found. + */ +#define I915_EXEC_FENCE_SUBMIT (1 << 20) + +#define __I915_EXEC_UNKNOWN_FLAGS (-(I915_EXEC_FENCE_SUBMIT << 1)) #define I915_EXEC_CONTEXT_ID_MASK (0xffffffff) #define i915_execbuffer2_set_context_id(eb2, context) \ -- cgit v1.2.3-59-g8ed1b From c5d3e39caa456b1e061644b739131f2b54c84c08 Mon Sep 17 00:00:00 2001 From: Tvrtko Ursulin Date: Wed, 22 May 2019 10:00:54 +0100 Subject: drm/i915: Engine discovery query Engine discovery query allows userspace to enumerate engines, probe their configuration features, all without needing to maintain the internal PCI ID based database. A new query for the generic i915 query ioctl is added named DRM_I915_QUERY_ENGINE_INFO, together with accompanying structure drm_i915_query_engine_info. The address of latter should be passed to the kernel in the query.data_ptr field, and should be large enough for the kernel to fill out all known engines as struct drm_i915_engine_info elements trailing the query. As with other queries, setting the item query length to zero allows userspace to query minimum required buffer size. Enumerated engines have common type mask which can be used to query all hardware engines, versus engines userspace can submit to using the execbuf uAPI. Engines also have capabilities which are per engine class namespace of bits describing features not present on all engine instances. v2: * Fixed HEVC assignment. * Reorder some fields, rename type to flags, increase width. (Lionel) * No need to allocate temporary storage if we do it engine by engine. (Lionel) v3: * Describe engine flags and mark mbz fields. (Lionel) * HEVC only applies to VCS. v4: * Squash SFC flag into main patch. * Tidy some comments. v5: * Add uabi_ prefix to engine capabilities. (Chris Wilson) * Report exact size of engine info array. (Chris Wilson) * Drop the engine flags. (Joonas Lahtinen) * Added some more reserved fields. * Move flags after class/instance. v6: * Do not check engine info array was zeroed by userspace but zero the unused fields for them instead. v7: * Simplify length calculation loop. (Lionel Landwerlin) v8: * Remove MBZ comments where not applicable. * Rename ABI flags to match engine class define naming. * Rename SFC ABI flag to reflect it applies to VCS and VECS. * SFC is wired to even _logical_ engine instances. * SFC applies to VCS and VECS. * HEVC is present on all instances on Gen11. (Tony) * Simplify length calculation even more. (Chris Wilson) * Move info_ptr assigment closer to loop for clarity. (Chris Wilson) * Use vdbox_sfc_access from runtime info. * Rebase for RUNTIME_INFO. * Refactor for lower indentation. * Rename uAPI class/instance to engine_class/instance to avoid C++ keyword. v9: * Rebase for s/num_rings/num_engines/ in RUNTIME_INFO. v10: * Use new copy_query_item. v11: * Consolidate with struct i915_engine_class_instnace. Signed-off-by: Tvrtko Ursulin Cc: Chris Wilson Cc: Jon Bloomfield Cc: Dmitry Rogozhkin Cc: Lionel Landwerlin Cc: Joonas Lahtinen Cc: Tony Ye Reviewed-by: Lionel Landwerlin # v7 Reviewed-by: Chris Wilson Link: https://patchwork.freedesktop.org/patch/msgid/20190522090054.6007-1-tvrtko.ursulin@linux.intel.com --- drivers/gpu/drm/i915/gt/intel_engine_cs.c | 41 +++++++++++++++++++++++ drivers/gpu/drm/i915/gt/intel_engine_types.h | 2 ++ drivers/gpu/drm/i915/i915_query.c | 49 ++++++++++++++++++++++++++++ include/uapi/drm/i915_drm.h | 42 ++++++++++++++++++++++++ 4 files changed, 134 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/i915/gt/intel_engine_cs.c b/drivers/gpu/drm/i915/gt/intel_engine_cs.c index 4c3753c1b573..2590f5904b67 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_cs.c +++ b/drivers/gpu/drm/i915/gt/intel_engine_cs.c @@ -349,6 +349,45 @@ intel_engine_setup(struct drm_i915_private *dev_priv, return 0; } +static void __setup_engine_capabilities(struct intel_engine_cs *engine) +{ + struct drm_i915_private *i915 = engine->i915; + + if (engine->class == VIDEO_DECODE_CLASS) { + /* + * HEVC support is present on first engine instance + * before Gen11 and on all instances afterwards. + */ + if (INTEL_GEN(i915) >= 11 || + (INTEL_GEN(i915) >= 9 && engine->instance == 0)) + engine->uabi_capabilities |= + I915_VIDEO_CLASS_CAPABILITY_HEVC; + + /* + * SFC block is present only on even logical engine + * instances. + */ + if ((INTEL_GEN(i915) >= 11 && + RUNTIME_INFO(i915)->vdbox_sfc_access & engine->mask) || + (INTEL_GEN(i915) >= 9 && engine->instance == 0)) + engine->uabi_capabilities |= + I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; + } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) { + if (INTEL_GEN(i915) >= 9) + engine->uabi_capabilities |= + I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC; + } +} + +static void intel_setup_engine_capabilities(struct drm_i915_private *i915) +{ + struct intel_engine_cs *engine; + enum intel_engine_id id; + + for_each_engine(engine, i915, id) + __setup_engine_capabilities(engine); +} + /** * intel_engines_cleanup() - free the resources allocated for Command Streamers * @i915: the i915 devic @@ -414,6 +453,8 @@ int intel_engines_init_mmio(struct drm_i915_private *i915) i915_check_and_clear_faults(i915); + intel_setup_engine_capabilities(i915); + return 0; cleanup: diff --git a/drivers/gpu/drm/i915/gt/intel_engine_types.h b/drivers/gpu/drm/i915/gt/intel_engine_types.h index f3fc2e8acc90..40e774acc2cd 100644 --- a/drivers/gpu/drm/i915/gt/intel_engine_types.h +++ b/drivers/gpu/drm/i915/gt/intel_engine_types.h @@ -280,6 +280,8 @@ struct intel_engine_cs { u32 context_size; u32 mmio_base; + u32 uabi_capabilities; + struct intel_sseu sseu; struct intel_ring *buffer; diff --git a/drivers/gpu/drm/i915/i915_query.c b/drivers/gpu/drm/i915/i915_query.c index 782183b78f49..414d0a6d1f70 100644 --- a/drivers/gpu/drm/i915/i915_query.c +++ b/drivers/gpu/drm/i915/i915_query.c @@ -96,9 +96,58 @@ static int query_topology_info(struct drm_i915_private *dev_priv, return total_length; } +static int +query_engine_info(struct drm_i915_private *i915, + struct drm_i915_query_item *query_item) +{ + struct drm_i915_query_engine_info __user *query_ptr = + u64_to_user_ptr(query_item->data_ptr); + struct drm_i915_engine_info __user *info_ptr; + struct drm_i915_query_engine_info query; + struct drm_i915_engine_info info = { }; + struct intel_engine_cs *engine; + enum intel_engine_id id; + int len, ret; + + if (query_item->flags) + return -EINVAL; + + len = sizeof(struct drm_i915_query_engine_info) + + RUNTIME_INFO(i915)->num_engines * + sizeof(struct drm_i915_engine_info); + + ret = copy_query_item(&query, sizeof(query), len, query_item); + if (ret != 0) + return ret; + + if (query.num_engines || query.rsvd[0] || query.rsvd[1] || + query.rsvd[2]) + return -EINVAL; + + info_ptr = &query_ptr->engines[0]; + + for_each_engine(engine, i915, id) { + info.engine.engine_class = engine->uabi_class; + info.engine.engine_instance = engine->instance; + info.capabilities = engine->uabi_capabilities; + + if (__copy_to_user(info_ptr, &info, sizeof(info))) + return -EFAULT; + + query.num_engines++; + info_ptr++; + } + + if (__copy_to_user(query_ptr, &query, sizeof(query))) + return -EFAULT; + + return len; +} + static int (* const i915_query_funcs[])(struct drm_i915_private *dev_priv, struct drm_i915_query_item *query_item) = { query_topology_info, + query_engine_info, }; int i915_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index bdb00ec1f8be..328d05e77d9f 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -1982,6 +1982,7 @@ struct drm_i915_perf_oa_config { struct drm_i915_query_item { __u64 query_id; #define DRM_I915_QUERY_TOPOLOGY_INFO 1 +#define DRM_I915_QUERY_ENGINE_INFO 2 /* Must be kept compact -- no holes and well documented */ /* @@ -2080,6 +2081,47 @@ struct drm_i915_query_topology_info { __u8 data[]; }; +/** + * struct drm_i915_engine_info + * + * Describes one engine and it's capabilities as known to the driver. + */ +struct drm_i915_engine_info { + /** Engine class and instance. */ + struct i915_engine_class_instance engine; + + /** Reserved field. */ + __u32 rsvd0; + + /** Engine flags. */ + __u64 flags; + + /** Capabilities of this engine. */ + __u64 capabilities; +#define I915_VIDEO_CLASS_CAPABILITY_HEVC (1 << 0) +#define I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC (1 << 1) + + /** Reserved fields. */ + __u64 rsvd1[4]; +}; + +/** + * struct drm_i915_query_engine_info + * + * Engine info query enumerates all engines known to the driver by filling in + * an array of struct drm_i915_engine_info structures. + */ +struct drm_i915_query_engine_info { + /** Number of struct drm_i915_engine_info structs following. */ + __u32 num_engines; + + /** MBZ */ + __u32 rsvd[3]; + + /** Marker for drm_i915_engine_info structures. */ + struct drm_i915_engine_info engines[]; +}; + #if defined(__cplusplus) } #endif -- cgit v1.2.3-59-g8ed1b From fbb5d0353c62d10c3699ec844d2d015a762952d7 Mon Sep 17 00:00:00 2001 From: Uma Shankar Date: Thu, 16 May 2019 19:40:06 +0530 Subject: drm: Add HDR source metadata property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds a blob property to get HDR metadata information from userspace. This will be send as part of AVI Infoframe to panel. It also implements get() and set() functions for HDR output metadata property.The blob data is received from userspace and saved in connector state, the same is returned as blob in get property call to userspace. v2: Rebase and modified the metadata structure elements as per Ville's POC changes. v3: No Change v4: Addressed Shashank's review comments v5: Rebase. v6: Addressed Brian Starkey's review comments, defined new structure with header for dynamic metadata scalability. Merge get/set property functions for metadata in this patch. v7: Addressed Jonas Karlman review comments and defined separate structure for infoframe to better align with CTA 861.G spec. Added Shashank's RB. v8: Addressed Ville's review comments. Moved sink metadata structure out of uapi headers as suggested by Jonas Karlman. v9: Rebase and addressed Jonas Karlman review comments. v10: Addressed Ville's review comments, dropped the metdata_changed state variable as its not needed anymore. Signed-off-by: Uma Shankar Reviewed-by: Shashank Sharma Signed-off-by: Ville Syrjälä Link: https://patchwork.freedesktop.org/patch/msgid/1558015817-12025-2-git-send-email-uma.shankar@intel.com --- drivers/gpu/drm/drm_atomic_uapi.c | 12 ++++++++++++ drivers/gpu/drm/drm_connector.c | 6 ++++++ include/drm/drm_connector.h | 10 ++++++++++ include/drm/drm_mode_config.h | 7 +++++++ include/linux/hdmi.h | 26 ++++++++++++++++++++++++++ include/uapi/drm/drm_mode.h | 23 +++++++++++++++++++++++ 6 files changed, 84 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index 428d82662dc4..125605ff45af 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -676,6 +676,8 @@ static int drm_atomic_connector_set_property(struct drm_connector *connector, { struct drm_device *dev = connector->dev; struct drm_mode_config *config = &dev->mode_config; + bool replaced = false; + int ret; if (property == config->prop_crtc_id) { struct drm_crtc *crtc = drm_crtc_find(dev, file_priv, val); @@ -726,6 +728,13 @@ static int drm_atomic_connector_set_property(struct drm_connector *connector, */ if (state->link_status != DRM_LINK_STATUS_GOOD) state->link_status = val; + } else if (property == config->hdr_output_metadata_property) { + ret = drm_atomic_replace_property_blob_from_id(dev, + &state->hdr_output_metadata, + val, + sizeof(struct hdr_output_metadata), -1, + &replaced); + return ret; } else if (property == config->aspect_ratio_property) { state->picture_aspect_ratio = val; } else if (property == config->content_type_property) { @@ -814,6 +823,9 @@ drm_atomic_connector_get_property(struct drm_connector *connector, *val = state->colorspace; } else if (property == connector->scaling_mode_property) { *val = state->scaling_mode; + } else if (property == config->hdr_output_metadata_property) { + *val = state->hdr_output_metadata ? + state->hdr_output_metadata->base.id : 0; } else if (property == connector->content_protection_property) { *val = state->content_protection; } else if (property == config->writeback_fb_id_property) { diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index b34c3d38bf15..365ace0c0c9e 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -1058,6 +1058,12 @@ int drm_connector_create_standard_properties(struct drm_device *dev) return -ENOMEM; dev->mode_config.non_desktop_property = prop; + prop = drm_property_create(dev, DRM_MODE_PROP_BLOB, + "HDR_OUTPUT_METADATA", 0); + if (!prop) + return -ENOMEM; + dev->mode_config.hdr_output_metadata_property = prop; + return 0; } diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h index f43f40d5888a..f0e987df4c1e 100644 --- a/include/drm/drm_connector.h +++ b/include/drm/drm_connector.h @@ -603,6 +603,12 @@ struct drm_connector_state { * and the connector bpc limitations obtained from edid. */ u8 max_bpc; + + /** + * @hdr_output_metadata: + * DRM blob property for HDR output metadata + */ + struct drm_property_blob *hdr_output_metadata; }; /** @@ -1243,6 +1249,10 @@ struct drm_connector { * &drm_mode_config.connector_free_work. */ struct llist_node free_node; + + /* HDR metdata */ + struct hdr_output_metadata hdr_output_metadata; + struct hdr_sink_metadata hdr_sink_metadata; }; #define obj_to_connector(x) container_of(x, struct drm_connector, base) diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h index 7f60e8eb269a..c031b5a9d8d1 100644 --- a/include/drm/drm_mode_config.h +++ b/include/drm/drm_mode_config.h @@ -836,6 +836,13 @@ struct drm_mode_config { */ struct drm_property *writeback_out_fence_ptr_property; + /** + * hdr_output_metadata_property: Connector property containing hdr + * metatda. This will be provided by userspace compositors based + * on HDR content + */ + struct drm_property *hdr_output_metadata_property; + /* dumb ioctl parameters */ uint32_t preferred_depth, prefer_shadow; diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h index 927ad6451105..6780476dcbff 100644 --- a/include/linux/hdmi.h +++ b/include/linux/hdmi.h @@ -152,6 +152,16 @@ enum hdmi_content_type { HDMI_CONTENT_TYPE_GAME, }; +enum hdmi_metadata_type { + HDMI_STATIC_METADATA_TYPE1 = 1, +}; + +enum hdmi_eotf { + HDMI_EOTF_TRADITIONAL_GAMMA_SDR, + HDMI_EOTF_TRADITIONAL_GAMMA_HDR, + HDMI_EOTF_SMPTE_ST2084, +}; + struct hdmi_avi_infoframe { enum hdmi_infoframe_type type; unsigned char version; @@ -320,6 +330,22 @@ struct hdmi_vendor_infoframe { unsigned int s3d_ext_data; }; +/* HDR Metadata as per 861.G spec */ +struct hdr_static_metadata { + __u8 eotf; + __u8 metadata_type; + __u16 max_cll; + __u16 max_fall; + __u16 min_cll; +}; + +struct hdr_sink_metadata { + __u32 metadata_type; + union { + struct hdr_static_metadata hdmi_type1; + }; +}; + int hdmi_vendor_infoframe_init(struct hdmi_vendor_infoframe *frame); ssize_t hdmi_vendor_infoframe_pack(struct hdmi_vendor_infoframe *frame, void *buffer, size_t size); diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 83cd1636b9be..997a7e05c0c6 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -630,6 +630,29 @@ struct drm_color_lut { __u16 reserved; }; +/* HDR Metadata Infoframe as per 861.G spec */ +struct hdr_metadata_infoframe { + __u8 eotf; + __u8 metadata_type; + struct { + __u16 x, y; + } display_primaries[3]; + struct { + __u16 x, y; + } white_point; + __u16 max_display_mastering_luminance; + __u16 min_display_mastering_luminance; + __u16 max_cll; + __u16 max_fall; +}; + +struct hdr_output_metadata { + __u32 metadata_type; + union { + struct hdr_metadata_infoframe hdmi_metadata_type1; + }; +}; + #define DRM_MODE_PAGE_FLIP_EVENT 0x01 #define DRM_MODE_PAGE_FLIP_ASYNC 0x02 #define DRM_MODE_PAGE_FLIP_TARGET_ABSOLUTE 0x4 -- cgit v1.2.3-59-g8ed1b From 88807dc8d573c0f718d0d26f592f212c5a487cf0 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 4 Apr 2019 15:47:34 -0500 Subject: drm/amdgpu: Remap hdp coherency registers Remap HDP_MEM_COHERENCY_FLUSH_CNTL and HDP_REG_COHERENCY_FLUSH_CNTL to an empty page in mmio space. We will later map this page to process space so application can flush hdp. This can't be done properly at those registers' original location because it will expose more than desired registers to process space. v2: Use explicit register hole location v3: Moved remapped hdp registers into adev struct v4: Use more generic name for remapped page Expose register offset in kfd_ioctl.h v5: Move hdp register remap function to nbio ip function v6: Fixed operator precedence issue and other bugs Signed-off-by: Oak Zeng Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu.h | 7 +++++++ drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c | 15 ++++++++++++--- drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c | 15 ++++++++++++--- drivers/gpu/drm/amd/amdgpu/soc15.c | 11 +++++++++++ include/uapi/linux/kfd_ioctl.h | 7 +++++++ 5 files changed, 49 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h index 14398f55f602..23c3375623d7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h @@ -639,6 +639,11 @@ struct nbio_hdp_flush_reg { u32 ref_and_mask_sdma1; }; +struct amdgpu_mmio_remap { + u32 reg_offset; + resource_size_t bus_addr; +}; + struct amdgpu_nbio_funcs { const struct nbio_hdp_flush_reg *hdp_flush_reg; u32 (*get_hdp_flush_req_offset)(struct amdgpu_device *adev); @@ -666,6 +671,7 @@ struct amdgpu_nbio_funcs { void (*ih_control)(struct amdgpu_device *adev); void (*init_registers)(struct amdgpu_device *adev); void (*detect_hw_virt)(struct amdgpu_device *adev); + void (*remap_hdp_registers)(struct amdgpu_device *adev); }; struct amdgpu_df_funcs { @@ -764,6 +770,7 @@ struct amdgpu_device { void __iomem *rmmio; /* protects concurrent MM_INDEX/DATA based register access */ spinlock_t mmio_idx_lock; + struct amdgpu_mmio_remap rmmio_remap; /* protects concurrent SMC based register access */ spinlock_t smc_idx_lock; amdgpu_rreg_t smc_rreg; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c index 1cdb98ad2db3..73419fa38159 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_0.c @@ -29,9 +29,18 @@ #include "nbio/nbio_7_0_sh_mask.h" #include "nbio/nbio_7_0_smn.h" #include "vega10_enum.h" +#include #define smnNBIF_MGCG_CTRL_LCLK 0x1013a05c +static void nbio_v7_0_remap_hdp_registers(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); +} + static u32 nbio_v7_0_get_rev_id(struct amdgpu_device *adev) { u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); @@ -55,10 +64,9 @@ static void nbio_v7_0_hdp_flush(struct amdgpu_device *adev, struct amdgpu_ring *ring) { if (!ring || !ring->funcs->emit_wreg) - WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0); + WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); else - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0); + amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); } static u32 nbio_v7_0_get_memsize(struct amdgpu_device *adev) @@ -283,4 +291,5 @@ const struct amdgpu_nbio_funcs nbio_v7_0_funcs = { .ih_control = nbio_v7_0_ih_control, .init_registers = nbio_v7_0_init_registers, .detect_hw_virt = nbio_v7_0_detect_hw_virt, + .remap_hdp_registers = nbio_v7_0_remap_hdp_registers, }; diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c index c69d51598cfe..bfaaa327ae3c 100644 --- a/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c +++ b/drivers/gpu/drm/amd/amdgpu/nbio_v7_4.c @@ -27,9 +27,18 @@ #include "nbio/nbio_7_4_offset.h" #include "nbio/nbio_7_4_sh_mask.h" #include "nbio/nbio_7_4_0_smn.h" +#include #define smnNBIF_MGCG_CTRL_LCLK 0x1013a21c +static void nbio_v7_4_remap_hdp_registers(struct amdgpu_device *adev) +{ + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_MEM_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL); + WREG32_SOC15(NBIO, 0, mmREMAP_HDP_REG_FLUSH_CNTL, + adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL); +} + static u32 nbio_v7_4_get_rev_id(struct amdgpu_device *adev) { u32 tmp = RREG32_SOC15(NBIO, 0, mmRCC_DEV0_EPF0_STRAP0); @@ -53,10 +62,9 @@ static void nbio_v7_4_hdp_flush(struct amdgpu_device *adev, struct amdgpu_ring *ring) { if (!ring || !ring->funcs->emit_wreg) - WREG32_SOC15_NO_KIQ(NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0); + WREG32_NO_KIQ((adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); else - amdgpu_ring_emit_wreg(ring, SOC15_REG_OFFSET( - NBIO, 0, mmHDP_MEM_COHERENCY_FLUSH_CNTL), 0); + amdgpu_ring_emit_wreg(ring, (adev->rmmio_remap.reg_offset + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL) >> 2, 0); } static u32 nbio_v7_4_get_memsize(struct amdgpu_device *adev) @@ -262,4 +270,5 @@ const struct amdgpu_nbio_funcs nbio_v7_4_funcs = { .ih_control = nbio_v7_4_ih_control, .init_registers = nbio_v7_4_init_registers, .detect_hw_virt = nbio_v7_4_detect_hw_virt, + .remap_hdp_registers = nbio_v7_4_remap_hdp_registers, }; diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index 4900e4958dec..78bd00a0142f 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -44,6 +44,7 @@ #include "smuio/smuio_9_0_offset.h" #include "smuio/smuio_9_0_sh_mask.h" #include "nbio/nbio_7_0_default.h" +#include "nbio/nbio_7_0_offset.h" #include "nbio/nbio_7_0_sh_mask.h" #include "nbio/nbio_7_0_smn.h" #include "mp/mp_9_0_offset.h" @@ -64,6 +65,7 @@ #include "dce_virtual.h" #include "mxgpu_ai.h" #include "amdgpu_smu.h" +#include #define mmMP0_MISC_CGTT_CTRL0 0x01b9 #define mmMP0_MISC_CGTT_CTRL0_BASE_IDX 0 @@ -783,8 +785,11 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs = static int soc15_common_early_init(void *handle) { +#define MMIO_REG_HOLE_OFFSET (0x80000 - PAGE_SIZE) struct amdgpu_device *adev = (struct amdgpu_device *)handle; + adev->rmmio_remap.reg_offset = MMIO_REG_HOLE_OFFSET; + adev->rmmio_remap.bus_addr = adev->rmmio_base + MMIO_REG_HOLE_OFFSET; adev->smc_rreg = NULL; adev->smc_wreg = NULL; adev->pcie_rreg = &soc15_pcie_rreg; @@ -1014,6 +1019,12 @@ static int soc15_common_hw_init(void *handle) soc15_program_aspm(adev); /* setup nbio registers */ adev->nbio_funcs->init_registers(adev); + /* remap HDP registers to a hole in mmio space, + * for the purpose of expose those registers + * to process space + */ + if (adev->nbio_funcs->remap_hdp_registers) + adev->nbio_funcs->remap_hdp_registers(adev); /* enable the doorbell aperture */ soc15_enable_doorbell_aperture(adev, true); /* HW doorbell routing policy: doorbell writing not diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index dc067ed0b72d..bb1b4280f53d 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -426,6 +426,13 @@ struct kfd_ioctl_import_dmabuf_args { __u32 dmabuf_fd; /* to KFD */ }; +/* Register offset inside the remapped mmio page + */ +enum kfd_mmio_remap { + KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = 0, + KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4, +}; + #define AMDKFD_IOCTL_BASE 'K' #define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr) #define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type) -- cgit v1.2.3-59-g8ed1b From d8e408a82704c86ba87c3d58cfe69dcdb758aa07 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 11 Apr 2019 14:43:39 -0500 Subject: drm/amdkfd: Expose HDP registers to user space Introduce a new memory type (KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) and expose mmio page of HDP registers to user space through this new memory type. v2: moved remapped hdp regs to adev struct v3: rename the new memory type to ALLOC_MEM_FLAGS_MMIO_REMAP v4: use more generic function name v5: Fail remapped mmio allocation for asics before gfx9 Signed-off-by: Oak Zeng Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 7 +++++++ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 1 + drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 7 ++++--- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 6 ++++++ drivers/gpu/drm/amd/include/kgd_kfd_interface.h | 1 + include/uapi/linux/kfd_ioctl.h | 1 + 6 files changed, 20 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index aeead072fa79..401edb605fdd 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -519,6 +519,13 @@ uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd) return adev->gmc.xgmi.hive_id; } +uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd) +{ + struct amdgpu_device *adev = (struct amdgpu_device *)kgd; + + return adev->rmmio_remap.bus_addr; +} + int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine, uint32_t vmid, uint64_t gpu_addr, uint32_t *ib_cmd, uint32_t ib_len) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 4e37fa7e85b1..ea1f141db3ff 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -169,6 +169,7 @@ int amdgpu_amdkfd_get_dmabuf_info(struct kgd_dev *kgd, int dma_buf_fd, uint32_t *flags); uint64_t amdgpu_amdkfd_get_vram_usage(struct kgd_dev *kgd); uint64_t amdgpu_amdkfd_get_hive_id(struct kgd_dev *kgd); +uint64_t amdgpu_amdkfd_get_mmio_remap_phys_addr(struct kgd_dev *kgd); #define read_user_wptr(mmptr, wptr, dst) \ ({ \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index a6e5184d436c..00e013581a70 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -1109,7 +1109,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( if (!offset || !*offset) return -EINVAL; user_addr = *offset; - } else if (flags & ALLOC_MEM_FLAGS_DOORBELL) { + } else if (flags & (ALLOC_MEM_FLAGS_DOORBELL | + ALLOC_MEM_FLAGS_MMIO_REMAP)) { domain = AMDGPU_GEM_DOMAIN_GTT; alloc_domain = AMDGPU_GEM_DOMAIN_CPU; bo_type = ttm_bo_type_sg; @@ -1294,8 +1295,8 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( /* Free the sync object */ amdgpu_sync_free(&mem->sync); - /* If the SG is not NULL, it's one we created for a doorbell - * BO. We need to free it. + /* If the SG is not NULL, it's one we created for a doorbell or mmio + * remap BO. We need to free it. */ if (mem->bo->tbo.sg) { sg_free_table(mem->bo->tbo.sg); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 083bd8114db1..d795e5018270 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1272,6 +1272,12 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, if (args->size != kfd_doorbell_process_slice(dev)) return -EINVAL; offset = kfd_get_process_doorbells(dev, p); + } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) { + if (args->size != PAGE_SIZE) + return -EINVAL; + offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd); + if (!offset) + return -ENOMEM; } mutex_lock(&p->mutex); diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h index b897aca9b4c9..98b9533e672b 100644 --- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h +++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h @@ -174,6 +174,7 @@ struct tile_config { #define ALLOC_MEM_FLAGS_GTT (1 << 1) #define ALLOC_MEM_FLAGS_USERPTR (1 << 2) #define ALLOC_MEM_FLAGS_DOORBELL (1 << 3) +#define ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4) /* * Allocation flags attributes/access options. diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index bb1b4280f53d..1e7d5f3376b0 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -338,6 +338,7 @@ struct kfd_ioctl_acquire_vm_args { #define KFD_IOC_ALLOC_MEM_FLAGS_GTT (1 << 1) #define KFD_IOC_ALLOC_MEM_FLAGS_USERPTR (1 << 2) #define KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL (1 << 3) +#define KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP (1 << 4) /* Allocation flags: attributes/access options */ #define KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE (1 << 31) #define KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE (1 << 30) -- cgit v1.2.3-59-g8ed1b From 1b4670f6983156526c286723465fdf805070b45d Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Thu, 7 Feb 2019 14:02:27 -0600 Subject: drm/amdkfd: Introduce XGMI SDMA queue type Existing QUEUE_TYPE_SDMA means PCIe optimized SDMA queues. Introduce a new QUEUE_TYPE_SDMA_XGMI, which is optimized for non-PCIe transfer such as XGMI. Signed-off-by: Oak Zeng Reviewed-by: Felix Kuehling Signed-off-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 2 + drivers/gpu/drm/amd/amdkfd/kfd_device.c | 15 +++ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 123 +++++++++++++++------ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h | 3 + drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c | 2 + drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c | 2 + drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c | 3 +- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 4 +- .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 10 +- include/uapi/linux/kfd_ioctl.h | 7 +- 10 files changed, 132 insertions(+), 39 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index 3ccaa38779ea..38ae53fe8182 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -213,6 +213,8 @@ static int set_queue_properties_from_user(struct queue_properties *q_properties, q_properties->type = KFD_QUEUE_TYPE_COMPUTE; else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) q_properties->type = KFD_QUEUE_TYPE_SDMA; + else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI) + q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI; else return -ENOTSUPP; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index 8202a5db3a35..1368b41cb92b 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -54,6 +54,7 @@ static const struct kfd_device_info kaveri_device_info = { .needs_iommu_device = true, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -71,6 +72,7 @@ static const struct kfd_device_info carrizo_device_info = { .needs_iommu_device = true, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -87,6 +89,7 @@ static const struct kfd_device_info raven_device_info = { .needs_iommu_device = true, .needs_pci_atomics = true, .num_sdma_engines = 1, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; #endif @@ -105,6 +108,7 @@ static const struct kfd_device_info hawaii_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -121,6 +125,7 @@ static const struct kfd_device_info tonga_device_info = { .needs_iommu_device = false, .needs_pci_atomics = true, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -137,6 +142,7 @@ static const struct kfd_device_info fiji_device_info = { .needs_iommu_device = false, .needs_pci_atomics = true, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -153,6 +159,7 @@ static const struct kfd_device_info fiji_vf_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -170,6 +177,7 @@ static const struct kfd_device_info polaris10_device_info = { .needs_iommu_device = false, .needs_pci_atomics = true, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -186,6 +194,7 @@ static const struct kfd_device_info polaris10_vf_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -202,6 +211,7 @@ static const struct kfd_device_info polaris11_device_info = { .needs_iommu_device = false, .needs_pci_atomics = true, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -218,6 +228,7 @@ static const struct kfd_device_info polaris12_device_info = { .needs_iommu_device = false, .needs_pci_atomics = true, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -234,6 +245,7 @@ static const struct kfd_device_info vega10_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -250,6 +262,7 @@ static const struct kfd_device_info vega10_vf_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -266,6 +279,7 @@ static const struct kfd_device_info vega12_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 2, }; @@ -282,6 +296,7 @@ static const struct kfd_device_info vega20_device_info = { .needs_iommu_device = false, .needs_pci_atomics = false, .num_sdma_engines = 2, + .num_xgmi_sdma_engines = 0, .num_sdma_queues_per_engine = 8, }; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index d41045d3fc3a..1562590d837e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -60,14 +60,14 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, struct qcm_process_device *qpd); static void deallocate_sdma_queue(struct device_queue_manager *dqm, - unsigned int sdma_queue_id); + struct queue *q); static void kfd_process_hw_exception(struct work_struct *work); static inline enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) { - if (type == KFD_QUEUE_TYPE_SDMA) + if (type == KFD_QUEUE_TYPE_SDMA || type == KFD_QUEUE_TYPE_SDMA_XGMI) return KFD_MQD_TYPE_SDMA; return KFD_MQD_TYPE_CP; } @@ -107,12 +107,23 @@ static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm) return dqm->dev->device_info->num_sdma_engines; } +static unsigned int get_num_xgmi_sdma_engines(struct device_queue_manager *dqm) +{ + return dqm->dev->device_info->num_xgmi_sdma_engines; +} + unsigned int get_num_sdma_queues(struct device_queue_manager *dqm) { return dqm->dev->device_info->num_sdma_engines * dqm->dev->device_info->num_sdma_queues_per_engine; } +unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm) +{ + return dqm->dev->device_info->num_xgmi_sdma_engines + * dqm->dev->device_info->num_sdma_queues_per_engine; +} + void program_sh_mem_settings(struct device_queue_manager *dqm, struct qcm_process_device *qpd) { @@ -133,7 +144,8 @@ static int allocate_doorbell(struct qcm_process_device *qpd, struct queue *q) * preserve the user mode ABI. */ q->doorbell_id = q->properties.queue_id; - } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { /* For SDMA queues on SOC15 with 8-byte doorbell, use static * doorbell assignments based on the engine and queue id. * The doobell index distance between RLC (2*i) and (2*i+1) @@ -174,7 +186,8 @@ static void deallocate_doorbell(struct qcm_process_device *qpd, struct kfd_dev *dev = qpd->dqm->dev; if (!KFD_IS_SOC15(dev->device_info->asic_family) || - q->properties.type == KFD_QUEUE_TYPE_SDMA) + q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) return; old = test_and_clear_bit(q->doorbell_id, qpd->doorbell_bitmap); @@ -289,7 +302,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) retval = create_compute_queue_nocpsch(dqm, q, qpd); - else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) + else if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) retval = create_sdma_queue_nocpsch(dqm, q, qpd); else retval = -EINVAL; @@ -307,6 +321,8 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, if (q->properties.type == KFD_QUEUE_TYPE_SDMA) dqm->sdma_queue_count++; + else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + dqm->xgmi_sdma_queue_count++; /* * Unconditionally increment this counter, regardless of the queue's @@ -430,7 +446,10 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, deallocate_hqd(dqm, q); } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { dqm->sdma_queue_count--; - deallocate_sdma_queue(dqm, q->sdma_id); + deallocate_sdma_queue(dqm, q); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + dqm->xgmi_sdma_queue_count--; + deallocate_sdma_queue(dqm, q); } else { pr_debug("q->properties.type %d is invalid\n", q->properties.type); @@ -521,7 +540,8 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) } } else if (prev_active && (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || - q->properties.type == KFD_QUEUE_TYPE_SDMA)) { + q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd, KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN, KFD_UNMAP_LATENCY_MS, q->pipe, q->queue); @@ -548,7 +568,8 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) retval = map_queues_cpsch(dqm); else if (q->properties.is_active && (q->properties.type == KFD_QUEUE_TYPE_COMPUTE || - q->properties.type == KFD_QUEUE_TYPE_SDMA)) { + q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { if (WARN(q->process->mm != current->mm, "should only run in user thread")) retval = -EFAULT; @@ -840,6 +861,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm) INIT_LIST_HEAD(&dqm->queues); dqm->queue_count = dqm->next_pipe_to_allocate = 0; dqm->sdma_queue_count = 0; + dqm->xgmi_sdma_queue_count = 0; for (pipe = 0; pipe < get_pipes_per_mec(dqm); pipe++) { int pipe_offset = pipe * get_queues_per_pipe(dqm); @@ -852,6 +874,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm) dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1; dqm->sdma_bitmap = (1ULL << get_num_sdma_queues(dqm)) - 1; + dqm->xgmi_sdma_bitmap = (1ULL << get_num_xgmi_sdma_queues(dqm)) - 1; return 0; } @@ -886,17 +909,34 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, { int bit; - if (dqm->sdma_bitmap == 0) - return -ENOMEM; - - bit = __ffs64(dqm->sdma_bitmap); - dqm->sdma_bitmap &= ~(1ULL << bit); - q->sdma_id = bit; - - q->properties.sdma_engine_id = q->sdma_id % get_num_sdma_engines(dqm); - q->properties.sdma_queue_id = q->sdma_id / get_num_sdma_engines(dqm); + if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + if (dqm->sdma_bitmap == 0) + return -ENOMEM; + bit = __ffs64(dqm->sdma_bitmap); + dqm->sdma_bitmap &= ~(1ULL << bit); + q->sdma_id = bit; + q->properties.sdma_engine_id = q->sdma_id % + get_num_sdma_engines(dqm); + q->properties.sdma_queue_id = q->sdma_id / + get_num_sdma_engines(dqm); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + if (dqm->xgmi_sdma_bitmap == 0) + return -ENOMEM; + bit = __ffs64(dqm->xgmi_sdma_bitmap); + dqm->xgmi_sdma_bitmap &= ~(1ULL << bit); + q->sdma_id = bit; + /* sdma_engine_id is sdma id including + * both PCIe-optimized SDMAs and XGMI- + * optimized SDMAs. The calculation below + * assumes the first N engines are always + * PCIe-optimized ones + */ + q->properties.sdma_engine_id = get_num_sdma_engines(dqm) + + q->sdma_id % get_num_xgmi_sdma_engines(dqm); + q->properties.sdma_queue_id = q->sdma_id / + get_num_xgmi_sdma_engines(dqm); + } - pr_debug("SDMA id is: %d\n", q->sdma_id); pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id); pr_debug("SDMA queue id: %d\n", q->properties.sdma_queue_id); @@ -904,11 +944,17 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, } static void deallocate_sdma_queue(struct device_queue_manager *dqm, - unsigned int sdma_id) + struct queue *q) { - if (sdma_id >= get_num_sdma_queues(dqm)) - return; - dqm->sdma_bitmap |= (1ULL << sdma_id); + if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + if (q->sdma_id >= get_num_sdma_queues(dqm)) + return; + dqm->sdma_bitmap |= (1ULL << q->sdma_id); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + if (q->sdma_id >= get_num_xgmi_sdma_queues(dqm)) + return; + dqm->xgmi_sdma_bitmap |= (1ULL << q->sdma_id); + } } static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, @@ -946,7 +992,7 @@ out_uninit_mqd: out_deallocate_doorbell: deallocate_doorbell(qpd, q); out_deallocate_sdma_queue: - deallocate_sdma_queue(dqm, q->sdma_id); + deallocate_sdma_queue(dqm, q); return retval; } @@ -1004,8 +1050,10 @@ static int initialize_cpsch(struct device_queue_manager *dqm) INIT_LIST_HEAD(&dqm->queues); dqm->queue_count = dqm->processes_count = 0; dqm->sdma_queue_count = 0; + dqm->xgmi_sdma_queue_count = 0; dqm->active_runlist = false; dqm->sdma_bitmap = (1ULL << get_num_sdma_queues(dqm)) - 1; + dqm->xgmi_sdma_bitmap = (1ULL << get_num_xgmi_sdma_queues(dqm)) - 1; INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception); @@ -1127,7 +1175,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, goto out; } - if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { + if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { retval = allocate_sdma_queue(dqm, q); if (retval) goto out; @@ -1167,6 +1216,8 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, if (q->properties.type == KFD_QUEUE_TYPE_SDMA) dqm->sdma_queue_count++; + else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + dqm->xgmi_sdma_queue_count++; /* * Unconditionally increment this counter, regardless of the queue's * type or whether the queue is active. @@ -1182,8 +1233,9 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, out_deallocate_doorbell: deallocate_doorbell(qpd, q); out_deallocate_sdma_queue: - if (q->properties.type == KFD_QUEUE_TYPE_SDMA) - deallocate_sdma_queue(dqm, q->sdma_id); + if (q->properties.type == KFD_QUEUE_TYPE_SDMA || + q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) + deallocate_sdma_queue(dqm, q); out: return retval; } @@ -1216,7 +1268,8 @@ static int unmap_sdma_queues(struct device_queue_manager *dqm) { int i, retval = 0; - for (i = 0; i < dqm->dev->device_info->num_sdma_engines; i++) { + for (i = 0; i < dqm->dev->device_info->num_sdma_engines + + dqm->dev->device_info->num_xgmi_sdma_engines; i++) { retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_SDMA, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0, false, i); if (retval) @@ -1258,10 +1311,10 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm, if (!dqm->active_runlist) return retval; - pr_debug("Before destroying queues, sdma queue count is : %u\n", - dqm->sdma_queue_count); + pr_debug("Before destroying queues, sdma queue count is : %u, xgmi sdma queue count is : %u\n", + dqm->sdma_queue_count, dqm->xgmi_sdma_queue_count); - if (dqm->sdma_queue_count > 0) + if (dqm->sdma_queue_count > 0 || dqm->xgmi_sdma_queue_count) unmap_sdma_queues(dqm); retval = pm_send_unmap_queue(&dqm->packets, KFD_QUEUE_TYPE_COMPUTE, @@ -1333,7 +1386,10 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { dqm->sdma_queue_count--; - deallocate_sdma_queue(dqm, q->sdma_id); + deallocate_sdma_queue(dqm, q); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + dqm->xgmi_sdma_queue_count--; + deallocate_sdma_queue(dqm, q); } list_del(&q->list); @@ -1550,7 +1606,10 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, list_for_each_entry(q, &qpd->queues_list, list) { if (q->properties.type == KFD_QUEUE_TYPE_SDMA) { dqm->sdma_queue_count--; - deallocate_sdma_queue(dqm, q->sdma_id); + deallocate_sdma_queue(dqm, q); + } else if (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI) { + dqm->xgmi_sdma_queue_count--; + deallocate_sdma_queue(dqm, q); } if (q->properties.is_active) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h index 3742fd340ec3..88b4c007696e 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h @@ -181,10 +181,12 @@ struct device_queue_manager { unsigned int processes_count; unsigned int queue_count; unsigned int sdma_queue_count; + unsigned int xgmi_sdma_queue_count; unsigned int total_queue_count; unsigned int next_pipe_to_allocate; unsigned int *allocated_queues; uint64_t sdma_bitmap; + uint64_t xgmi_sdma_bitmap; unsigned int vmid_bitmap; uint64_t pipelines_addr; struct kfd_mem_obj *pipeline_mem; @@ -216,6 +218,7 @@ unsigned int get_queues_num(struct device_queue_manager *dqm); unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); +unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c index 33830b1a5a54..604570bea6bd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_v9.c @@ -175,6 +175,7 @@ static int pm_map_queues_v9(struct packet_manager *pm, uint32_t *buffer, queue_type__mes_map_queues__debug_interface_queue_vi; break; case KFD_QUEUE_TYPE_SDMA: + case KFD_QUEUE_TYPE_SDMA_XGMI: packet->bitfields2.engine_sel = q->properties.sdma_engine_id + engine_sel__mes_map_queues__sdma0_vi; use_static = false; /* no static queues under SDMA */ @@ -221,6 +222,7 @@ static int pm_unmap_queues_v9(struct packet_manager *pm, uint32_t *buffer, engine_sel__mes_unmap_queues__compute; break; case KFD_QUEUE_TYPE_SDMA: + case KFD_QUEUE_TYPE_SDMA_XGMI: packet->bitfields2.engine_sel = engine_sel__mes_unmap_queues__sdma0 + sdma_engine; break; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c index bf20c6d32ef3..3cdb19826927 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue_vi.c @@ -212,6 +212,7 @@ static int pm_map_queues_vi(struct packet_manager *pm, uint32_t *buffer, queue_type__mes_map_queues__debug_interface_queue_vi; break; case KFD_QUEUE_TYPE_SDMA: + case KFD_QUEUE_TYPE_SDMA_XGMI: packet->bitfields2.engine_sel = q->properties.sdma_engine_id + engine_sel__mes_map_queues__sdma0_vi; use_static = false; /* no static queues under SDMA */ @@ -258,6 +259,7 @@ static int pm_unmap_queues_vi(struct packet_manager *pm, uint32_t *buffer, engine_sel__mes_unmap_queues__compute; break; case KFD_QUEUE_TYPE_SDMA: + case KFD_QUEUE_TYPE_SDMA_XGMI: packet->bitfields2.engine_sel = engine_sel__mes_unmap_queues__sdma0 + sdma_engine; break; diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c index 045a229436a0..077c47fd4fee 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c @@ -48,7 +48,8 @@ static void pm_calc_rlib_size(struct packet_manager *pm, process_count = pm->dqm->processes_count; queue_count = pm->dqm->queue_count; - compute_queue_count = queue_count - pm->dqm->sdma_queue_count; + compute_queue_count = queue_count - pm->dqm->sdma_queue_count - + pm->dqm->xgmi_sdma_queue_count; /* check if there is over subscription * Note: the arbitration between the number of VMIDs and diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index 10bd1abe1646..8f02d7817162 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -188,6 +188,7 @@ struct kfd_device_info { bool needs_iommu_device; bool needs_pci_atomics; unsigned int num_sdma_engines; + unsigned int num_xgmi_sdma_engines; unsigned int num_sdma_queues_per_engine; }; @@ -329,7 +330,8 @@ enum kfd_queue_type { KFD_QUEUE_TYPE_COMPUTE, KFD_QUEUE_TYPE_SDMA, KFD_QUEUE_TYPE_HIQ, - KFD_QUEUE_TYPE_DIQ + KFD_QUEUE_TYPE_DIQ, + KFD_QUEUE_TYPE_SDMA_XGMI }; enum kfd_queue_format { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index f18d9cdf9aac..e652e25ede75 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -186,8 +186,13 @@ int pqm_create_queue(struct process_queue_manager *pqm, switch (type) { case KFD_QUEUE_TYPE_SDMA: - if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) { - pr_err("Over-subscription is not allowed for SDMA.\n"); + case KFD_QUEUE_TYPE_SDMA_XGMI: + if ((type == KFD_QUEUE_TYPE_SDMA && dev->dqm->sdma_queue_count + >= get_num_sdma_queues(dev->dqm)) || + (type == KFD_QUEUE_TYPE_SDMA_XGMI && + dev->dqm->xgmi_sdma_queue_count + >= get_num_xgmi_sdma_queues(dev->dqm))) { + pr_debug("Over-subscription is not allowed for SDMA.\n"); retval = -EPERM; goto err_create_queue; } @@ -446,6 +451,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data) q = pqn->q; switch (q->properties.type) { case KFD_QUEUE_TYPE_SDMA: + case KFD_QUEUE_TYPE_SDMA_XGMI: seq_printf(m, " SDMA queue on device %x\n", q->device->id); mqd_type = KFD_MQD_TYPE_SDMA; diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index 1e7d5f3376b0..20917c59f39c 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -35,9 +35,10 @@ struct kfd_ioctl_get_version_args { }; /* For kfd_ioctl_create_queue_args.queue_type. */ -#define KFD_IOC_QUEUE_TYPE_COMPUTE 0 -#define KFD_IOC_QUEUE_TYPE_SDMA 1 -#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 2 +#define KFD_IOC_QUEUE_TYPE_COMPUTE 0x0 +#define KFD_IOC_QUEUE_TYPE_SDMA 0x1 +#define KFD_IOC_QUEUE_TYPE_COMPUTE_AQL 0x2 +#define KFD_IOC_QUEUE_TYPE_SDMA_XGMI 0x3 #define KFD_MAX_QUEUE_PERCENTAGE 100 #define KFD_MAX_QUEUE_PRIORITY 15 -- cgit v1.2.3-59-g8ed1b From 1a058c3376765ee31d65e28cbbb9d4ff15120056 Mon Sep 17 00:00:00 2001 From: Oak Zeng Date: Mon, 6 May 2019 22:11:14 -0500 Subject: drm/amdkfd: New IOCTL to allocate queue GWS Add a new kfd ioctl to allocate queue GWS. Queue GWS is released on queue destroy. Signed-off-by: Oak Zeng Reviewed-by: Felix Kuehling Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdkfd/kfd_chardev.c | 27 +++++++++++++++++++++++++++ include/uapi/linux/kfd_ioctl.h | 20 +++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c index c92e931ceb27..aab2aa6c1dee 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c @@ -1567,6 +1567,31 @@ copy_from_user_failed: return err; } +static int kfd_ioctl_alloc_queue_gws(struct file *filep, + struct kfd_process *p, void *data) +{ + int retval; + struct kfd_ioctl_alloc_queue_gws_args *args = data; + struct kfd_dev *dev = NULL; + + if (!hws_gws_support || + dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) + return -EINVAL; + + dev = kfd_device_by_id(args->gpu_id); + if (!dev) { + pr_debug("Could not find gpu id 0x%x\n", args->gpu_id); + return -EINVAL; + } + + mutex_lock(&p->mutex); + retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL); + mutex_unlock(&p->mutex); + + args->first_gws = 0; + return retval; +} + static int kfd_ioctl_get_dmabuf_info(struct file *filep, struct kfd_process *p, void *data) { @@ -1769,6 +1794,8 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF, kfd_ioctl_import_dmabuf, 0), + AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS, + kfd_ioctl_alloc_queue_gws, 0), }; #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h index 20917c59f39c..070d1bc7e725 100644 --- a/include/uapi/linux/kfd_ioctl.h +++ b/include/uapi/linux/kfd_ioctl.h @@ -410,6 +410,21 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { __u32 n_success; /* to/from KFD */ }; +/* Allocate GWS for specific queue + * + * @gpu_id: device identifier + * @queue_id: queue's id that GWS is allocated for + * @num_gws: how many GWS to allocate + * @first_gws: index of the first GWS allocated. + * only support contiguous GWS allocation + */ +struct kfd_ioctl_alloc_queue_gws_args { + __u32 gpu_id; /* to KFD */ + __u32 queue_id; /* to KFD */ + __u32 num_gws; /* to KFD */ + __u32 first_gws; /* from KFD */ +}; + struct kfd_ioctl_get_dmabuf_info_args { __u64 size; /* from KFD */ __u64 metadata_ptr; /* to KFD */ @@ -529,7 +544,10 @@ enum kfd_mmio_remap { #define AMDKFD_IOC_IMPORT_DMABUF \ AMDKFD_IOWR(0x1D, struct kfd_ioctl_import_dmabuf_args) +#define AMDKFD_IOC_ALLOC_QUEUE_GWS \ + AMDKFD_IOWR(0x1E, struct kfd_ioctl_alloc_queue_gws_args) + #define AMDKFD_COMMAND_START 0x01 -#define AMDKFD_COMMAND_END 0x1E +#define AMDKFD_COMMAND_END 0x1F #endif -- cgit v1.2.3-59-g8ed1b From cfc1ce7e5212b19423d38ed2035318a35f983309 Mon Sep 17 00:00:00 2001 From: Uma Shankar Date: Thu, 30 May 2019 01:29:03 +0530 Subject: drm: Fixed doc warnings in drm uapi header Fixed doc warnings in drm uapi header. All the UAPI structures are now documented in kernel doc. Signed-off-by: Uma Shankar Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/1559159944-21103-4-git-send-email-uma.shankar@intel.com --- include/uapi/drm/drm_mode.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 997a7e05c0c6..19b5cf368cff 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -826,6 +826,10 @@ struct drm_format_modifier { }; /** + * struct drm_mode_create_blob - Create New block property + * @data: Pointer to data to copy. + * @length: Length of data to copy. + * @blob_id: new property ID. * Create a new 'blob' data property, copying length bytes from data pointer, * and returning new blob ID. */ @@ -839,6 +843,8 @@ struct drm_mode_create_blob { }; /** + * struct drm_mode_destroy_blob - Destroy user blob + * @blob_id: blob_id to destroy * Destroy a user-created blob property. */ struct drm_mode_destroy_blob { @@ -846,6 +852,12 @@ struct drm_mode_destroy_blob { }; /** + * struct drm_mode_create_lease - Create lease + * @object_ids: Pointer to array of object ids. + * @object_count: Number of object ids. + * @flags: flags for new FD. + * @lessee_id: unique identifier for lessee. + * @fd: file descriptor to new drm_master file. * Lease mode resources, creating another drm_master. */ struct drm_mode_create_lease { @@ -863,6 +875,10 @@ struct drm_mode_create_lease { }; /** + * struct drm_mode_list_lessees - List lessees + * @count_lessees: Number of lessees. + * @pad: pad. + * @lessees_ptr: Pointer to lessess. * List lesses from a drm_master */ struct drm_mode_list_lessees { @@ -883,6 +899,10 @@ struct drm_mode_list_lessees { }; /** + * struct drm_mode_get_lease - Get Lease + * @count_objects: Number of leased objects. + * @pad: pad. + * @objects_ptr: Pointer to objects. * Get leased objects */ struct drm_mode_get_lease { @@ -903,6 +923,8 @@ struct drm_mode_get_lease { }; /** + * struct drm_mode_revoke_lease - Revoke lease + * @lessee_id: Unique ID of lessee. * Revoke lease */ struct drm_mode_revoke_lease { -- cgit v1.2.3-59-g8ed1b From a09db883e5d938b525a86a4630fc04f98ff1063d Mon Sep 17 00:00:00 2001 From: Uma Shankar Date: Tue, 4 Jun 2019 16:47:02 +0530 Subject: drm: Fix docbook warnings in hdr metadata helper structures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the following warnings: ./include/drm/drm_mode_config.h:841: warning: Incorrect use of kernel-doc format: * hdr_output_metadata_property: Connector property containing hdr ./include/drm/drm_mode_config.h:918: warning: Function parameter or member 'hdr_output_metadata_property' not described in 'drm_mode_config' ./include/drm/drm_connector.h:1251: warning: Function parameter or member 'hdr_output_metadata' not described in 'drm_connector' ./include/drm/drm_connector.h:1251: warning: Function parameter or member 'hdr_sink_metadata' not described in 'drm_connector' Also adds some property documentation for HDR Metadata Connector Property in connector property create function. v2: Fixed Sean Paul's review comments. v3: Fixed Daniel Vetter's review comments, added the UAPI structure definition section in kernel docs. v4: Fixed Daniel Vetter's review comments. v5: Added structure member references as per Daniel's suggestion. Cc: Shashank Sharma Cc: Ville Syrjä Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Sean Paul Cc: David Airlie Cc: Daniel Vetter Cc: Bartlomiej Zolnierkiewicz Cc: "Ville Syrjä" Cc: Hans Verkuil Cc: dri-devel@lists.freedesktop.org Cc: linux-fbdev@vger.kernel.org Reviewed-by: Sean Paul (v1) Signed-off-by: Uma Shankar [danvet: Fix up markup: () for functions, & for structs. Style guide also recommends to prepend struct for structures.] Signed-off-by: Daniel Vetter Link: https://patchwork.freedesktop.org/patch/msgid/1559647022-7336-1-git-send-email-uma.shankar@intel.com --- drivers/gpu/drm/drm_connector.c | 40 ++++++++++++++++++++++ include/drm/drm_connector.h | 1 + include/drm/drm_mode_config.h | 4 +-- include/linux/hdmi.h | 12 +++++++ include/uapi/drm/drm_mode.h | 74 ++++++++++++++++++++++++++++++++++++++++- 5 files changed, 128 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index c9ac8b9e83ea..e17586aaa80f 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -956,6 +956,46 @@ static const struct drm_prop_enum_list hdmi_colorspaces[] = { * is no longer protected and userspace should take appropriate action * (whatever that might be). * + * HDR_OUTPUT_METADATA: + * Connector property to enable userspace to send HDR Metadata to + * driver. This metadata is based on the composition and blending + * policies decided by user, taking into account the hardware and + * sink capabilities. The driver gets this metadata and creates a + * Dynamic Range and Mastering Infoframe (DRM) in case of HDMI, + * SDP packet (Non-audio INFOFRAME SDP v1.3) for DP. This is then + * sent to sink. This notifies the sink of the upcoming frame's Color + * Encoding and Luminance parameters. + * + * Userspace first need to detect the HDR capabilities of sink by + * reading and parsing the EDID. Details of HDR metadata for HDMI + * are added in CTA 861.G spec. For DP , its defined in VESA DP + * Standard v1.4. It needs to then get the metadata information + * of the video/game/app content which are encoded in HDR (basically + * using HDR transfer functions). With this information it needs to + * decide on a blending policy and compose the relevant + * layers/overlays into a common format. Once this blending is done, + * userspace will be aware of the metadata of the composed frame to + * be send to sink. It then uses this property to communicate this + * metadata to driver which then make a Infoframe packet and sends + * to sink based on the type of encoder connected. + * + * Userspace will be responsible to do Tone mapping operation in case: + * - Some layers are HDR and others are SDR + * - HDR layers luminance is not same as sink + * It will even need to do colorspace conversion and get all layers + * to one common colorspace for blending. It can use either GL, Media + * or display engine to get this done based on the capabilties of the + * associated hardware. + * + * Driver expects metadata to be put in &struct hdr_output_metadata + * structure from userspace. This is received as blob and stored in + * &drm_connector_state.hdr_output_metadata. It parses EDID and saves the + * sink metadata in &struct hdr_sink_metadata, as + * &drm_connector.hdr_sink_metadata. Driver uses + * drm_hdmi_infoframe_set_hdr_metadata() helper to set the HDR metadata, + * hdmi_drm_infoframe_pack() to pack the infoframe as per spec, in case of + * HDMI encoder. + * * max bpc: * This range property is used by userspace to limit the bit depth. When * used the driver would limit the bpc in accordance with the valid range diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h index 547656173c74..47e749b74e5f 100644 --- a/include/drm/drm_connector.h +++ b/include/drm/drm_connector.h @@ -1244,6 +1244,7 @@ struct drm_connector { */ struct llist_node free_node; + /** @hdr_sink_metadata: HDR Metadata Information read from sink */ struct hdr_sink_metadata hdr_sink_metadata; }; diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h index 4f88cc972407..759d462d028b 100644 --- a/include/drm/drm_mode_config.h +++ b/include/drm/drm_mode_config.h @@ -837,8 +837,8 @@ struct drm_mode_config { struct drm_property *writeback_out_fence_ptr_property; /** - * hdr_output_metadata_property: Connector property containing hdr - * metatda. This will be provided by userspace compositors based + * @hdr_output_metadata_property: Connector property containing hdr + * metatada. This will be provided by userspace compositors based * on HDR content */ struct drm_property *hdr_output_metadata_property; diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h index ee55ba589cdc..9918a6c910c5 100644 --- a/include/linux/hdmi.h +++ b/include/linux/hdmi.h @@ -367,8 +367,19 @@ struct hdr_static_metadata { __u16 min_cll; }; +/** + * struct hdr_sink_metadata - HDR sink metadata + * + * Metadata Information read from Sink's EDID + */ struct hdr_sink_metadata { + /** + * @metadata_type: Static_Metadata_Descriptor_ID. + */ __u32 metadata_type; + /** + * @hdmi_type1: HDR Metadata Infoframe. + */ union { struct hdr_static_metadata hdmi_type1; }; @@ -398,6 +409,7 @@ union hdmi_vendor_any_infoframe { * @spd: spd infoframe * @vendor: union of all vendor infoframes * @audio: audio infoframe + * @drm: Dynamic Range and Mastering infoframe * * This is used by the generic pack function. This works since all infoframes * have the same header which also indicates which type of infoframe should be diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 19b5cf368cff..5ab331e5dc23 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -33,6 +33,15 @@ extern "C" { #endif +/** + * DOC: overview + * + * DRM exposes many UAPI and structure definition to have a consistent + * and standardized interface with user. + * Userspace can refer to these structure definitions and UAPI formats + * to communicate to driver + */ + #define DRM_CONNECTOR_NAME_LEN 32 #define DRM_DISPLAY_MODE_LEN 32 #define DRM_PROP_NAME_LEN 32 @@ -630,24 +639,87 @@ struct drm_color_lut { __u16 reserved; }; -/* HDR Metadata Infoframe as per 861.G spec */ +/** + * struct hdr_metadata_infoframe - HDR Metadata Infoframe Data. + * + * HDR Metadata Infoframe as per CTA 861.G spec. This is expected + * to match exactly with the spec. + * + * Userspace is expected to pass the metadata information as per + * the format described in this structure. + */ struct hdr_metadata_infoframe { + /** + * @eotf: Electro-Optical Transfer Function (EOTF) + * used in the stream. + */ __u8 eotf; + /** + * @metadata_type: Static_Metadata_Descriptor_ID. + */ __u8 metadata_type; + /** + * @display_primaries: Color Primaries of the Data. + * These are coded as unsigned 16-bit values in units of + * 0.00002, where 0x0000 represents zero and 0xC350 + * represents 1.0000. + * @display_primaries.x: X cordinate of color primary. + * @display_primaries.y: Y cordinate of color primary. + */ struct { __u16 x, y; } display_primaries[3]; + /** + * @white_point: White Point of Colorspace Data. + * These are coded as unsigned 16-bit values in units of + * 0.00002, where 0x0000 represents zero and 0xC350 + * represents 1.0000. + * @white_point.x: X cordinate of whitepoint of color primary. + * @white_point.y: Y cordinate of whitepoint of color primary. + */ struct { __u16 x, y; } white_point; + /** + * @max_display_mastering_luminance: Max Mastering Display Luminance. + * This value is coded as an unsigned 16-bit value in units of 1 cd/m2, + * where 0x0001 represents 1 cd/m2 and 0xFFFF represents 65535 cd/m2. + */ __u16 max_display_mastering_luminance; + /** + * @min_display_mastering_luminance: Min Mastering Display Luminance. + * This value is coded as an unsigned 16-bit value in units of + * 0.0001 cd/m2, where 0x0001 represents 0.0001 cd/m2 and 0xFFFF + * represents 6.5535 cd/m2. + */ __u16 min_display_mastering_luminance; + /** + * @max_cll: Max Content Light Level. + * This value is coded as an unsigned 16-bit value in units of 1 cd/m2, + * where 0x0001 represents 1 cd/m2 and 0xFFFF represents 65535 cd/m2. + */ __u16 max_cll; + /** + * @max_fall: Max Frame Average Light Level. + * This value is coded as an unsigned 16-bit value in units of 1 cd/m2, + * where 0x0001 represents 1 cd/m2 and 0xFFFF represents 65535 cd/m2. + */ __u16 max_fall; }; +/** + * struct hdr_output_metadata - HDR output metadata + * + * Metadata Information to be passed from userspace + */ struct hdr_output_metadata { + /** + * @metadata_type: Static_Metadata_Descriptor_ID. + */ __u32 metadata_type; + /** + * @hdmi_metadata_type1: HDR Metadata Infoframe. + */ union { struct hdr_metadata_infoframe hdmi_metadata_type1; }; -- cgit v1.2.3-59-g8ed1b From ed63bb1d1f8469586006a9ca63c42344401aa2ab Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Thu, 13 Jun 2019 15:34:06 -0700 Subject: dma-buf: give each buffer a full-fledged inode By traversing /proc/*/fd and /proc/*/map_files, processes with CAP_ADMIN can get a lot of fine-grained data about how shmem buffers are shared among processes. stat(2) on each entry gives the caller a unique ID (st_ino), the buffer's size (st_size), and even the number of pages currently charged to the buffer (st_blocks / 512). In contrast, all dma-bufs share the same anonymous inode. So while we can count how many dma-buf fds or mappings a process has, we can't get the size of the backing buffers or tell if two entries point to the same dma-buf. On systems with debugfs, we can get a per-buffer breakdown of size and reference count, but can't tell which processes are actually holding the references to each buffer. Replace the singleton inode with full-fledged inodes allocated by alloc_anon_inode(). This involves creating and mounting a mini-pseudo-filesystem for dma-buf, following the example in fs/aio.c. Signed-off-by: Greg Hackmann Signed-off-by: Chenbo Feng Signed-off-by: Sumit Semwal Link: https://patchwork.freedesktop.org/patch/msgid/20190613223408.139221-2-fengc@google.com --- drivers/dma-buf/dma-buf.c | 63 +++++++++++++++++++++++++++++++++++++++++----- include/uapi/linux/magic.h | 1 + 2 files changed, 58 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index f4104a21b069..3612ccededd6 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -34,8 +34,10 @@ #include #include #include +#include #include +#include static inline int is_dma_buf_file(struct file *); @@ -46,6 +48,25 @@ struct dma_buf_list { static struct dma_buf_list db_list; +static const struct dentry_operations dma_buf_dentry_ops = { + .d_dname = simple_dname, +}; + +static struct vfsmount *dma_buf_mnt; + +static struct dentry *dma_buf_fs_mount(struct file_system_type *fs_type, + int flags, const char *name, void *data) +{ + return mount_pseudo(fs_type, "dmabuf:", NULL, &dma_buf_dentry_ops, + DMA_BUF_MAGIC); +} + +static struct file_system_type dma_buf_fs_type = { + .name = "dmabuf", + .mount = dma_buf_fs_mount, + .kill_sb = kill_anon_super, +}; + static int dma_buf_release(struct inode *inode, struct file *file) { struct dma_buf *dmabuf; @@ -342,6 +363,31 @@ static inline int is_dma_buf_file(struct file *file) return file->f_op == &dma_buf_fops; } +static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) +{ + struct file *file; + struct inode *inode = alloc_anon_inode(dma_buf_mnt->mnt_sb); + + if (IS_ERR(inode)) + return ERR_CAST(inode); + + inode->i_size = dmabuf->size; + inode_set_bytes(inode, dmabuf->size); + + file = alloc_file_pseudo(inode, dma_buf_mnt, "dmabuf", + flags, &dma_buf_fops); + if (IS_ERR(file)) + goto err_alloc_file; + file->f_flags = flags & (O_ACCMODE | O_NONBLOCK); + file->private_data = dmabuf; + + return file; + +err_alloc_file: + iput(inode); + return file; +} + /** * DOC: dma buf device access * @@ -436,8 +482,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info) } dmabuf->resv = resv; - file = anon_inode_getfile("dmabuf", &dma_buf_fops, dmabuf, - exp_info->flags); + file = dma_buf_getfile(dmabuf, exp_info->flags); if (IS_ERR(file)) { ret = PTR_ERR(file); goto err_dmabuf; @@ -1055,8 +1100,8 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) return ret; seq_puts(s, "\nDma-buf Objects:\n"); - seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\n", - "size", "flags", "mode", "count"); + seq_printf(s, "%-8s\t%-8s\t%-8s\t%-8s\texp_name\t%-8s\n", + "size", "flags", "mode", "count", "ino"); list_for_each_entry(buf_obj, &db_list.head, list_node) { ret = mutex_lock_interruptible(&buf_obj->lock); @@ -1067,11 +1112,12 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) continue; } - seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\n", + seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\n", buf_obj->size, buf_obj->file->f_flags, buf_obj->file->f_mode, file_count(buf_obj->file), - buf_obj->exp_name); + buf_obj->exp_name, + file_inode(buf_obj->file)->i_ino); robj = buf_obj->resv; while (true) { @@ -1167,6 +1213,10 @@ static inline void dma_buf_uninit_debugfs(void) static int __init dma_buf_init(void) { + dma_buf_mnt = kern_mount(&dma_buf_fs_type); + if (IS_ERR(dma_buf_mnt)) + return PTR_ERR(dma_buf_mnt); + mutex_init(&db_list.lock); INIT_LIST_HEAD(&db_list.head); dma_buf_init_debugfs(); @@ -1177,5 +1227,6 @@ subsys_initcall(dma_buf_init); static void __exit dma_buf_deinit(void) { dma_buf_uninit_debugfs(); + kern_unmount(dma_buf_mnt); } __exitcall(dma_buf_deinit); diff --git a/include/uapi/linux/magic.h b/include/uapi/linux/magic.h index f8c00045d537..665e18627f78 100644 --- a/include/uapi/linux/magic.h +++ b/include/uapi/linux/magic.h @@ -91,5 +91,6 @@ #define UDF_SUPER_MAGIC 0x15013346 #define BALLOON_KVM_MAGIC 0x13661366 #define ZSMALLOC_MAGIC 0x58295829 +#define DMA_BUF_MAGIC 0x444d4142 /* "DMAB" */ #endif /* __LINUX_MAGIC_H__ */ -- cgit v1.2.3-59-g8ed1b From bb2bb903042517b8fb17b2bc21e00512f2dcac01 Mon Sep 17 00:00:00 2001 From: Greg Hackmann Date: Thu, 13 Jun 2019 15:34:07 -0700 Subject: dma-buf: add DMA_BUF_SET_NAME ioctls This patch adds complimentary DMA_BUF_SET_NAME ioctls, which lets userspace processes attach a free-form name to each buffer. This information can be extremely helpful for tracking and accounting shared buffers. For example, on Android, we know what each buffer will be used for at allocation time: GL, multimedia, camera, etc. The userspace allocator can use DMA_BUF_SET_NAME to associate that information with the buffer, so we can later give developers a breakdown of how much memory they're allocating for graphics, camera, etc. Signed-off-by: Greg Hackmann Signed-off-by: Chenbo Feng Signed-off-by: Sumit Semwal Link: https://patchwork.freedesktop.org/patch/msgid/20190613223408.139221-3-fengc@google.com --- drivers/dma-buf/dma-buf.c | 65 ++++++++++++++++++++++++++++++++++++++++++-- include/linux/dma-buf.h | 5 +++- include/uapi/linux/dma-buf.h | 3 ++ 3 files changed, 69 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 3612ccededd6..ab96410d1dcd 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -48,8 +48,24 @@ struct dma_buf_list { static struct dma_buf_list db_list; +static char *dmabuffs_dname(struct dentry *dentry, char *buffer, int buflen) +{ + struct dma_buf *dmabuf; + char name[DMA_BUF_NAME_LEN]; + size_t ret = 0; + + dmabuf = dentry->d_fsdata; + mutex_lock(&dmabuf->lock); + if (dmabuf->name) + ret = strlcpy(name, dmabuf->name, DMA_BUF_NAME_LEN); + mutex_unlock(&dmabuf->lock); + + return dynamic_dname(dentry, buffer, buflen, "/%s:%s", + dentry->d_name.name, ret > 0 ? name : ""); +} + static const struct dentry_operations dma_buf_dentry_ops = { - .d_dname = simple_dname, + .d_dname = dmabuffs_dname, }; static struct vfsmount *dma_buf_mnt; @@ -301,6 +317,43 @@ out: return events; } +/** + * dma_buf_set_name - Set a name to a specific dma_buf to track the usage. + * The name of the dma-buf buffer can only be set when the dma-buf is not + * attached to any devices. It could theoritically support changing the + * name of the dma-buf if the same piece of memory is used for multiple + * purpose between different devices. + * + * @dmabuf [in] dmabuf buffer that will be renamed. + * @buf: [in] A piece of userspace memory that contains the name of + * the dma-buf. + * + * Returns 0 on success. If the dma-buf buffer is already attached to + * devices, return -EBUSY. + * + */ +static long dma_buf_set_name(struct dma_buf *dmabuf, const char __user *buf) +{ + char *name = strndup_user(buf, DMA_BUF_NAME_LEN); + long ret = 0; + + if (IS_ERR(name)) + return PTR_ERR(name); + + mutex_lock(&dmabuf->lock); + if (!list_empty(&dmabuf->attachments)) { + ret = -EBUSY; + kfree(name); + goto out_unlock; + } + kfree(dmabuf->name); + dmabuf->name = name; + +out_unlock: + mutex_unlock(&dmabuf->lock); + return ret; +} + static long dma_buf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -339,6 +392,10 @@ static long dma_buf_ioctl(struct file *file, ret = dma_buf_begin_cpu_access(dmabuf, direction); return ret; + + case DMA_BUF_SET_NAME: + return dma_buf_set_name(dmabuf, (const char __user *)arg); + default: return -ENOTTY; } @@ -380,6 +437,7 @@ static struct file *dma_buf_getfile(struct dma_buf *dmabuf, int flags) goto err_alloc_file; file->f_flags = flags & (O_ACCMODE | O_NONBLOCK); file->private_data = dmabuf; + file->f_path.dentry->d_fsdata = dmabuf; return file; @@ -1112,12 +1170,13 @@ static int dma_buf_debug_show(struct seq_file *s, void *unused) continue; } - seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\n", + seq_printf(s, "%08zu\t%08x\t%08x\t%08ld\t%s\t%08lu\t%s\n", buf_obj->size, buf_obj->file->f_flags, buf_obj->file->f_mode, file_count(buf_obj->file), buf_obj->exp_name, - file_inode(buf_obj->file)->i_ino); + file_inode(buf_obj->file)->i_ino, + buf_obj->name ?: ""); robj = buf_obj->resv; while (true) { diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 8a327566d7f4..01ad5b942a6f 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -280,10 +280,12 @@ struct dma_buf_ops { * @file: file pointer used for sharing buffers across, and for refcounting. * @attachments: list of dma_buf_attachment that denotes all devices attached. * @ops: dma_buf_ops associated with this buffer object. - * @lock: used internally to serialize list manipulation, attach/detach and vmap/unmap + * @lock: used internally to serialize list manipulation, attach/detach and + * vmap/unmap, and accesses to name * @vmapping_counter: used internally to refcnt the vmaps * @vmap_ptr: the current vmap ptr if vmapping_counter > 0 * @exp_name: name of the exporter; useful for debugging. + * @name: userspace-provided name; useful for accounting and debugging. * @owner: pointer to exporter module; used for refcounting when exporter is a * kernel module. * @list_node: node for dma_buf accounting and debugging. @@ -311,6 +313,7 @@ struct dma_buf { unsigned vmapping_counter; void *vmap_ptr; const char *exp_name; + const char *name; struct module *owner; struct list_head list_node; void *priv; diff --git a/include/uapi/linux/dma-buf.h b/include/uapi/linux/dma-buf.h index d75df5210a4a..dbc7092e04b5 100644 --- a/include/uapi/linux/dma-buf.h +++ b/include/uapi/linux/dma-buf.h @@ -35,7 +35,10 @@ struct dma_buf_sync { #define DMA_BUF_SYNC_VALID_FLAGS_MASK \ (DMA_BUF_SYNC_RW | DMA_BUF_SYNC_END) +#define DMA_BUF_NAME_LEN 32 + #define DMA_BUF_BASE 'b' #define DMA_BUF_IOCTL_SYNC _IOW(DMA_BUF_BASE, 0, struct dma_buf_sync) +#define DMA_BUF_SET_NAME _IOW(DMA_BUF_BASE, 1, const char *) #endif -- cgit v1.2.3-59-g8ed1b From 4ba397671237784a212378c271f700e99c66cf39 Mon Sep 17 00:00:00 2001 From: Thomas Hellstrom Date: Tue, 19 Mar 2019 13:27:50 +0100 Subject: drm/vmwgfx: Add surface dirty-tracking callbacks Add the callbacks necessary to implement emulated coherent memory for surfaces. Add a flag to the gb_surface_create ioctl to indicate that surface memory should be coherent. Also bump the drm minor version to signal the availability of coherent surfaces. Signed-off-by: Thomas Hellstrom Reviewed-by: Deepak Rawat --- .../drm/vmwgfx/device_include/svga3d_surfacedefs.h | 233 +++++++++++- drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 4 +- drivers/gpu/drm/vmwgfx/vmwgfx_surface.c | 395 ++++++++++++++++++++- include/uapi/drm/vmwgfx_drm.h | 4 +- 4 files changed, 629 insertions(+), 7 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h index f2bfd3d80598..61414f105c67 100644 --- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h +++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h @@ -1280,7 +1280,6 @@ svga3dsurface_get_pixel_offset(SVGA3dSurfaceFormat format, return offset; } - static inline u32 svga3dsurface_get_image_offset(SVGA3dSurfaceFormat format, surf_size_struct baseLevelSize, @@ -1375,4 +1374,236 @@ svga3dsurface_is_screen_target_format(SVGA3dSurfaceFormat format) return svga3dsurface_is_dx_screen_target_format(format); } +/** + * struct svga3dsurface_mip - Mimpmap level information + * @bytes: Bytes required in the backing store of this mipmap level. + * @img_stride: Byte stride per image. + * @row_stride: Byte stride per block row. + * @size: The size of the mipmap. + */ +struct svga3dsurface_mip { + size_t bytes; + size_t img_stride; + size_t row_stride; + struct drm_vmw_size size; + +}; + +/** + * struct svga3dsurface_cache - Cached surface information + * @desc: Pointer to the surface descriptor + * @mip: Array of mipmap level information. Valid size is @num_mip_levels. + * @mip_chain_bytes: Bytes required in the backing store for the whole chain + * of mip levels. + * @sheet_bytes: Bytes required in the backing store for a sheet + * representing a single sample. + * @num_mip_levels: Valid size of the @mip array. Number of mipmap levels in + * a chain. + * @num_layers: Number of slices in an array texture or number of faces in + * a cubemap texture. + */ +struct svga3dsurface_cache { + const struct svga3d_surface_desc *desc; + struct svga3dsurface_mip mip[DRM_VMW_MAX_MIP_LEVELS]; + size_t mip_chain_bytes; + size_t sheet_bytes; + u32 num_mip_levels; + u32 num_layers; +}; + +/** + * struct svga3dsurface_loc - Surface location + * @sub_resource: Surface subresource. Defined as layer * num_mip_levels + + * mip_level. + * @x: X coordinate. + * @y: Y coordinate. + * @z: Z coordinate. + */ +struct svga3dsurface_loc { + u32 sub_resource; + u32 x, y, z; +}; + +/** + * svga3dsurface_subres - Compute the subresource from layer and mipmap. + * @cache: Surface layout data. + * @mip_level: The mipmap level. + * @layer: The surface layer (face or array slice). + * + * Return: The subresource. + */ +static inline u32 svga3dsurface_subres(const struct svga3dsurface_cache *cache, + u32 mip_level, u32 layer) +{ + return cache->num_mip_levels * layer + mip_level; +} + +/** + * svga3dsurface_setup_cache - Build a surface cache entry + * @size: The surface base level dimensions. + * @format: The surface format. + * @num_mip_levels: Number of mipmap levels. + * @num_layers: Number of layers. + * @cache: Pointer to a struct svga3dsurface_cach object to be filled in. + * + * Return: Zero on success, -EINVAL on invalid surface layout. + */ +static inline int svga3dsurface_setup_cache(const struct drm_vmw_size *size, + SVGA3dSurfaceFormat format, + u32 num_mip_levels, + u32 num_layers, + u32 num_samples, + struct svga3dsurface_cache *cache) +{ + const struct svga3d_surface_desc *desc; + u32 i; + + memset(cache, 0, sizeof(*cache)); + cache->desc = desc = svga3dsurface_get_desc(format); + cache->num_mip_levels = num_mip_levels; + cache->num_layers = num_layers; + for (i = 0; i < cache->num_mip_levels; i++) { + struct svga3dsurface_mip *mip = &cache->mip[i]; + + mip->size = svga3dsurface_get_mip_size(*size, i); + mip->bytes = svga3dsurface_get_image_buffer_size + (desc, &mip->size, 0); + mip->row_stride = + __KERNEL_DIV_ROUND_UP(mip->size.width, + desc->block_size.width) * + desc->bytes_per_block * num_samples; + if (!mip->row_stride) + goto invalid_dim; + + mip->img_stride = + __KERNEL_DIV_ROUND_UP(mip->size.height, + desc->block_size.height) * + mip->row_stride; + if (!mip->img_stride) + goto invalid_dim; + + cache->mip_chain_bytes += mip->bytes; + } + cache->sheet_bytes = cache->mip_chain_bytes * num_layers; + if (!cache->sheet_bytes) + goto invalid_dim; + + return 0; + +invalid_dim: + VMW_DEBUG_USER("Invalid surface layout for dirty tracking.\n"); + return -EINVAL; +} + +/** + * svga3dsurface_get_loc - Get a surface location from an offset into the + * backing store + * @cache: Surface layout data. + * @loc: Pointer to a struct svga3dsurface_loc to be filled in. + * @offset: Offset into the surface backing store. + */ +static inline void +svga3dsurface_get_loc(const struct svga3dsurface_cache *cache, + struct svga3dsurface_loc *loc, + size_t offset) +{ + const struct svga3dsurface_mip *mip = &cache->mip[0]; + const struct svga3d_surface_desc *desc = cache->desc; + u32 layer; + int i; + + if (offset >= cache->sheet_bytes) + offset %= cache->sheet_bytes; + + layer = offset / cache->mip_chain_bytes; + offset -= layer * cache->mip_chain_bytes; + for (i = 0; i < cache->num_mip_levels; ++i, ++mip) { + if (mip->bytes > offset) + break; + offset -= mip->bytes; + } + + loc->sub_resource = svga3dsurface_subres(cache, i, layer); + loc->z = offset / mip->img_stride; + offset -= loc->z * mip->img_stride; + loc->z *= desc->block_size.depth; + loc->y = offset / mip->row_stride; + offset -= loc->y * mip->row_stride; + loc->y *= desc->block_size.height; + loc->x = offset / desc->bytes_per_block; + loc->x *= desc->block_size.width; +} + +/** + * svga3dsurface_inc_loc - Clamp increment a surface location with one block + * size + * in each dimension. + * @loc: Pointer to a struct svga3dsurface_loc to be incremented. + * + * When computing the size of a range as size = end - start, the range does not + * include the end element. However a location representing the last byte + * of a touched region in the backing store *is* included in the range. + * This function modifies such a location to match the end definition + * given as start + size which is the one used in a SVGA3dBox. + */ +static inline void +svga3dsurface_inc_loc(const struct svga3dsurface_cache *cache, + struct svga3dsurface_loc *loc) +{ + const struct svga3d_surface_desc *desc = cache->desc; + u32 mip = loc->sub_resource % cache->num_mip_levels; + const struct drm_vmw_size *size = &cache->mip[mip].size; + + loc->sub_resource++; + loc->x += desc->block_size.width; + if (loc->x > size->width) + loc->x = size->width; + loc->y += desc->block_size.height; + if (loc->y > size->height) + loc->y = size->height; + loc->z += desc->block_size.depth; + if (loc->z > size->depth) + loc->z = size->depth; +} + +/** + * svga3dsurface_min_loc - The start location in a subresource + * @cache: Surface layout data. + * @sub_resource: The subresource. + * @loc: Pointer to a struct svga3dsurface_loc to be filled in. + */ +static inline void +svga3dsurface_min_loc(const struct svga3dsurface_cache *cache, + u32 sub_resource, + struct svga3dsurface_loc *loc) +{ + loc->sub_resource = sub_resource; + loc->x = loc->y = loc->z = 0; +} + +/** + * svga3dsurface_min_loc - The end location in a subresource + * @cache: Surface layout data. + * @sub_resource: The subresource. + * @loc: Pointer to a struct svga3dsurface_loc to be filled in. + * + * Following the end definition given in svga3dsurface_inc_loc(), + * Compute the end location of a surface subresource. + */ +static inline void +svga3dsurface_max_loc(const struct svga3dsurface_cache *cache, + u32 sub_resource, + struct svga3dsurface_loc *loc) +{ + const struct drm_vmw_size *size; + u32 mip; + + loc->sub_resource = sub_resource + 1; + mip = sub_resource % cache->num_mip_levels; + size = &cache->mip[mip].size; + loc->x = size->width; + loc->y = size->height; + loc->z = size->depth; +} + #endif /* _SVGA3D_SURFACEDEFS_H_ */ diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h index dae3a39bf402..5971c0d47507 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h @@ -44,9 +44,9 @@ #include #define VMWGFX_DRIVER_NAME "vmwgfx" -#define VMWGFX_DRIVER_DATE "20180704" +#define VMWGFX_DRIVER_DATE "20190328" #define VMWGFX_DRIVER_MAJOR 2 -#define VMWGFX_DRIVER_MINOR 15 +#define VMWGFX_DRIVER_MINOR 16 #define VMWGFX_DRIVER_PATCHLEVEL 0 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024) #define VMWGFX_MAX_RELOCATIONS 2048 diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c index c40d44f4d9af..637043f1befa 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c @@ -68,6 +68,20 @@ struct vmw_surface_offset { uint32_t bo_offset; }; +/** + * vmw_surface_dirty - Surface dirty-tracker + * @cache: Cached layout information of the surface. + * @size: Accounting size for the struct vmw_surface_dirty. + * @num_subres: Number of subresources. + * @boxes: Array of SVGA3dBoxes indicating dirty regions. One per subresource. + */ +struct vmw_surface_dirty { + struct svga3dsurface_cache cache; + size_t size; + u32 num_subres; + SVGA3dBox boxes[0]; +}; + static void vmw_user_surface_free(struct vmw_resource *res); static struct vmw_resource * vmw_user_surface_base_to_res(struct ttm_base_object *base); @@ -96,6 +110,13 @@ vmw_gb_surface_reference_internal(struct drm_device *dev, struct drm_vmw_gb_surface_ref_ext_rep *rep, struct drm_file *file_priv); +static void vmw_surface_dirty_free(struct vmw_resource *res); +static int vmw_surface_dirty_alloc(struct vmw_resource *res); +static int vmw_surface_dirty_sync(struct vmw_resource *res); +static void vmw_surface_dirty_range_add(struct vmw_resource *res, size_t start, + size_t end); +static int vmw_surface_clean(struct vmw_resource *res); + static const struct vmw_user_resource_conv user_surface_conv = { .object_type = VMW_RES_SURFACE, .base_obj_to_res = vmw_user_surface_base_to_res, @@ -133,7 +154,12 @@ static const struct vmw_res_func vmw_gb_surface_func = { .create = vmw_gb_surface_create, .destroy = vmw_gb_surface_destroy, .bind = vmw_gb_surface_bind, - .unbind = vmw_gb_surface_unbind + .unbind = vmw_gb_surface_unbind, + .dirty_alloc = vmw_surface_dirty_alloc, + .dirty_free = vmw_surface_dirty_free, + .dirty_sync = vmw_surface_dirty_sync, + .dirty_range_add = vmw_surface_dirty_range_add, + .clean = vmw_surface_clean, }; /** @@ -641,6 +667,7 @@ static void vmw_user_surface_free(struct vmw_resource *res) struct vmw_private *dev_priv = srf->res.dev_priv; uint32_t size = user_srf->size; + WARN_ON_ONCE(res->dirty); if (user_srf->master) drm_master_put(&user_srf->master); kfree(srf->offsets); @@ -1174,10 +1201,16 @@ static int vmw_gb_surface_bind(struct vmw_resource *res, cmd2->header.id = SVGA_3D_CMD_UPDATE_GB_SURFACE; cmd2->header.size = sizeof(cmd2->body); cmd2->body.sid = res->id; - res->backup_dirty = false; } vmw_fifo_commit(dev_priv, submit_size); + if (res->backup->dirty && res->backup_dirty) { + /* We've just made a full upload. Cear dirty regions. */ + vmw_bo_dirty_clear_res(res); + } + + res->backup_dirty = false; + return 0; } @@ -1642,7 +1675,8 @@ vmw_gb_surface_define_internal(struct drm_device *dev, } } } else if (req->base.drm_surface_flags & - drm_vmw_surface_flag_create_buffer) + (drm_vmw_surface_flag_create_buffer | + drm_vmw_surface_flag_coherent)) ret = vmw_user_bo_alloc(dev_priv, tfile, res->backup_size, req->base.drm_surface_flags & @@ -1656,6 +1690,26 @@ vmw_gb_surface_define_internal(struct drm_device *dev, goto out_unlock; } + if (req->base.drm_surface_flags & drm_vmw_surface_flag_coherent) { + struct vmw_buffer_object *backup = res->backup; + + ttm_bo_reserve(&backup->base, false, false, NULL); + if (!res->func->dirty_alloc) + ret = -EINVAL; + if (!ret) + ret = vmw_bo_dirty_add(backup); + if (!ret) { + res->coherent = true; + ret = res->func->dirty_alloc(res); + } + ttm_bo_unreserve(&backup->base); + if (ret) { + vmw_resource_unreference(&res); + goto out_unlock; + } + + } + tmp = vmw_resource_reference(res); ret = ttm_prime_object_init(tfile, res->backup_size, &user_srf->prime, req->base.drm_surface_flags & @@ -1764,3 +1818,338 @@ out_bad_resource: return ret; } + +/** + * vmw_subres_dirty_add - Add a dirty region to a subresource + * @dirty: The surfaces's dirty tracker. + * @loc_start: The location corresponding to the start of the region. + * @loc_end: The location corresponding to the end of the region. + * + * As we are assuming that @loc_start and @loc_end represent a sequential + * range of backing store memory, if the region spans multiple lines then + * regardless of the x coordinate, the full lines are dirtied. + * Correspondingly if the region spans multiple z slices, then full rather + * than partial z slices are dirtied. + */ +static void vmw_subres_dirty_add(struct vmw_surface_dirty *dirty, + const struct svga3dsurface_loc *loc_start, + const struct svga3dsurface_loc *loc_end) +{ + const struct svga3dsurface_cache *cache = &dirty->cache; + SVGA3dBox *box = &dirty->boxes[loc_start->sub_resource]; + u32 mip = loc_start->sub_resource % cache->num_mip_levels; + const struct drm_vmw_size *size = &cache->mip[mip].size; + u32 box_c2 = box->z + box->d; + + if (WARN_ON(loc_start->sub_resource >= dirty->num_subres)) + return; + + if (box->d == 0 || box->z > loc_start->z) + box->z = loc_start->z; + if (box_c2 < loc_end->z) + box->d = loc_end->z - box->z; + + if (loc_start->z + 1 == loc_end->z) { + box_c2 = box->y + box->h; + if (box->h == 0 || box->y > loc_start->y) + box->y = loc_start->y; + if (box_c2 < loc_end->y) + box->h = loc_end->y - box->y; + + if (loc_start->y + 1 == loc_end->y) { + box_c2 = box->x + box->w; + if (box->w == 0 || box->x > loc_start->x) + box->x = loc_start->x; + if (box_c2 < loc_end->x) + box->w = loc_end->x - box->x; + } else { + box->x = 0; + box->w = size->width; + } + } else { + box->y = 0; + box->h = size->height; + box->x = 0; + box->w = size->width; + } +} + +/** + * vmw_subres_dirty_full - Mark a full subresource as dirty + * @dirty: The surface's dirty tracker. + * @subres: The subresource + */ +static void vmw_subres_dirty_full(struct vmw_surface_dirty *dirty, u32 subres) +{ + const struct svga3dsurface_cache *cache = &dirty->cache; + u32 mip = subres % cache->num_mip_levels; + const struct drm_vmw_size *size = &cache->mip[mip].size; + SVGA3dBox *box = &dirty->boxes[subres]; + + box->x = 0; + box->y = 0; + box->z = 0; + box->w = size->width; + box->h = size->height; + box->d = size->depth; +} + +/* + * vmw_surface_tex_dirty_add_range - The dirty_add_range callback for texture + * surfaces. + */ +static void vmw_surface_tex_dirty_range_add(struct vmw_resource *res, + size_t start, size_t end) +{ + struct vmw_surface_dirty *dirty = + (struct vmw_surface_dirty *) res->dirty; + size_t backup_end = res->backup_offset + res->backup_size; + struct svga3dsurface_loc loc1, loc2; + const struct svga3dsurface_cache *cache; + + start = max_t(size_t, start, res->backup_offset) - res->backup_offset; + end = min(end, backup_end) - res->backup_offset; + cache = &dirty->cache; + svga3dsurface_get_loc(cache, &loc1, start); + svga3dsurface_get_loc(cache, &loc2, end - 1); + svga3dsurface_inc_loc(cache, &loc2); + + if (loc1.sub_resource + 1 == loc2.sub_resource) { + /* Dirty range covers a single sub-resource */ + vmw_subres_dirty_add(dirty, &loc1, &loc2); + } else { + /* Dirty range covers multiple sub-resources */ + struct svga3dsurface_loc loc_min, loc_max; + u32 sub_res = loc1.sub_resource; + + svga3dsurface_max_loc(cache, loc1.sub_resource, &loc_max); + vmw_subres_dirty_add(dirty, &loc1, &loc_max); + svga3dsurface_min_loc(cache, loc2.sub_resource - 1, &loc_min); + vmw_subres_dirty_add(dirty, &loc_min, &loc2); + for (sub_res = loc1.sub_resource + 1; + sub_res < loc2.sub_resource - 1; ++sub_res) + vmw_subres_dirty_full(dirty, sub_res); + } +} + +/* + * vmw_surface_tex_dirty_add_range - The dirty_add_range callback for buffer + * surfaces. + */ +static void vmw_surface_buf_dirty_range_add(struct vmw_resource *res, + size_t start, size_t end) +{ + struct vmw_surface_dirty *dirty = + (struct vmw_surface_dirty *) res->dirty; + const struct svga3dsurface_cache *cache = &dirty->cache; + size_t backup_end = res->backup_offset + cache->mip_chain_bytes; + SVGA3dBox *box = &dirty->boxes[0]; + u32 box_c2; + + box->h = box->d = 1; + start = max_t(size_t, start, res->backup_offset) - res->backup_offset; + end = min(end, backup_end) - res->backup_offset; + box_c2 = box->x + box->w; + if (box->w == 0 || box->x > start) + box->x = start; + if (box_c2 < end) + box->w = end - box->x; +} + +/* + * vmw_surface_tex_dirty_add_range - The dirty_add_range callback for surfaces + */ +static void vmw_surface_dirty_range_add(struct vmw_resource *res, size_t start, + size_t end) +{ + struct vmw_surface *srf = vmw_res_to_srf(res); + + if (WARN_ON(end <= res->backup_offset || + start >= res->backup_offset + res->backup_size)) + return; + + if (srf->format == SVGA3D_BUFFER) + vmw_surface_buf_dirty_range_add(res, start, end); + else + vmw_surface_tex_dirty_range_add(res, start, end); +} + +/* + * vmw_surface_dirty_sync - The surface's dirty_sync callback. + */ +static int vmw_surface_dirty_sync(struct vmw_resource *res) +{ + struct vmw_private *dev_priv = res->dev_priv; + bool has_dx = 0; + u32 i, num_dirty; + struct vmw_surface_dirty *dirty = + (struct vmw_surface_dirty *) res->dirty; + size_t alloc_size; + const struct svga3dsurface_cache *cache = &dirty->cache; + struct { + SVGA3dCmdHeader header; + SVGA3dCmdDXUpdateSubResource body; + } *cmd1; + struct { + SVGA3dCmdHeader header; + SVGA3dCmdUpdateGBImage body; + } *cmd2; + void *cmd; + + num_dirty = 0; + for (i = 0; i < dirty->num_subres; ++i) { + const SVGA3dBox *box = &dirty->boxes[i]; + + if (box->d) + num_dirty++; + } + + if (!num_dirty) + goto out; + + alloc_size = num_dirty * ((has_dx) ? sizeof(*cmd1) : sizeof(*cmd2)); + cmd = VMW_FIFO_RESERVE(dev_priv, alloc_size); + if (!cmd) + return -ENOMEM; + + cmd1 = cmd; + cmd2 = cmd; + + for (i = 0; i < dirty->num_subres; ++i) { + const SVGA3dBox *box = &dirty->boxes[i]; + + if (!box->d) + continue; + + /* + * DX_UPDATE_SUBRESOURCE is aware of array surfaces. + * UPDATE_GB_IMAGE is not. + */ + if (has_dx) { + cmd1->header.id = SVGA_3D_CMD_DX_UPDATE_SUBRESOURCE; + cmd1->header.size = sizeof(cmd1->body); + cmd1->body.sid = res->id; + cmd1->body.subResource = i; + cmd1->body.box = *box; + cmd1++; + } else { + cmd2->header.id = SVGA_3D_CMD_UPDATE_GB_IMAGE; + cmd2->header.size = sizeof(cmd2->body); + cmd2->body.image.sid = res->id; + cmd2->body.image.face = i / cache->num_mip_levels; + cmd2->body.image.mipmap = i - + (cache->num_mip_levels * cmd2->body.image.face); + cmd2->body.box = *box; + cmd2++; + } + + } + vmw_fifo_commit(dev_priv, alloc_size); + out: + memset(&dirty->boxes[0], 0, sizeof(dirty->boxes[0]) * + dirty->num_subres); + + return 0; +} + +/* + * vmw_surface_dirty_alloc - The surface's dirty_alloc callback. + */ +static int vmw_surface_dirty_alloc(struct vmw_resource *res) +{ + struct vmw_surface *srf = vmw_res_to_srf(res); + struct vmw_surface_dirty *dirty; + u32 num_layers = 1; + u32 num_mip; + u32 num_subres; + u32 num_samples; + size_t dirty_size, acc_size; + static struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false + }; + int ret; + + if (srf->array_size) + num_layers = srf->array_size; + else if (srf->flags & SVGA3D_SURFACE_CUBEMAP) + num_layers *= SVGA3D_MAX_SURFACE_FACES; + + num_mip = srf->mip_levels[0]; + if (!num_mip) + num_mip = 1; + + num_subres = num_layers * num_mip; + dirty_size = sizeof(*dirty) + num_subres * sizeof(dirty->boxes[0]); + acc_size = ttm_round_pot(dirty_size); + ret = ttm_mem_global_alloc(vmw_mem_glob(res->dev_priv), + acc_size, &ctx); + if (ret) { + VMW_DEBUG_USER("Out of graphics memory for surface " + "dirty tracker.\n"); + return ret; + } + + dirty = kvzalloc(dirty_size, GFP_KERNEL); + if (!dirty) { + ret = -ENOMEM; + goto out_no_dirty; + } + + num_samples = max_t(u32, 1, srf->multisample_count); + ret = svga3dsurface_setup_cache(&srf->base_size, srf->format, num_mip, + num_layers, num_samples, &dirty->cache); + if (ret) + goto out_no_cache; + + dirty->num_subres = num_subres; + dirty->size = acc_size; + res->dirty = (struct vmw_resource_dirty *) dirty; + + return 0; + +out_no_cache: + kvfree(dirty); +out_no_dirty: + ttm_mem_global_free(vmw_mem_glob(res->dev_priv), acc_size); + return ret; +} + +/* + * vmw_surface_dirty_free - The surface's dirty_free callback + */ +static void vmw_surface_dirty_free(struct vmw_resource *res) +{ + struct vmw_surface_dirty *dirty = + (struct vmw_surface_dirty *) res->dirty; + size_t acc_size = dirty->size; + + kvfree(dirty); + ttm_mem_global_free(vmw_mem_glob(res->dev_priv), acc_size); + res->dirty = NULL; +} + +/* + * vmw_surface_clean - The surface's clean callback + */ +static int vmw_surface_clean(struct vmw_resource *res) +{ + struct vmw_private *dev_priv = res->dev_priv; + size_t alloc_size; + struct { + SVGA3dCmdHeader header; + SVGA3dCmdReadbackGBSurface body; + } *cmd; + + alloc_size = sizeof(*cmd); + cmd = VMW_FIFO_RESERVE(dev_priv, alloc_size); + if (!cmd) + return -ENOMEM; + + cmd->header.id = SVGA_3D_CMD_READBACK_GB_SURFACE; + cmd->header.size = sizeof(cmd->body); + cmd->body.sid = res->id; + vmw_fifo_commit(dev_priv, alloc_size); + + return 0; +} diff --git a/include/uapi/drm/vmwgfx_drm.h b/include/uapi/drm/vmwgfx_drm.h index 399f58317cff..02cab33f2f25 100644 --- a/include/uapi/drm/vmwgfx_drm.h +++ b/include/uapi/drm/vmwgfx_drm.h @@ -891,11 +891,13 @@ struct drm_vmw_shader_arg { * surface. * @drm_vmw_surface_flag_create_buffer: Create a backup buffer if none is * given. + * @drm_vmw_surface_flag_coherent: Back surface with coherent memory. */ enum drm_vmw_surface_flags { drm_vmw_surface_flag_shareable = (1 << 0), drm_vmw_surface_flag_scanout = (1 << 1), - drm_vmw_surface_flag_create_buffer = (1 << 2) + drm_vmw_surface_flag_create_buffer = (1 << 2), + drm_vmw_surface_flag_coherent = (1 << 3), }; /** -- cgit v1.2.3-59-g8ed1b From 7786fd1087774c6090775932290f33ac73044f41 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Tue, 18 Jun 2019 10:16:48 +0200 Subject: drm/panfrost: Expose performance counters through unstable ioctls Expose performance counters through 2 driver specific ioctls: one to enable/disable the perfcnt block, and one to dump the counter values. There are discussions to expose global performance monitors (those counters that can't be retrieved on a per-job basis) in a consistent way, but this is likely to take time to settle on something that works for various HW/users. The ioctls are marked unstable so we can get rid of them when the time comes. We initally went for a debugfs-based interface, but this was making the transition to per-FD address space more complicated (we need to specify the namespace the GPU has to use when dumping the perf counters), hence the decision to switch back to driver specific ioctls which are passed the FD they operate on and thus will have a dedicated address space attached to them. Other than that, the implementation is pretty simple: it basically dumps all counters and copy the values to a userspace buffer. The parsing is left to userspace which has to know the specific layout that's used by the GPU (layout differs on a per-revision basis). Signed-off-by: Boris Brezillon Acked-by: Alyssa Rosenzweig Signed-off-by: Rob Herring Link: https://patchwork.freedesktop.org/patch/msgid/20190618081648.17297-5-boris.brezillon@collabora.com --- drivers/gpu/drm/panfrost/Makefile | 3 +- drivers/gpu/drm/panfrost/panfrost_device.c | 8 + drivers/gpu/drm/panfrost/panfrost_device.h | 3 + drivers/gpu/drm/panfrost/panfrost_drv.c | 4 + drivers/gpu/drm/panfrost/panfrost_gpu.c | 7 + drivers/gpu/drm/panfrost/panfrost_perfcnt.c | 329 ++++++++++++++++++++++++++++ drivers/gpu/drm/panfrost/panfrost_perfcnt.h | 18 ++ drivers/gpu/drm/panfrost/panfrost_regs.h | 19 ++ include/uapi/drm/panfrost_drm.h | 24 ++ 9 files changed, 414 insertions(+), 1 deletion(-) create mode 100644 drivers/gpu/drm/panfrost/panfrost_perfcnt.c create mode 100644 drivers/gpu/drm/panfrost/panfrost_perfcnt.h (limited to 'include/uapi') diff --git a/drivers/gpu/drm/panfrost/Makefile b/drivers/gpu/drm/panfrost/Makefile index 6de72d13c58f..ecf0864cb515 100644 --- a/drivers/gpu/drm/panfrost/Makefile +++ b/drivers/gpu/drm/panfrost/Makefile @@ -7,6 +7,7 @@ panfrost-y := \ panfrost_gem.o \ panfrost_gpu.o \ panfrost_job.o \ - panfrost_mmu.o + panfrost_mmu.o \ + panfrost_perfcnt.o obj-$(CONFIG_DRM_PANFROST) += panfrost.o diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c index ccb8eb2a518c..8a111d7c0200 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.c +++ b/drivers/gpu/drm/panfrost/panfrost_device.c @@ -14,6 +14,7 @@ #include "panfrost_gpu.h" #include "panfrost_job.h" #include "panfrost_mmu.h" +#include "panfrost_perfcnt.h" static int panfrost_reset_init(struct panfrost_device *pfdev) { @@ -171,7 +172,13 @@ int panfrost_device_init(struct panfrost_device *pfdev) pm_runtime_mark_last_busy(pfdev->dev); pm_runtime_put_autosuspend(pfdev->dev); + err = panfrost_perfcnt_init(pfdev); + if (err) + goto err_out5; + return 0; +err_out5: + panfrost_job_fini(pfdev); err_out4: panfrost_mmu_fini(pfdev); err_out3: @@ -187,6 +194,7 @@ err_out0: void panfrost_device_fini(struct panfrost_device *pfdev) { + panfrost_perfcnt_fini(pfdev); panfrost_job_fini(pfdev); panfrost_mmu_fini(pfdev); panfrost_gpu_fini(pfdev); diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index e86581c4af7b..83cc01cafde1 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -14,6 +14,7 @@ struct panfrost_device; struct panfrost_mmu; struct panfrost_job_slot; struct panfrost_job; +struct panfrost_perfcnt; #define NUM_JOB_SLOTS 3 @@ -78,6 +79,8 @@ struct panfrost_device { struct panfrost_job *jobs[NUM_JOB_SLOTS]; struct list_head scheduled_jobs; + struct panfrost_perfcnt *perfcnt; + struct mutex sched_lock; struct mutex reset_lock; diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index 754881ece8d7..e34e86a7378a 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -19,6 +19,7 @@ #include "panfrost_mmu.h" #include "panfrost_job.h" #include "panfrost_gpu.h" +#include "panfrost_perfcnt.h" static bool unstable_ioctls; module_param_unsafe(unstable_ioctls, bool, 0600); @@ -329,6 +330,7 @@ panfrost_postclose(struct drm_device *dev, struct drm_file *file) { struct panfrost_file_priv *panfrost_priv = file->driver_priv; + panfrost_perfcnt_close(panfrost_priv); panfrost_job_close(panfrost_priv); kfree(panfrost_priv); @@ -348,6 +350,8 @@ static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { PANFROST_IOCTL(MMAP_BO, mmap_bo, DRM_RENDER_ALLOW), PANFROST_IOCTL(GET_PARAM, get_param, DRM_RENDER_ALLOW), PANFROST_IOCTL(GET_BO_OFFSET, get_bo_offset, DRM_RENDER_ALLOW), + PANFROST_IOCTL(PERFCNT_ENABLE, perfcnt_enable, DRM_RENDER_ALLOW), + PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), }; DEFINE_DRM_GEM_SHMEM_FOPS(panfrost_drm_driver_fops); diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c index 6e68a100291c..20ab333fc925 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gpu.c +++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c @@ -15,6 +15,7 @@ #include "panfrost_features.h" #include "panfrost_issues.h" #include "panfrost_gpu.h" +#include "panfrost_perfcnt.h" #include "panfrost_regs.h" static irqreturn_t panfrost_gpu_irq_handler(int irq, void *data) @@ -40,6 +41,12 @@ static irqreturn_t panfrost_gpu_irq_handler(int irq, void *data) gpu_write(pfdev, GPU_INT_MASK, 0); } + if (state & GPU_IRQ_PERFCNT_SAMPLE_COMPLETED) + panfrost_perfcnt_sample_done(pfdev); + + if (state & GPU_IRQ_CLEAN_CACHES_COMPLETED) + panfrost_perfcnt_clean_cache_done(pfdev); + gpu_write(pfdev, GPU_INT_CLEAR, state); return IRQ_HANDLED; diff --git a/drivers/gpu/drm/panfrost/panfrost_perfcnt.c b/drivers/gpu/drm/panfrost/panfrost_perfcnt.c new file mode 100644 index 000000000000..83c57d325ca8 --- /dev/null +++ b/drivers/gpu/drm/panfrost/panfrost_perfcnt.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright 2019 Collabora Ltd */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "panfrost_device.h" +#include "panfrost_features.h" +#include "panfrost_gem.h" +#include "panfrost_issues.h" +#include "panfrost_job.h" +#include "panfrost_mmu.h" +#include "panfrost_regs.h" + +#define COUNTERS_PER_BLOCK 64 +#define BYTES_PER_COUNTER 4 +#define BLOCKS_PER_COREGROUP 8 +#define V4_SHADERS_PER_COREGROUP 4 + +struct panfrost_perfcnt { + struct panfrost_gem_object *bo; + size_t bosize; + void *buf; + struct panfrost_file_priv *user; + struct mutex lock; + struct completion dump_comp; +}; + +void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev) +{ + complete(&pfdev->perfcnt->dump_comp); +} + +void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev) +{ + gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES); +} + +static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev) +{ + u64 gpuva; + int ret; + + reinit_completion(&pfdev->perfcnt->dump_comp); + gpuva = pfdev->perfcnt->bo->node.start << PAGE_SHIFT; + gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva); + gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32); + gpu_write(pfdev, GPU_INT_CLEAR, + GPU_IRQ_CLEAN_CACHES_COMPLETED | + GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); + gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE); + ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp, + msecs_to_jiffies(1000)); + if (!ret) + ret = -ETIMEDOUT; + else if (ret > 0) + ret = 0; + + return ret; +} + +static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, + struct panfrost_file_priv *user, + unsigned int counterset) +{ + struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; + struct drm_gem_shmem_object *bo; + u32 cfg; + int ret; + + if (user == perfcnt->user) + return 0; + else if (perfcnt->user) + return -EBUSY; + + ret = pm_runtime_get_sync(pfdev->dev); + if (ret < 0) + return ret; + + bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + perfcnt->bo = to_panfrost_bo(&bo->base); + + /* Map the perfcnt buf in the address space attached to file_priv. */ + ret = panfrost_mmu_map(perfcnt->bo); + if (ret) + goto err_put_bo; + + perfcnt->buf = drm_gem_shmem_vmap(&bo->base); + if (IS_ERR(perfcnt->buf)) { + ret = PTR_ERR(perfcnt->buf); + goto err_put_bo; + } + + /* + * Invalidate the cache and clear the counters to start from a fresh + * state. + */ + reinit_completion(&pfdev->perfcnt->dump_comp); + gpu_write(pfdev, GPU_INT_CLEAR, + GPU_IRQ_CLEAN_CACHES_COMPLETED | + GPU_IRQ_PERFCNT_SAMPLE_COMPLETED); + gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR); + gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES); + ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp, + msecs_to_jiffies(1000)); + if (!ret) { + ret = -ETIMEDOUT; + goto err_vunmap; + } + + perfcnt->user = user; + + /* + * Always use address space 0 for now. + * FIXME: this needs to be updated when we start using different + * address space. + */ + cfg = GPU_PERFCNT_CFG_AS(0) | + GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL); + + /* + * Bifrost GPUs have 2 set of counters, but we're only interested by + * the first one for now. + */ + if (panfrost_model_is_bifrost(pfdev)) + cfg |= GPU_PERFCNT_CFG_SETSEL(counterset); + + gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff); + gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff); + gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff); + + /* + * Due to PRLAM-8186 we need to disable the Tiler before we enable HW + * counters. + */ + if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) + gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); + else + gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); + + gpu_write(pfdev, GPU_PERFCNT_CFG, cfg); + + if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186)) + gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff); + + return 0; + +err_vunmap: + drm_gem_shmem_vunmap(&perfcnt->bo->base.base, perfcnt->buf); +err_put_bo: + drm_gem_object_put_unlocked(&bo->base); + return ret; +} + +static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, + struct panfrost_file_priv *user) +{ + struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; + + if (user != perfcnt->user) + return -EINVAL; + + gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0); + gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0); + gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0); + gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); + gpu_write(pfdev, GPU_PERFCNT_CFG, + GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); + + perfcnt->user = NULL; + drm_gem_shmem_vunmap(&perfcnt->bo->base.base, perfcnt->buf); + perfcnt->buf = NULL; + drm_gem_object_put_unlocked(&perfcnt->bo->base.base); + perfcnt->bo = NULL; + pm_runtime_mark_last_busy(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->dev); + + return 0; +} + +int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct panfrost_file_priv *pfile = file_priv->driver_priv; + struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; + struct drm_panfrost_perfcnt_enable *req = data; + int ret; + + ret = panfrost_unstable_ioctl_check(); + if (ret) + return ret; + + /* Only Bifrost GPUs have 2 set of counters. */ + if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0)) + return -EINVAL; + + mutex_lock(&perfcnt->lock); + if (req->enable) + ret = panfrost_perfcnt_enable_locked(pfdev, pfile, + req->counterset); + else + ret = panfrost_perfcnt_disable_locked(pfdev, pfile); + mutex_unlock(&perfcnt->lock); + + return ret; +} + +int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; + struct drm_panfrost_perfcnt_dump *req = data; + void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr; + int ret; + + ret = panfrost_unstable_ioctl_check(); + if (ret) + return ret; + + mutex_lock(&perfcnt->lock); + if (perfcnt->user != file_priv->driver_priv) { + ret = -EINVAL; + goto out; + } + + ret = panfrost_perfcnt_dump_locked(pfdev); + if (ret) + goto out; + + if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize)) + ret = -EFAULT; + +out: + mutex_unlock(&perfcnt->lock); + + return ret; +} + +void panfrost_perfcnt_close(struct panfrost_file_priv *pfile) +{ + struct panfrost_device *pfdev = pfile->pfdev; + struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; + + pm_runtime_get_sync(pfdev->dev); + mutex_lock(&perfcnt->lock); + if (perfcnt->user == pfile) + panfrost_perfcnt_disable_locked(pfdev, pfile); + mutex_unlock(&perfcnt->lock); + pm_runtime_mark_last_busy(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->dev); +} + +int panfrost_perfcnt_init(struct panfrost_device *pfdev) +{ + struct panfrost_perfcnt *perfcnt; + size_t size; + + if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) { + unsigned int ncoregroups; + + ncoregroups = hweight64(pfdev->features.l2_present); + size = ncoregroups * BLOCKS_PER_COREGROUP * + COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; + } else { + unsigned int nl2c, ncores; + + /* + * TODO: define a macro to extract the number of l2 caches from + * mem_features. + */ + nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1; + + /* + * shader_present might be sparse, but the counters layout + * forces to dump unused regions too, hence the fls64() call + * instead of hweight64(). + */ + ncores = fls64(pfdev->features.shader_present); + + /* + * There's always one JM and one Tiler block, hence the '+ 2' + * here. + */ + size = (nl2c + ncores + 2) * + COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; + } + + perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL); + if (!perfcnt) + return -ENOMEM; + + perfcnt->bosize = size; + + /* Start with everything disabled. */ + gpu_write(pfdev, GPU_PERFCNT_CFG, + GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); + gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); + gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); + gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); + gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); + + init_completion(&perfcnt->dump_comp); + mutex_init(&perfcnt->lock); + pfdev->perfcnt = perfcnt; + + return 0; +} + +void panfrost_perfcnt_fini(struct panfrost_device *pfdev) +{ + /* Disable everything before leaving. */ + gpu_write(pfdev, GPU_PERFCNT_CFG, + GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); + gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0); + gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0); + gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0); + gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0); +} diff --git a/drivers/gpu/drm/panfrost/panfrost_perfcnt.h b/drivers/gpu/drm/panfrost/panfrost_perfcnt.h new file mode 100644 index 000000000000..13b8fdaa1b43 --- /dev/null +++ b/drivers/gpu/drm/panfrost/panfrost_perfcnt.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright 2019 Collabora Ltd */ +#ifndef __PANFROST_PERFCNT_H__ +#define __PANFROST_PERFCNT_H__ + +#include "panfrost_device.h" + +void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev); +void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev); +int panfrost_perfcnt_init(struct panfrost_device *pfdev); +void panfrost_perfcnt_fini(struct panfrost_device *pfdev); +void panfrost_perfcnt_close(struct panfrost_file_priv *pfile); +int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, + struct drm_file *file_priv); +int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +#endif diff --git a/drivers/gpu/drm/panfrost/panfrost_regs.h b/drivers/gpu/drm/panfrost/panfrost_regs.h index 42d08860fd76..ea38ac60581c 100644 --- a/drivers/gpu/drm/panfrost/panfrost_regs.h +++ b/drivers/gpu/drm/panfrost/panfrost_regs.h @@ -44,12 +44,31 @@ GPU_IRQ_MULTIPLE_FAULT) #define GPU_CMD 0x30 #define GPU_CMD_SOFT_RESET 0x01 +#define GPU_CMD_PERFCNT_CLEAR 0x03 +#define GPU_CMD_PERFCNT_SAMPLE 0x04 +#define GPU_CMD_CLEAN_CACHES 0x07 +#define GPU_CMD_CLEAN_INV_CACHES 0x08 #define GPU_STATUS 0x34 +#define GPU_STATUS_PRFCNT_ACTIVE BIT(2) #define GPU_LATEST_FLUSH_ID 0x38 #define GPU_FAULT_STATUS 0x3C #define GPU_FAULT_ADDRESS_LO 0x40 #define GPU_FAULT_ADDRESS_HI 0x44 +#define GPU_PERFCNT_BASE_LO 0x60 +#define GPU_PERFCNT_BASE_HI 0x64 +#define GPU_PERFCNT_CFG 0x68 +#define GPU_PERFCNT_CFG_MODE(x) (x) +#define GPU_PERFCNT_CFG_MODE_OFF 0 +#define GPU_PERFCNT_CFG_MODE_MANUAL 1 +#define GPU_PERFCNT_CFG_MODE_TILE 2 +#define GPU_PERFCNT_CFG_AS(x) ((x) << 4) +#define GPU_PERFCNT_CFG_SETSEL(x) ((x) << 8) +#define GPU_PRFCNT_JM_EN 0x6c +#define GPU_PRFCNT_SHADER_EN 0x70 +#define GPU_PRFCNT_TILER_EN 0x74 +#define GPU_PRFCNT_MMU_L2_EN 0x7c + #define GPU_THREAD_MAX_THREADS 0x0A0 /* (RO) Maximum number of threads per core */ #define GPU_THREAD_MAX_WORKGROUP_SIZE 0x0A4 /* (RO) Maximum workgroup size */ #define GPU_THREAD_MAX_BARRIER_SIZE 0x0A8 /* (RO) Maximum threads waiting at a barrier */ diff --git a/include/uapi/drm/panfrost_drm.h b/include/uapi/drm/panfrost_drm.h index a52e0283b90d..b5d370638846 100644 --- a/include/uapi/drm/panfrost_drm.h +++ b/include/uapi/drm/panfrost_drm.h @@ -18,6 +18,8 @@ extern "C" { #define DRM_PANFROST_MMAP_BO 0x03 #define DRM_PANFROST_GET_PARAM 0x04 #define DRM_PANFROST_GET_BO_OFFSET 0x05 +#define DRM_PANFROST_PERFCNT_ENABLE 0x06 +#define DRM_PANFROST_PERFCNT_DUMP 0x07 #define DRM_IOCTL_PANFROST_SUBMIT DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_SUBMIT, struct drm_panfrost_submit) #define DRM_IOCTL_PANFROST_WAIT_BO DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_WAIT_BO, struct drm_panfrost_wait_bo) @@ -26,6 +28,15 @@ extern "C" { #define DRM_IOCTL_PANFROST_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_GET_PARAM, struct drm_panfrost_get_param) #define DRM_IOCTL_PANFROST_GET_BO_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_PANFROST_GET_BO_OFFSET, struct drm_panfrost_get_bo_offset) +/* + * Unstable ioctl(s): only exposed when the unsafe unstable_ioctls module + * param is set to true. + * All these ioctl(s) are subject to deprecation, so please don't rely on + * them for anything but debugging purpose. + */ +#define DRM_IOCTL_PANFROST_PERFCNT_ENABLE DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_PERFCNT_ENABLE, struct drm_panfrost_perfcnt_enable) +#define DRM_IOCTL_PANFROST_PERFCNT_DUMP DRM_IOW(DRM_COMMAND_BASE + DRM_PANFROST_PERFCNT_DUMP, struct drm_panfrost_perfcnt_dump) + #define PANFROST_JD_REQ_FS (1 << 0) /** * struct drm_panfrost_submit - ioctl argument for submitting commands to the 3D @@ -135,6 +146,19 @@ struct drm_panfrost_get_bo_offset { __u64 offset; }; +struct drm_panfrost_perfcnt_enable { + __u32 enable; + /* + * On bifrost we have 2 sets of counters, this parameter defines the + * one to track. + */ + __u32 counterset; +}; + +struct drm_panfrost_perfcnt_dump { + __u64 buf_ptr; +}; + #if defined(__cplusplus) } #endif -- cgit v1.2.3-59-g8ed1b From 107c34bcbf4737b457ca44156aa11e788cbb6b83 Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Wed, 19 Jul 2017 09:43:26 +0800 Subject: drm/amdgpu: add NV series gpu family id Signed-off-by: Huang Rui Reviewed-by: Alex Deucher Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- include/uapi/drm/amdgpu_drm.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 4788730dbe78..571bf550859f 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -1044,6 +1044,7 @@ struct drm_amdgpu_info_vce_clock_table { #define AMDGPU_FAMILY_CZ 135 /* Carrizo, Stoney */ #define AMDGPU_FAMILY_AI 141 /* Vega10 */ #define AMDGPU_FAMILY_RV 142 /* Raven */ +#define AMDGPU_FAMILY_NV 143 /* Navi10 */ #if defined(__cplusplus) } -- cgit v1.2.3-59-g8ed1b From d67383e6b764957b39667af2754920e38a5eb43e Mon Sep 17 00:00:00 2001 From: Huang Rui Date: Tue, 18 Jul 2017 18:59:24 +0800 Subject: drm/amdgpu: add GDDR6 vram type New vram type. Signed-off-by: Huang Rui Reviewed-by: Alex Deucher Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- include/uapi/drm/amdgpu_drm.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index 571bf550859f..e1e5e7860570 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -912,6 +912,7 @@ struct drm_amdgpu_info_firmware { #define AMDGPU_VRAM_TYPE_HBM 6 #define AMDGPU_VRAM_TYPE_DDR3 7 #define AMDGPU_VRAM_TYPE_DDR4 8 +#define AMDGPU_VRAM_TYPE_GDDR6 9 struct drm_amdgpu_info_device { /** PCI Device ID */ -- cgit v1.2.3-59-g8ed1b From 22e96fa62ec66ca9d21a76b74601fc200908fec3 Mon Sep 17 00:00:00 2001 From: Hawking Zhang Date: Tue, 12 Jun 2018 18:30:04 +0800 Subject: drm/amdgpu: add pa_sc_tile_steering_override to drm_amdgpu_info_device the initial/default value of pa_sc_tile_steering_override need to be exposed to user mode driver Signed-off-by: Hawking Zhang Reviewed-by: Alex Deucher Signed-off-by: Alex Deucher --- include/uapi/drm/amdgpu_drm.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index e1e5e7860570..d799858b9e53 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -992,6 +992,8 @@ struct drm_amdgpu_info_device { __u64 high_va_offset; /** The maximum high virtual address */ __u64 high_va_max; + /* gfx10 pa_sc_tile_steering_override */ + __u32 pa_sc_tile_steering_override; }; struct drm_amdgpu_info_hw_ip { -- cgit v1.2.3-59-g8ed1b From 3729fe2bc2a01f4cc1aa88be8f64af06084c87d6 Mon Sep 17 00:00:00 2001 From: Dave Airlie Date: Tue, 16 Jul 2019 04:07:13 +1000 Subject: Revert "Merge branch 'vmwgfx-next' of git://people.freedesktop.org/~thomash/linux into drm-next" This reverts commit 031e610a6a21448a63dff7a0416e5e206724caac, reversing changes made to 52d2d44eee8091e740d0d275df1311fb8373c9a9. The mm changes in there we premature and not fully ack or reviewed by core mm folks, I dropped the ball by merging them via this tree, so lets take em all back out. Signed-off-by: Dave Airlie --- MAINTAINERS | 1 - drivers/gpu/drm/ttm/ttm_bo.c | 1 - drivers/gpu/drm/ttm/ttm_bo_vm.c | 169 +++----- drivers/gpu/drm/vmwgfx/Kconfig | 1 - drivers/gpu/drm/vmwgfx/Makefile | 2 +- .../drm/vmwgfx/device_include/svga3d_surfacedefs.h | 233 +--------- drivers/gpu/drm/vmwgfx/ttm_lock.c | 100 +++++ drivers/gpu/drm/vmwgfx/ttm_lock.h | 30 ++ drivers/gpu/drm/vmwgfx/vmwgfx_bo.c | 12 +- drivers/gpu/drm/vmwgfx/vmwgfx_context.c | 4 - drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c | 13 +- drivers/gpu/drm/vmwgfx/vmwgfx_drv.c | 167 +++++++- drivers/gpu/drm/vmwgfx/vmwgfx_drv.h | 139 ++---- drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c | 1 + drivers/gpu/drm/vmwgfx/vmwgfx_kms.c | 23 +- drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c | 472 --------------------- drivers/gpu/drm/vmwgfx/vmwgfx_resource.c | 245 ++--------- drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h | 15 - drivers/gpu/drm/vmwgfx/vmwgfx_shader.c | 8 +- drivers/gpu/drm/vmwgfx/vmwgfx_surface.c | 405 +----------------- drivers/gpu/drm/vmwgfx/vmwgfx_validation.c | 74 +--- drivers/gpu/drm/vmwgfx/vmwgfx_validation.h | 16 +- include/drm/ttm/ttm_bo_api.h | 10 - include/drm/ttm/ttm_bo_driver.h | 6 - include/linux/mm.h | 19 +- include/uapi/drm/vmwgfx_drm.h | 4 +- mm/Kconfig | 3 - mm/Makefile | 1 - mm/as_dirty_helpers.c | 300 ------------- mm/memory.c | 145 ++----- 30 files changed, 483 insertions(+), 2136 deletions(-) delete mode 100644 drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c delete mode 100644 mm/as_dirty_helpers.c (limited to 'include/uapi') diff --git a/MAINTAINERS b/MAINTAINERS index d6600715a662..2abf6d28db64 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5191,7 +5191,6 @@ T: git git://people.freedesktop.org/~thomash/linux S: Supported F: drivers/gpu/drm/vmwgfx/ F: include/uapi/drm/vmwgfx_drm.h -F: mm/as_dirty_helpers.c DRM DRIVERS M: David Airlie diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c index a7fd5a4955c9..58c403eda04e 100644 --- a/drivers/gpu/drm/ttm/ttm_bo.c +++ b/drivers/gpu/drm/ttm/ttm_bo.c @@ -1739,7 +1739,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, mutex_lock(&ttm_global_mutex); list_add_tail(&bdev->device_list, &glob->device_list); mutex_unlock(&ttm_global_mutex); - bdev->vm_ops = &ttm_bo_vm_ops; return 0; out_no_sys: diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c index 0c4576cbafcf..6dacff49c1cc 100644 --- a/drivers/gpu/drm/ttm/ttm_bo_vm.c +++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c @@ -42,6 +42,8 @@ #include #include +#define TTM_BO_VM_NUM_PREFAULT 16 + static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, struct vm_fault *vmf) { @@ -104,30 +106,25 @@ static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, + page_offset; } -/** - * ttm_bo_vm_reserve - Reserve a buffer object in a retryable vm callback - * @bo: The buffer object - * @vmf: The fault structure handed to the callback - * - * vm callbacks like fault() and *_mkwrite() allow for the mm_sem to be dropped - * during long waits, and after the wait the callback will be restarted. This - * is to allow other threads using the same virtual memory space concurrent - * access to map(), unmap() completely unrelated buffer objects. TTM buffer - * object reservations sometimes wait for GPU and should therefore be - * considered long waits. This function reserves the buffer object interruptibly - * taking this into account. Starvation is avoided by the vm system not - * allowing too many repeated restarts. - * This function is intended to be used in customized fault() and _mkwrite() - * handlers. - * - * Return: - * 0 on success and the bo was reserved. - * VM_FAULT_RETRY if blocking wait. - * VM_FAULT_NOPAGE if blocking wait and retrying was not allowed. - */ -vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, - struct vm_fault *vmf) +static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) { + struct vm_area_struct *vma = vmf->vma; + struct ttm_buffer_object *bo = (struct ttm_buffer_object *) + vma->vm_private_data; + struct ttm_bo_device *bdev = bo->bdev; + unsigned long page_offset; + unsigned long page_last; + unsigned long pfn; + struct ttm_tt *ttm = NULL; + struct page *page; + int err; + int i; + vm_fault_t ret = VM_FAULT_NOPAGE; + unsigned long address = vmf->address; + struct ttm_mem_type_manager *man = + &bdev->man[bo->mem.mem_type]; + struct vm_area_struct cvma; + /* * Work around locking order reversal in fault / nopfn * between mmap_sem and bo_reserve: Perform a trylock operation @@ -154,55 +151,14 @@ vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, return VM_FAULT_NOPAGE; } - return 0; -} -EXPORT_SYMBOL(ttm_bo_vm_reserve); - -/** - * ttm_bo_vm_fault_reserved - TTM fault helper - * @vmf: The struct vm_fault given as argument to the fault callback - * @prot: The page protection to be used for this memory area. - * @num_prefault: Maximum number of prefault pages. The caller may want to - * specify this based on madvice settings and the size of the GPU object - * backed by the memory. - * - * This function inserts one or more page table entries pointing to the - * memory backing the buffer object, and then returns a return code - * instructing the caller to retry the page access. - * - * Return: - * VM_FAULT_NOPAGE on success or pending signal - * VM_FAULT_SIGBUS on unspecified error - * VM_FAULT_OOM on out-of-memory - * VM_FAULT_RETRY if retryable wait - */ -vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, - pgprot_t prot, - pgoff_t num_prefault) -{ - struct vm_area_struct *vma = vmf->vma; - struct vm_area_struct cvma = *vma; - struct ttm_buffer_object *bo = (struct ttm_buffer_object *) - vma->vm_private_data; - struct ttm_bo_device *bdev = bo->bdev; - unsigned long page_offset; - unsigned long page_last; - unsigned long pfn; - struct ttm_tt *ttm = NULL; - struct page *page; - int err; - pgoff_t i; - vm_fault_t ret = VM_FAULT_NOPAGE; - unsigned long address = vmf->address; - struct ttm_mem_type_manager *man = - &bdev->man[bo->mem.mem_type]; - /* * Refuse to fault imported pages. This should be handled * (if at all) by redirecting mmap to the exporter. */ - if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) - return VM_FAULT_SIGBUS; + if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) { + ret = VM_FAULT_SIGBUS; + goto out_unlock; + } if (bdev->driver->fault_reserve_notify) { struct dma_fence *moving = dma_fence_get(bo->moving); @@ -213,9 +169,11 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, break; case -EBUSY: case -ERESTARTSYS: - return VM_FAULT_NOPAGE; + ret = VM_FAULT_NOPAGE; + goto out_unlock; default: - return VM_FAULT_SIGBUS; + ret = VM_FAULT_SIGBUS; + goto out_unlock; } if (bo->moving != moving) { @@ -231,12 +189,21 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, * move. */ ret = ttm_bo_vm_fault_idle(bo, vmf); - if (unlikely(ret != 0)) - return ret; + if (unlikely(ret != 0)) { + if (ret == VM_FAULT_RETRY && + !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { + /* The BO has already been unreserved. */ + return ret; + } + + goto out_unlock; + } err = ttm_mem_io_lock(man, true); - if (unlikely(err != 0)) - return VM_FAULT_NOPAGE; + if (unlikely(err != 0)) { + ret = VM_FAULT_NOPAGE; + goto out_unlock; + } err = ttm_mem_io_reserve_vm(bo); if (unlikely(err != 0)) { ret = VM_FAULT_SIGBUS; @@ -253,8 +220,18 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, goto out_io_unlock; } - cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, prot); - if (!bo->mem.bus.is_iomem) { + /* + * Make a local vma copy to modify the page_prot member + * and vm_flags if necessary. The vma parameter is protected + * by mmap_sem in write mode. + */ + cvma = *vma; + cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags); + + if (bo->mem.bus.is_iomem) { + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); + } else { struct ttm_operation_ctx ctx = { .interruptible = false, .no_wait_gpu = false, @@ -263,21 +240,24 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, }; ttm = bo->ttm; - if (ttm_tt_populate(bo->ttm, &ctx)) { + cvma.vm_page_prot = ttm_io_prot(bo->mem.placement, + cvma.vm_page_prot); + + /* Allocate all page at once, most common usage */ + if (ttm_tt_populate(ttm, &ctx)) { ret = VM_FAULT_OOM; goto out_io_unlock; } - } else { - /* Iomem should not be marked encrypted */ - cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); } /* * Speculatively prefault a number of pages. Only error on * first page. */ - for (i = 0; i < num_prefault; ++i) { + for (i = 0; i < TTM_BO_VM_NUM_PREFAULT; ++i) { if (bo->mem.bus.is_iomem) { + /* Iomem should not be marked encrypted */ + cvma.vm_page_prot = pgprot_decrypted(cvma.vm_page_prot); pfn = ttm_bo_io_mem_pfn(bo, page_offset); } else { page = ttm->pages[page_offset]; @@ -315,26 +295,7 @@ vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, ret = VM_FAULT_NOPAGE; out_io_unlock: ttm_mem_io_unlock(man); - return ret; -} -EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); - -static vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - pgprot_t prot; - struct ttm_buffer_object *bo = vma->vm_private_data; - vm_fault_t ret; - - ret = ttm_bo_vm_reserve(bo, vmf); - if (ret) - return ret; - - prot = vm_get_page_prot(vma->vm_flags); - ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT); - if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - return ret; - +out_unlock: reservation_object_unlock(bo->resv); return ret; } @@ -434,7 +395,7 @@ static int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, return ret; } -const struct vm_operations_struct ttm_bo_vm_ops = { +static const struct vm_operations_struct ttm_bo_vm_ops = { .fault = ttm_bo_vm_fault, .open = ttm_bo_vm_open, .close = ttm_bo_vm_close, @@ -487,7 +448,7 @@ int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, if (unlikely(ret != 0)) goto out_unref; - vma->vm_ops = bdev->vm_ops; + vma->vm_ops = &ttm_bo_vm_ops; /* * Note: We're transferring the bo reference to @@ -519,7 +480,7 @@ int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) ttm_bo_get(bo); - vma->vm_ops = bo->bdev->vm_ops; + vma->vm_ops = &ttm_bo_vm_ops; vma->vm_private_data = bo; vma->vm_flags |= VM_MIXEDMAP; vma->vm_flags |= VM_IO | VM_DONTEXPAND; diff --git a/drivers/gpu/drm/vmwgfx/Kconfig b/drivers/gpu/drm/vmwgfx/Kconfig index d5fd81a521f6..6b28a326f8bb 100644 --- a/drivers/gpu/drm/vmwgfx/Kconfig +++ b/drivers/gpu/drm/vmwgfx/Kconfig @@ -8,7 +8,6 @@ config DRM_VMWGFX select FB_CFB_IMAGEBLIT select DRM_TTM select FB - select AS_DIRTY_HELPERS # Only needed for the transitional use of drm_crtc_init - can be removed # again once vmwgfx sets up the primary plane itself. select DRM_KMS_HELPER diff --git a/drivers/gpu/drm/vmwgfx/Makefile b/drivers/gpu/drm/vmwgfx/Makefile index c877a21a0739..8841bd30e1e5 100644 --- a/drivers/gpu/drm/vmwgfx/Makefile +++ b/drivers/gpu/drm/vmwgfx/Makefile @@ -8,7 +8,7 @@ vmwgfx-y := vmwgfx_execbuf.o vmwgfx_gmr.o vmwgfx_kms.o vmwgfx_drv.o \ vmwgfx_cmdbuf_res.o vmwgfx_cmdbuf.o vmwgfx_stdu.o \ vmwgfx_cotable.o vmwgfx_so.o vmwgfx_binding.o vmwgfx_msg.o \ vmwgfx_simple_resource.o vmwgfx_va.o vmwgfx_blit.o \ - vmwgfx_validation.o vmwgfx_page_dirty.o \ + vmwgfx_validation.o \ ttm_object.o ttm_lock.o obj-$(CONFIG_DRM_VMWGFX) := vmwgfx.o diff --git a/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h b/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h index 61414f105c67..f2bfd3d80598 100644 --- a/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h +++ b/drivers/gpu/drm/vmwgfx/device_include/svga3d_surfacedefs.h @@ -1280,6 +1280,7 @@ svga3dsurface_get_pixel_offset(SVGA3dSurfaceFormat format, return offset; } + static inline u32 svga3dsurface_get_image_offset(SVGA3dSurfaceFormat format, surf_size_struct baseLevelSize, @@ -1374,236 +1375,4 @@ svga3dsurface_is_screen_target_format(SVGA3dSurfaceFormat format) return svga3dsurface_is_dx_screen_target_format(format); } -/** - * struct svga3dsurface_mip - Mimpmap level information - * @bytes: Bytes required in the backing store of this mipmap level. - * @img_stride: Byte stride per image. - * @row_stride: Byte stride per block row. - * @size: The size of the mipmap. - */ -struct svga3dsurface_mip { - size_t bytes; - size_t img_stride; - size_t row_stride; - struct drm_vmw_size size; - -}; - -/** - * struct svga3dsurface_cache - Cached surface information - * @desc: Pointer to the surface descriptor - * @mip: Array of mipmap level information. Valid size is @num_mip_levels. - * @mip_chain_bytes: Bytes required in the backing store for the whole chain - * of mip levels. - * @sheet_bytes: Bytes required in the backing store for a sheet - * representing a single sample. - * @num_mip_levels: Valid size of the @mip array. Number of mipmap levels in - * a chain. - * @num_layers: Number of slices in an array texture or number of faces in - * a cubemap texture. - */ -struct svga3dsurface_cache { - const struct svga3d_surface_desc *desc; - struct svga3dsurface_mip mip[DRM_VMW_MAX_MIP_LEVELS]; - size_t mip_chain_bytes; - size_t sheet_bytes; - u32 num_mip_levels; - u32 num_layers; -}; - -/** - * struct svga3dsurface_loc - Surface location - * @sub_resource: Surface subresource. Defined as layer * num_mip_levels + - * mip_level. - * @x: X coordinate. - * @y: Y coordinate. - * @z: Z coordinate. - */ -struct svga3dsurface_loc { - u32 sub_resource; - u32 x, y, z; -}; - -/** - * svga3dsurface_subres - Compute the subresource from layer and mipmap. - * @cache: Surface layout data. - * @mip_level: The mipmap level. - * @layer: The surface layer (face or array slice). - * - * Return: The subresource. - */ -static inline u32 svga3dsurface_subres(const struct svga3dsurface_cache *cache, - u32 mip_level, u32 layer) -{ - return cache->num_mip_levels * layer + mip_level; -} - -/** - * svga3dsurface_setup_cache - Build a surface cache entry - * @size: The surface base level dimensions. - * @format: The surface format. - * @num_mip_levels: Number of mipmap levels. - * @num_layers: Number of layers. - * @cache: Pointer to a struct svga3dsurface_cach object to be filled in. - * - * Return: Zero on success, -EINVAL on invalid surface layout. - */ -static inline int svga3dsurface_setup_cache(const struct drm_vmw_size *size, - SVGA3dSurfaceFormat format, - u32 num_mip_levels, - u32 num_layers, - u32 num_samples, - struct svga3dsurface_cache *cache) -{ - const struct svga3d_surface_desc *desc; - u32 i; - - memset(cache, 0, sizeof(*cache)); - cache->desc = desc = svga3dsurface_get_desc(format); - cache->num_mip_levels = num_mip_levels; - cache->num_layers = num_layers; - for (i = 0; i < cache->num_mip_levels; i++) { - struct svga3dsurface_mip *mip = &cache->mip[i]; - - mip->size = svga3dsurface_get_mip_size(*size, i); - mip->bytes = svga3dsurface_get_image_buffer_size - (desc, &mip->size, 0); - mip->row_stride = - __KERNEL_DIV_ROUND_UP(mip->size.width, - desc->block_size.width) * - desc->bytes_per_block * num_samples; - if (!mip->row_stride) - goto invalid_dim; - - mip->img_stride = - __KERNEL_DIV_ROUND_UP(mip->size.height, - desc->block_size.height) * - mip->row_stride; - if (!mip->img_stride) - goto invalid_dim; - - cache->mip_chain_bytes += mip->bytes; - } - cache->sheet_bytes = cache->mip_chain_bytes * num_layers; - if (!cache->sheet_bytes) - goto invalid_dim; - - return 0; - -invalid_dim: - VMW_DEBUG_USER("Invalid surface layout for dirty tracking.\n"); - return -EINVAL; -} - -/** - * svga3dsurface_get_loc - Get a surface location from an offset into the - * backing store - * @cache: Surface layout data. - * @loc: Pointer to a struct svga3dsurface_loc to be filled in. - * @offset: Offset into the surface backing store. - */ -static inline void -svga3dsurface_get_loc(const struct svga3dsurface_cache *cache, - struct svga3dsurface_loc *loc, - size_t offset) -{ - const struct svga3dsurface_mip *mip = &cache->mip[0]; - const struct svga3d_surface_desc *desc = cache->desc; - u32 layer; - int i; - - if (offset >= cache->sheet_bytes) - offset %= cache->sheet_bytes; - - layer = offset / cache->mip_chain_bytes; - offset -= layer * cache->mip_chain_bytes; - for (i = 0; i < cache->num_mip_levels; ++i, ++mip) { - if (mip->bytes > offset) - break; - offset -= mip->bytes; - } - - loc->sub_resource = svga3dsurface_subres(cache, i, layer); - loc->z = offset / mip->img_stride; - offset -= loc->z * mip->img_stride; - loc->z *= desc->block_size.depth; - loc->y = offset / mip->row_stride; - offset -= loc->y * mip->row_stride; - loc->y *= desc->block_size.height; - loc->x = offset / desc->bytes_per_block; - loc->x *= desc->block_size.width; -} - -/** - * svga3dsurface_inc_loc - Clamp increment a surface location with one block - * size - * in each dimension. - * @loc: Pointer to a struct svga3dsurface_loc to be incremented. - * - * When computing the size of a range as size = end - start, the range does not - * include the end element. However a location representing the last byte - * of a touched region in the backing store *is* included in the range. - * This function modifies such a location to match the end definition - * given as start + size which is the one used in a SVGA3dBox. - */ -static inline void -svga3dsurface_inc_loc(const struct svga3dsurface_cache *cache, - struct svga3dsurface_loc *loc) -{ - const struct svga3d_surface_desc *desc = cache->desc; - u32 mip = loc->sub_resource % cache->num_mip_levels; - const struct drm_vmw_size *size = &cache->mip[mip].size; - - loc->sub_resource++; - loc->x += desc->block_size.width; - if (loc->x > size->width) - loc->x = size->width; - loc->y += desc->block_size.height; - if (loc->y > size->height) - loc->y = size->height; - loc->z += desc->block_size.depth; - if (loc->z > size->depth) - loc->z = size->depth; -} - -/** - * svga3dsurface_min_loc - The start location in a subresource - * @cache: Surface layout data. - * @sub_resource: The subresource. - * @loc: Pointer to a struct svga3dsurface_loc to be filled in. - */ -static inline void -svga3dsurface_min_loc(const struct svga3dsurface_cache *cache, - u32 sub_resource, - struct svga3dsurface_loc *loc) -{ - loc->sub_resource = sub_resource; - loc->x = loc->y = loc->z = 0; -} - -/** - * svga3dsurface_min_loc - The end location in a subresource - * @cache: Surface layout data. - * @sub_resource: The subresource. - * @loc: Pointer to a struct svga3dsurface_loc to be filled in. - * - * Following the end definition given in svga3dsurface_inc_loc(), - * Compute the end location of a surface subresource. - */ -static inline void -svga3dsurface_max_loc(const struct svga3dsurface_cache *cache, - u32 sub_resource, - struct svga3dsurface_loc *loc) -{ - const struct drm_vmw_size *size; - u32 mip; - - loc->sub_resource = sub_resource + 1; - mip = sub_resource % cache->num_mip_levels; - size = &cache->mip[mip].size; - loc->x = size->width; - loc->y = size->height; - loc->z = size->depth; -} - #endif /* _SVGA3D_SURFACEDEFS_H_ */ diff --git a/drivers/gpu/drm/vmwgfx/ttm_lock.c b/drivers/gpu/drm/vmwgfx/ttm_lock.c index 5971c72e6d10..16b2083cb9d4 100644 --- a/drivers/gpu/drm/vmwgfx/ttm_lock.c +++ b/drivers/gpu/drm/vmwgfx/ttm_lock.c @@ -29,6 +29,7 @@ * Authors: Thomas Hellstrom */ +#include #include #include #include @@ -48,6 +49,8 @@ void ttm_lock_init(struct ttm_lock *lock) init_waitqueue_head(&lock->queue); lock->rw = 0; lock->flags = 0; + lock->kill_takers = false; + lock->signal = SIGKILL; } void ttm_read_unlock(struct ttm_lock *lock) @@ -63,6 +66,11 @@ static bool __ttm_read_lock(struct ttm_lock *lock) bool locked = false; spin_lock(&lock->lock); + if (unlikely(lock->kill_takers)) { + send_sig(lock->signal, current, 0); + spin_unlock(&lock->lock); + return false; + } if (lock->rw >= 0 && lock->flags == 0) { ++lock->rw; locked = true; @@ -90,6 +98,11 @@ static bool __ttm_read_trylock(struct ttm_lock *lock, bool *locked) *locked = false; spin_lock(&lock->lock); + if (unlikely(lock->kill_takers)) { + send_sig(lock->signal, current, 0); + spin_unlock(&lock->lock); + return false; + } if (lock->rw >= 0 && lock->flags == 0) { ++lock->rw; block = false; @@ -134,6 +147,11 @@ static bool __ttm_write_lock(struct ttm_lock *lock) bool locked = false; spin_lock(&lock->lock); + if (unlikely(lock->kill_takers)) { + send_sig(lock->signal, current, 0); + spin_unlock(&lock->lock); + return false; + } if (lock->rw == 0 && ((lock->flags & ~TTM_WRITE_LOCK_PENDING) == 0)) { lock->rw = -1; lock->flags &= ~TTM_WRITE_LOCK_PENDING; @@ -164,6 +182,88 @@ int ttm_write_lock(struct ttm_lock *lock, bool interruptible) return ret; } +static int __ttm_vt_unlock(struct ttm_lock *lock) +{ + int ret = 0; + + spin_lock(&lock->lock); + if (unlikely(!(lock->flags & TTM_VT_LOCK))) + ret = -EINVAL; + lock->flags &= ~TTM_VT_LOCK; + wake_up_all(&lock->queue); + spin_unlock(&lock->lock); + + return ret; +} + +static void ttm_vt_lock_remove(struct ttm_base_object **p_base) +{ + struct ttm_base_object *base = *p_base; + struct ttm_lock *lock = container_of(base, struct ttm_lock, base); + int ret; + + *p_base = NULL; + ret = __ttm_vt_unlock(lock); + BUG_ON(ret != 0); +} + +static bool __ttm_vt_lock(struct ttm_lock *lock) +{ + bool locked = false; + + spin_lock(&lock->lock); + if (lock->rw == 0) { + lock->flags &= ~TTM_VT_LOCK_PENDING; + lock->flags |= TTM_VT_LOCK; + locked = true; + } else { + lock->flags |= TTM_VT_LOCK_PENDING; + } + spin_unlock(&lock->lock); + return locked; +} + +int ttm_vt_lock(struct ttm_lock *lock, + bool interruptible, + struct ttm_object_file *tfile) +{ + int ret = 0; + + if (interruptible) { + ret = wait_event_interruptible(lock->queue, + __ttm_vt_lock(lock)); + if (unlikely(ret != 0)) { + spin_lock(&lock->lock); + lock->flags &= ~TTM_VT_LOCK_PENDING; + wake_up_all(&lock->queue); + spin_unlock(&lock->lock); + return ret; + } + } else + wait_event(lock->queue, __ttm_vt_lock(lock)); + + /* + * Add a base-object, the destructor of which will + * make sure the lock is released if the client dies + * while holding it. + */ + + ret = ttm_base_object_init(tfile, &lock->base, false, + ttm_lock_type, &ttm_vt_lock_remove, NULL); + if (ret) + (void)__ttm_vt_unlock(lock); + else + lock->vt_holder = tfile; + + return ret; +} + +int ttm_vt_unlock(struct ttm_lock *lock) +{ + return ttm_ref_object_base_unref(lock->vt_holder, + lock->base.handle, TTM_REF_USAGE); +} + void ttm_suspend_unlock(struct ttm_lock *lock) { spin_lock(&lock->lock); diff --git a/drivers/gpu/drm/vmwgfx/ttm_lock.h b/drivers/gpu/drm/vmwgfx/ttm_lock.h index 3d454e8b491f..0c3af9836863 100644 --- a/drivers/gpu/drm/vmwgfx/ttm_lock.h +++ b/drivers/gpu/drm/vmwgfx/ttm_lock.h @@ -63,6 +63,8 @@ * @lock: Spinlock protecting some lock members. * @rw: Read-write lock counter. Protected by @lock. * @flags: Lock state. Protected by @lock. + * @kill_takers: Boolean whether to kill takers of the lock. + * @signal: Signal to send when kill_takers is true. */ struct ttm_lock { @@ -71,6 +73,9 @@ struct ttm_lock { spinlock_t lock; int32_t rw; uint32_t flags; + bool kill_takers; + int signal; + struct ttm_object_file *vt_holder; }; @@ -215,4 +220,29 @@ extern void ttm_write_unlock(struct ttm_lock *lock); */ extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible); +/** + * ttm_lock_set_kill + * + * @lock: Pointer to a struct ttm_lock + * @val: Boolean whether to kill processes taking the lock. + * @signal: Signal to send to the process taking the lock. + * + * The kill-when-taking-lock functionality is used to kill processes that keep + * on using the TTM functionality when its resources has been taken down, for + * example when the X server exits. A typical sequence would look like this: + * - X server takes lock in write mode. + * - ttm_lock_set_kill() is called with @val set to true. + * - As part of X server exit, TTM resources are taken down. + * - X server releases the lock on file release. + * - Another dri client wants to render, takes the lock and is killed. + * + */ +static inline void ttm_lock_set_kill(struct ttm_lock *lock, bool val, + int signal) +{ + lock->kill_takers = val; + if (val) + lock->signal = signal; +} + #endif diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c index e8bc7a7ac031..5d5c2bce01f3 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_bo.c @@ -463,8 +463,6 @@ void vmw_bo_bo_free(struct ttm_buffer_object *bo) { struct vmw_buffer_object *vmw_bo = vmw_buffer_object(bo); - WARN_ON(vmw_bo->dirty); - WARN_ON(!RB_EMPTY_ROOT(&vmw_bo->res_tree)); vmw_bo_unmap(vmw_bo); kfree(vmw_bo); } @@ -478,11 +476,8 @@ void vmw_bo_bo_free(struct ttm_buffer_object *bo) static void vmw_user_bo_destroy(struct ttm_buffer_object *bo) { struct vmw_user_buffer_object *vmw_user_bo = vmw_user_buffer_object(bo); - struct vmw_buffer_object *vbo = &vmw_user_bo->vbo; - WARN_ON(vbo->dirty); - WARN_ON(!RB_EMPTY_ROOT(&vbo->res_tree)); - vmw_bo_unmap(vbo); + vmw_bo_unmap(&vmw_user_bo->vbo); ttm_prime_object_kfree(vmw_user_bo, prime); } @@ -515,9 +510,8 @@ int vmw_bo_init(struct vmw_private *dev_priv, acc_size = vmw_bo_acc_size(dev_priv, size, user); memset(vmw_bo, 0, sizeof(*vmw_bo)); - BUILD_BUG_ON(TTM_MAX_BO_PRIORITY <= 3); - vmw_bo->base.priority = 3; - vmw_bo->res_tree = RB_ROOT; + + INIT_LIST_HEAD(&vmw_bo->res_list); ret = ttm_bo_init(bdev, &vmw_bo->base, size, ttm_bo_type_device, placement, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c index a56c9d802382..63f111068a44 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c @@ -88,8 +88,6 @@ static const struct vmw_res_func vmw_gb_context_func = { .res_type = vmw_res_context, .needs_backup = true, .may_evict = true, - .prio = 3, - .dirty_prio = 3, .type_name = "guest backed contexts", .backup_placement = &vmw_mob_placement, .create = vmw_gb_context_create, @@ -102,8 +100,6 @@ static const struct vmw_res_func vmw_dx_context_func = { .res_type = vmw_res_dx_context, .needs_backup = true, .may_evict = true, - .prio = 3, - .dirty_prio = 3, .type_name = "dx contexts", .backup_placement = &vmw_mob_placement, .create = vmw_dx_context_create, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c index 8c699cb2565b..b4f6e1217c9d 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cotable.c @@ -116,8 +116,6 @@ static const struct vmw_res_func vmw_cotable_func = { .res_type = vmw_res_cotable, .needs_backup = true, .may_evict = true, - .prio = 3, - .dirty_prio = 3, .type_name = "context guest backed object tables", .backup_placement = &vmw_mob_placement, .create = vmw_cotable_create, @@ -309,7 +307,7 @@ static int vmw_cotable_unbind(struct vmw_resource *res, struct ttm_buffer_object *bo = val_buf->bo; struct vmw_fence_obj *fence; - if (!vmw_resource_mob_attached(res)) + if (list_empty(&res->mob_head)) return 0; WARN_ON_ONCE(bo->mem.mem_type != VMW_PL_MOB); @@ -455,7 +453,6 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size) goto out_wait; } - vmw_resource_mob_detach(res); res->backup = buf; res->backup_size = new_size; vcotbl->size_read_back = cur_size_read_back; @@ -470,12 +467,12 @@ static int vmw_cotable_resize(struct vmw_resource *res, size_t new_size) res->backup = old_buf; res->backup_size = old_size; vcotbl->size_read_back = old_size_read_back; - vmw_resource_mob_attach(res); goto out_wait; } - vmw_resource_mob_attach(res); /* Let go of the old mob. */ + list_del(&res->mob_head); + list_add_tail(&res->mob_head, &buf->res_list); vmw_bo_unreference(&old_buf); res->id = vcotbl->type; @@ -499,7 +496,7 @@ out_wait: * is called before bind() in the validation sequence is instead used for two * things. * 1) Unscrub the cotable if it is scrubbed and still attached to a backup - * buffer. + * buffer, that is, if @res->mob_head is non-empty. * 2) Resize the cotable if needed. */ static int vmw_cotable_create(struct vmw_resource *res) @@ -515,7 +512,7 @@ static int vmw_cotable_create(struct vmw_resource *res) new_size *= 2; if (likely(new_size <= res->backup_size)) { - if (vcotbl->scrubbed && vmw_resource_mob_attached(res)) { + if (vcotbl->scrubbed && !list_empty(&res->mob_head)) { ret = vmw_cotable_unscrub(res); if (ret) return ret; diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c index 8349a6cc126f..4ff11a0077e1 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c @@ -254,6 +254,7 @@ static int vmw_restrict_dma_mask; static int vmw_assume_16bpp; static int vmw_probe(struct pci_dev *, const struct pci_device_id *); +static void vmw_master_init(struct vmw_master *); static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val, void *ptr); @@ -761,6 +762,10 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) DRM_INFO("MMIO at 0x%08x size is %u kiB\n", dev_priv->mmio_start, dev_priv->mmio_size / 1024); + vmw_master_init(&dev_priv->fbdev_master); + ttm_lock_set_kill(&dev_priv->fbdev_master.lock, false, SIGTERM); + dev_priv->active_master = &dev_priv->fbdev_master; + dev_priv->mmio_virt = memremap(dev_priv->mmio_start, dev_priv->mmio_size, MEMREMAP_WB); @@ -828,11 +833,6 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset) DRM_ERROR("Failed initializing TTM buffer object driver.\n"); goto out_no_bdev; } - dev_priv->vm_ops = *dev_priv->bdev.vm_ops; - dev_priv->vm_ops.fault = vmw_bo_vm_fault; - dev_priv->vm_ops.pfn_mkwrite = vmw_bo_vm_mkwrite; - dev_priv->vm_ops.page_mkwrite = vmw_bo_vm_mkwrite; - dev_priv->bdev.vm_ops = &dev_priv->vm_ops; /* * Enable VRAM, but initially don't use it until SVGA is enabled and @@ -1007,7 +1007,18 @@ static void vmw_driver_unload(struct drm_device *dev) static void vmw_postclose(struct drm_device *dev, struct drm_file *file_priv) { - struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv); + struct vmw_fpriv *vmw_fp; + + vmw_fp = vmw_fpriv(file_priv); + + if (vmw_fp->locked_master) { + struct vmw_master *vmaster = + vmw_master(vmw_fp->locked_master); + + ttm_lock_set_kill(&vmaster->lock, true, SIGTERM); + ttm_vt_unlock(&vmaster->lock); + drm_master_put(&vmw_fp->locked_master); + } ttm_object_file_release(&vmw_fp->tfile); kfree(vmw_fp); @@ -1036,6 +1047,55 @@ out_no_tfile: return ret; } +static struct vmw_master *vmw_master_check(struct drm_device *dev, + struct drm_file *file_priv, + unsigned int flags) +{ + int ret; + struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv); + struct vmw_master *vmaster; + + if (!drm_is_primary_client(file_priv) || !(flags & DRM_AUTH)) + return NULL; + + ret = mutex_lock_interruptible(&dev->master_mutex); + if (unlikely(ret != 0)) + return ERR_PTR(-ERESTARTSYS); + + if (drm_is_current_master(file_priv)) { + mutex_unlock(&dev->master_mutex); + return NULL; + } + + /* + * Check if we were previously master, but now dropped. In that + * case, allow at least render node functionality. + */ + if (vmw_fp->locked_master) { + mutex_unlock(&dev->master_mutex); + + if (flags & DRM_RENDER_ALLOW) + return NULL; + + DRM_ERROR("Dropped master trying to access ioctl that " + "requires authentication.\n"); + return ERR_PTR(-EACCES); + } + mutex_unlock(&dev->master_mutex); + + /* + * Take the TTM lock. Possibly sleep waiting for the authenticating + * master to become master again, or for a SIGTERM if the + * authenticating master exits. + */ + vmaster = vmw_master(file_priv->master); + ret = ttm_read_lock(&vmaster->lock, true); + if (unlikely(ret != 0)) + vmaster = ERR_PTR(ret); + + return vmaster; +} + static long vmw_generic_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, long (*ioctl_func)(struct file *, unsigned int, @@ -1044,6 +1104,7 @@ static long vmw_generic_ioctl(struct file *filp, unsigned int cmd, struct drm_file *file_priv = filp->private_data; struct drm_device *dev = file_priv->minor->dev; unsigned int nr = DRM_IOCTL_NR(cmd); + struct vmw_master *vmaster; unsigned int flags; long ret; @@ -1079,7 +1140,21 @@ static long vmw_generic_ioctl(struct file *filp, unsigned int cmd, } else if (!drm_ioctl_flags(nr, &flags)) return -EINVAL; - return ioctl_func(filp, cmd, arg); + vmaster = vmw_master_check(dev, file_priv, flags); + if (IS_ERR(vmaster)) { + ret = PTR_ERR(vmaster); + + if (ret != -ERESTARTSYS) + DRM_INFO("IOCTL ERROR Command %d, Error %ld.\n", + nr, ret); + return ret; + } + + ret = ioctl_func(filp, cmd, arg); + if (vmaster) + ttm_read_unlock(&vmaster->lock); + + return ret; out_io_encoding: DRM_ERROR("Invalid command format, ioctl %d\n", @@ -1106,10 +1181,65 @@ static void vmw_lastclose(struct drm_device *dev) { } +static void vmw_master_init(struct vmw_master *vmaster) +{ + ttm_lock_init(&vmaster->lock); +} + +static int vmw_master_create(struct drm_device *dev, + struct drm_master *master) +{ + struct vmw_master *vmaster; + + vmaster = kzalloc(sizeof(*vmaster), GFP_KERNEL); + if (unlikely(!vmaster)) + return -ENOMEM; + + vmw_master_init(vmaster); + ttm_lock_set_kill(&vmaster->lock, true, SIGTERM); + master->driver_priv = vmaster; + + return 0; +} + +static void vmw_master_destroy(struct drm_device *dev, + struct drm_master *master) +{ + struct vmw_master *vmaster = vmw_master(master); + + master->driver_priv = NULL; + kfree(vmaster); +} + static int vmw_master_set(struct drm_device *dev, struct drm_file *file_priv, bool from_open) { + struct vmw_private *dev_priv = vmw_priv(dev); + struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv); + struct vmw_master *active = dev_priv->active_master; + struct vmw_master *vmaster = vmw_master(file_priv->master); + int ret = 0; + + if (active) { + BUG_ON(active != &dev_priv->fbdev_master); + ret = ttm_vt_lock(&active->lock, false, vmw_fp->tfile); + if (unlikely(ret != 0)) + return ret; + + ttm_lock_set_kill(&active->lock, true, SIGTERM); + dev_priv->active_master = NULL; + } + + ttm_lock_set_kill(&vmaster->lock, false, SIGTERM); + if (!from_open) { + ttm_vt_unlock(&vmaster->lock); + BUG_ON(vmw_fp->locked_master != file_priv->master); + drm_master_put(&vmw_fp->locked_master); + } + + dev_priv->active_master = vmaster; + /* * Inform a new master that the layout may have changed while * it was gone. @@ -1124,10 +1254,31 @@ static void vmw_master_drop(struct drm_device *dev, struct drm_file *file_priv) { struct vmw_private *dev_priv = vmw_priv(dev); + struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv); + struct vmw_master *vmaster = vmw_master(file_priv->master); + int ret; + + /** + * Make sure the master doesn't disappear while we have + * it locked. + */ + vmw_fp->locked_master = drm_master_get(file_priv->master); + ret = ttm_vt_lock(&vmaster->lock, false, vmw_fp->tfile); vmw_kms_legacy_hotspot_clear(dev_priv); + if (unlikely((ret != 0))) { + DRM_ERROR("Unable to lock TTM at VT switch.\n"); + drm_master_put(&vmw_fp->locked_master); + } + + ttm_lock_set_kill(&vmaster->lock, false, SIGTERM); + if (!dev_priv->enable_fb) vmw_svga_disable(dev_priv); + + dev_priv->active_master = &dev_priv->fbdev_master; + ttm_lock_set_kill(&dev_priv->fbdev_master.lock, false, SIGTERM); + ttm_vt_unlock(&dev_priv->fbdev_master.lock); } /** @@ -1406,6 +1557,8 @@ static struct drm_driver driver = { .disable_vblank = vmw_disable_vblank, .ioctls = vmw_ioctls, .num_ioctls = ARRAY_SIZE(vmw_ioctls), + .master_create = vmw_master_create, + .master_destroy = vmw_master_destroy, .master_set = vmw_master_set, .master_drop = vmw_master_drop, .open = vmw_driver_open, diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h index 3a358a5495e4..366dcfc1f9bb 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h @@ -44,9 +44,9 @@ #include #define VMWGFX_DRIVER_NAME "vmwgfx" -#define VMWGFX_DRIVER_DATE "20190328" +#define VMWGFX_DRIVER_DATE "20180704" #define VMWGFX_DRIVER_MAJOR 2 -#define VMWGFX_DRIVER_MINOR 16 +#define VMWGFX_DRIVER_MINOR 15 #define VMWGFX_DRIVER_PATCHLEVEL 0 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024) #define VMWGFX_MAX_RELOCATIONS 2048 @@ -81,30 +81,19 @@ #define VMW_RES_SHADER ttm_driver_type4 struct vmw_fpriv { + struct drm_master *locked_master; struct ttm_object_file *tfile; bool gb_aware; /* user-space is guest-backed aware */ }; -/** - * struct vmw_buffer_object - TTM buffer object with vmwgfx additions - * @base: The TTM buffer object - * @res_tree: RB tree of resources using this buffer object as a backing MOB - * @pin_count: pin depth - * @dx_query_ctx: DX context if this buffer object is used as a DX query MOB - * @map: Kmap object for semi-persistent mappings - * @res_prios: Eviction priority counts for attached resources - * @dirty: structure for user-space dirty-tracking - */ struct vmw_buffer_object { struct ttm_buffer_object base; - struct rb_root res_tree; + struct list_head res_list; s32 pin_count; /* Not ref-counted. Protected by binding_mutex */ struct vmw_resource *dx_query_ctx; /* Protected by reservation */ struct ttm_bo_kmap_obj map; - u32 res_prios[TTM_MAX_BO_PRIORITY]; - struct vmw_bo_dirty *dirty; }; /** @@ -135,8 +124,7 @@ struct vmw_res_func; * @res_dirty: Resource contains data not yet in the backup buffer. Protected * by resource reserved. * @backup_dirty: Backup buffer contains data not yet in the HW resource. - * Protected by resource reserved. - * @coherent: Emulate coherency by tracking vm accesses. + * Protecte by resource reserved. * @backup: The backup buffer if any. Protected by resource reserved. * @backup_offset: Offset into the backup buffer if any. Protected by resource * reserved. Note that only a few resource types can have a @backup_offset @@ -145,32 +133,28 @@ struct vmw_res_func; * pin-count greater than zero. It is not on the resource LRU lists and its * backup buffer is pinned. Hence it can't be evicted. * @func: Method vtable for this resource. Immutable. - * @mob_node; Node for the MOB backup rbtree. Protected by @backup reserved. * @lru_head: List head for the LRU list. Protected by @dev_priv::resource_lock. + * @mob_head: List head for the MOB backup list. Protected by @backup reserved. * @binding_head: List head for the context binding list. Protected by * the @dev_priv::binding_mutex * @res_free: The resource destructor. * @hw_destroy: Callback to destroy the resource on the device, as part of * resource destruction. */ -struct vmw_resource_dirty; struct vmw_resource { struct kref kref; struct vmw_private *dev_priv; int id; - u32 used_prio; unsigned long backup_size; - u32 res_dirty : 1; - u32 backup_dirty : 1; - u32 coherent : 1; + bool res_dirty; + bool backup_dirty; struct vmw_buffer_object *backup; unsigned long backup_offset; unsigned long pin_count; const struct vmw_res_func *func; - struct rb_node mob_node; struct list_head lru_head; + struct list_head mob_head; struct list_head binding_head; - struct vmw_resource_dirty *dirty; void (*res_free) (struct vmw_resource *res); void (*hw_destroy) (struct vmw_resource *res); }; @@ -392,6 +376,10 @@ struct vmw_sw_context{ struct vmw_legacy_display; struct vmw_overlay; +struct vmw_master { + struct ttm_lock lock; +}; + struct vmw_vga_topology_state { uint32_t width; uint32_t height; @@ -554,8 +542,11 @@ struct vmw_private { spinlock_t svga_lock; /** - * PM management. + * Master management. */ + + struct vmw_master *active_master; + struct vmw_master fbdev_master; struct notifier_block pm_nb; bool refuse_hibernation; bool suspend_locked; @@ -604,9 +595,6 @@ struct vmw_private { /* Validation memory reservation */ struct vmw_validation_mem vvm; - - /* VM operations */ - struct vm_operations_struct vm_ops; }; static inline struct vmw_surface *vmw_res_to_srf(struct vmw_resource *res) @@ -624,6 +612,11 @@ static inline struct vmw_fpriv *vmw_fpriv(struct drm_file *file_priv) return (struct vmw_fpriv *)file_priv->driver_priv; } +static inline struct vmw_master *vmw_master(struct drm_master *master) +{ + return (struct vmw_master *) master->driver_priv; +} + /* * The locking here is fine-grained, so that it is performed once * for every read- and write operation. This is of course costly, but we @@ -676,8 +669,7 @@ extern void vmw_resource_unreference(struct vmw_resource **p_res); extern struct vmw_resource *vmw_resource_reference(struct vmw_resource *res); extern struct vmw_resource * vmw_resource_reference_unless_doomed(struct vmw_resource *res); -extern int vmw_resource_validate(struct vmw_resource *res, bool intr, - bool dirtying); +extern int vmw_resource_validate(struct vmw_resource *res, bool intr); extern int vmw_resource_reserve(struct vmw_resource *res, bool interruptible, bool no_backup); extern bool vmw_resource_needs_backup(const struct vmw_resource *res); @@ -717,23 +709,6 @@ extern void vmw_query_move_notify(struct ttm_buffer_object *bo, extern int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob); extern void vmw_resource_evict_all(struct vmw_private *dev_priv); extern void vmw_resource_unbind_list(struct vmw_buffer_object *vbo); -void vmw_resource_mob_attach(struct vmw_resource *res); -void vmw_resource_mob_detach(struct vmw_resource *res); -void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start, - pgoff_t end); -int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start, - pgoff_t end, pgoff_t *num_prefault); - -/** - * vmw_resource_mob_attached - Whether a resource currently has a mob attached - * @res: The resource - * - * Return: true if the resource has a mob attached, false otherwise. - */ -static inline bool vmw_resource_mob_attached(const struct vmw_resource *res) -{ - return !RB_EMPTY_NODE(&res->mob_node); -} /** * vmw_user_resource_noref_release - release a user resource pointer looked up @@ -812,54 +787,6 @@ static inline void vmw_user_bo_noref_release(void) ttm_base_object_noref_release(); } -/** - * vmw_bo_adjust_prio - Adjust the buffer object eviction priority - * according to attached resources - * @vbo: The struct vmw_buffer_object - */ -static inline void vmw_bo_prio_adjust(struct vmw_buffer_object *vbo) -{ - int i = ARRAY_SIZE(vbo->res_prios); - - while (i--) { - if (vbo->res_prios[i]) { - vbo->base.priority = i; - return; - } - } - - vbo->base.priority = 3; -} - -/** - * vmw_bo_prio_add - Notify a buffer object of a newly attached resource - * eviction priority - * @vbo: The struct vmw_buffer_object - * @prio: The resource priority - * - * After being notified, the code assigns the highest resource eviction priority - * to the backing buffer object (mob). - */ -static inline void vmw_bo_prio_add(struct vmw_buffer_object *vbo, int prio) -{ - if (vbo->res_prios[prio]++ == 0) - vmw_bo_prio_adjust(vbo); -} - -/** - * vmw_bo_prio_del - Notify a buffer object of a resource with a certain - * priority being removed - * @vbo: The struct vmw_buffer_object - * @prio: The resource priority - * - * After being notified, the code assigns the highest resource eviction priority - * to the backing buffer object (mob). - */ -static inline void vmw_bo_prio_del(struct vmw_buffer_object *vbo, int prio) -{ - if (--vbo->res_prios[prio] == 0) - vmw_bo_prio_adjust(vbo); -} /** * Misc Ioctl functionality - vmwgfx_ioctl.c @@ -1089,6 +1016,7 @@ void vmw_kms_cursor_snoop(struct vmw_surface *srf, int vmw_kms_write_svga(struct vmw_private *vmw_priv, unsigned width, unsigned height, unsigned pitch, unsigned bpp, unsigned depth); +void vmw_kms_idle_workqueues(struct vmw_master *vmaster); bool vmw_kms_validate_mode_vram(struct vmw_private *dev_priv, uint32_t pitch, uint32_t height); @@ -1410,25 +1338,6 @@ int vmw_host_log(const char *log); #define VMW_DEBUG_USER(fmt, ...) \ DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__) -/** - * VMW_DEBUG_KMS - Debug output for kernel mode-setting - * - * This macro is for debugging vmwgfx mode-setting code. - */ -#define VMW_DEBUG_KMS(fmt, ...) \ - DRM_DEBUG_DRIVER(fmt, ##__VA_ARGS__) - -/* Resource dirtying - vmwgfx_page_dirty.c */ -void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo); -int vmw_bo_dirty_add(struct vmw_buffer_object *vbo); -void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res); -void vmw_bo_dirty_clear_res(struct vmw_resource *res); -void vmw_bo_dirty_release(struct vmw_buffer_object *vbo); -void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo, - pgoff_t start, pgoff_t end); -vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf); -vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf); - /** * Inline helper functions */ diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c index 319c1ca35663..33533d126277 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c @@ -2560,6 +2560,7 @@ static int vmw_cmd_dx_check_subresource(struct vmw_private *dev_priv, offsetof(typeof(*cmd), sid)); cmd = container_of(header, typeof(*cmd), header); + return vmw_cmd_res_check(dev_priv, sw_context, vmw_res_surface, VMW_RES_DIRTY_NONE, user_surface_converter, &cmd->sid, NULL); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c index e7222fa2cfdf..b97bc8e5944b 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c @@ -1462,7 +1462,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev, if (dev_priv->active_display_unit == vmw_du_screen_target && (drm_rect_width(&rects[i]) > dev_priv->stdu_max_width || drm_rect_height(&rects[i]) > dev_priv->stdu_max_height)) { - VMW_DEBUG_KMS("Screen size not supported.\n"); + DRM_ERROR("Screen size not supported.\n"); return -EINVAL; } @@ -1486,7 +1486,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev, * limit on primary bounding box */ if (pixel_mem > dev_priv->prim_bb_mem) { - VMW_DEBUG_KMS("Combined output size too large.\n"); + DRM_ERROR("Combined output size too large.\n"); return -EINVAL; } @@ -1496,7 +1496,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev, bb_mem = (u64) bounding_box.x2 * bounding_box.y2 * 4; if (bb_mem > dev_priv->prim_bb_mem) { - VMW_DEBUG_KMS("Topology is beyond supported limits.\n"); + DRM_ERROR("Topology is beyond supported limits.\n"); return -EINVAL; } } @@ -1645,7 +1645,6 @@ static int vmw_kms_check_topology(struct drm_device *dev, struct vmw_connector_state *vmw_conn_state; if (!du->pref_active && new_crtc_state->enable) { - VMW_DEBUG_KMS("Enabling a disabled display unit\n"); ret = -EINVAL; goto clean; } @@ -1702,10 +1701,8 @@ vmw_kms_atomic_check_modeset(struct drm_device *dev, return ret; ret = vmw_kms_check_implicit(dev, state); - if (ret) { - VMW_DEBUG_KMS("Invalid implicit state\n"); + if (ret) return ret; - } if (!state->allow_modeset) return ret; @@ -2350,9 +2347,6 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, if (!arg->num_outputs) { struct drm_rect def_rect = {0, 0, 800, 600}; - VMW_DEBUG_KMS("Default layout x1 = %d y1 = %d x2 = %d y2 = %d\n", - def_rect.x1, def_rect.y1, - def_rect.x2, def_rect.y2); vmw_du_update_layout(dev_priv, 1, &def_rect); return 0; } @@ -2373,7 +2367,6 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, drm_rects = (struct drm_rect *)rects; - VMW_DEBUG_KMS("Layout count = %u\n", arg->num_outputs); for (i = 0; i < arg->num_outputs; i++) { struct drm_vmw_rect curr_rect; @@ -2390,10 +2383,6 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, drm_rects[i].x2 = curr_rect.x + curr_rect.w; drm_rects[i].y2 = curr_rect.y + curr_rect.h; - VMW_DEBUG_KMS(" x1 = %d y1 = %d x2 = %d y2 = %d\n", - drm_rects[i].x1, drm_rects[i].y1, - drm_rects[i].x2, drm_rects[i].y2); - /* * Currently this check is limiting the topology within * mode_config->max (which actually is max texture size @@ -2404,9 +2393,7 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data, if (drm_rects[i].x1 < 0 || drm_rects[i].y1 < 0 || drm_rects[i].x2 > mode_config->max_width || drm_rects[i].y2 > mode_config->max_height) { - VMW_DEBUG_KMS("Invalid layout %d %d %d %d\n", - drm_rects[i].x1, drm_rects[i].y1, - drm_rects[i].x2, drm_rects[i].y2); + DRM_ERROR("Invalid GUI layout.\n"); ret = -EINVAL; goto out_free; } diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c b/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c deleted file mode 100644 index 730c51e397dd..000000000000 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_page_dirty.c +++ /dev/null @@ -1,472 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 OR MIT -/************************************************************************** - * - * Copyright 2019 VMware, Inc., Palo Alto, CA., USA - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL - * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, - * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - * USE OR OTHER DEALINGS IN THE SOFTWARE. - * - **************************************************************************/ -#include "vmwgfx_drv.h" - -/* - * Different methods for tracking dirty: - * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits - * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write- - * accesses in the VM mkwrite() callback - */ -enum vmw_bo_dirty_method { - VMW_BO_DIRTY_PAGETABLE, - VMW_BO_DIRTY_MKWRITE, -}; - -/* - * No dirtied pages at scan trigger a transition to the _MKWRITE method, - * similarly a certain percentage of dirty pages trigger a transition to - * the _PAGETABLE method. How many triggers should we wait for before - * changing method? - */ -#define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2 - -/* Percentage to trigger a transition to the _PAGETABLE method */ -#define VMW_DIRTY_PERCENTAGE 10 - -/** - * struct vmw_bo_dirty - Dirty information for buffer objects - * @start: First currently dirty bit - * @end: Last currently dirty bit + 1 - * @method: The currently used dirty method - * @change_count: Number of consecutive method change triggers - * @ref_count: Reference count for this structure - * @bitmap_size: The size of the bitmap in bits. Typically equal to the - * nuber of pages in the bo. - * @size: The accounting size for this struct. - * @bitmap: A bitmap where each bit represents a page. A set bit means a - * dirty page. - */ -struct vmw_bo_dirty { - unsigned long start; - unsigned long end; - enum vmw_bo_dirty_method method; - unsigned int change_count; - unsigned int ref_count; - unsigned long bitmap_size; - size_t size; - unsigned long bitmap[0]; -}; - -/** - * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits - * @vbo: The buffer object to scan - * - * Scans the pagetable for dirty bits. Clear those bits and modify the - * dirty structure with the results. This function may change the - * dirty-tracking method. - */ -static void vmw_bo_dirty_scan_pagetable(struct vmw_buffer_object *vbo) -{ - struct vmw_bo_dirty *dirty = vbo->dirty; - pgoff_t offset = drm_vma_node_start(&vbo->base.vma_node); - struct address_space *mapping = vbo->base.bdev->dev_mapping; - pgoff_t num_marked; - - num_marked = apply_as_clean(mapping, - offset, dirty->bitmap_size, - offset, &dirty->bitmap[0], - &dirty->start, &dirty->end); - if (num_marked == 0) - dirty->change_count++; - else - dirty->change_count = 0; - - if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) { - dirty->change_count = 0; - dirty->method = VMW_BO_DIRTY_MKWRITE; - apply_as_wrprotect(mapping, - offset, dirty->bitmap_size); - apply_as_clean(mapping, - offset, dirty->bitmap_size, - offset, &dirty->bitmap[0], - &dirty->start, &dirty->end); - } -} - -/** - * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method - * @vbo: The buffer object to scan - * - * Write-protect pages written to so that consecutive write accesses will - * trigger a call to mkwrite. - * - * This function may change the dirty-tracking method. - */ -static void vmw_bo_dirty_scan_mkwrite(struct vmw_buffer_object *vbo) -{ - struct vmw_bo_dirty *dirty = vbo->dirty; - unsigned long offset = drm_vma_node_start(&vbo->base.vma_node); - struct address_space *mapping = vbo->base.bdev->dev_mapping; - pgoff_t num_marked; - - if (dirty->end <= dirty->start) - return; - - num_marked = apply_as_wrprotect(vbo->base.bdev->dev_mapping, - dirty->start + offset, - dirty->end - dirty->start); - - if (100UL * num_marked / dirty->bitmap_size > - VMW_DIRTY_PERCENTAGE) { - dirty->change_count++; - } else { - dirty->change_count = 0; - } - - if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) { - pgoff_t start = 0; - pgoff_t end = dirty->bitmap_size; - - dirty->method = VMW_BO_DIRTY_PAGETABLE; - apply_as_clean(mapping, offset, end, offset, &dirty->bitmap[0], - &start, &end); - bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size); - if (dirty->start < dirty->end) - bitmap_set(&dirty->bitmap[0], dirty->start, - dirty->end - dirty->start); - dirty->change_count = 0; - } -} - -/** - * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty - * tracking structure - * @vbo: The buffer object to scan - * - * This function may change the dirty tracking method. - */ -void vmw_bo_dirty_scan(struct vmw_buffer_object *vbo) -{ - struct vmw_bo_dirty *dirty = vbo->dirty; - - if (dirty->method == VMW_BO_DIRTY_PAGETABLE) - vmw_bo_dirty_scan_pagetable(vbo); - else - vmw_bo_dirty_scan_mkwrite(vbo); -} - -/** - * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before - * an unmap_mapping_range operation. - * @vbo: The buffer object, - * @start: First page of the range within the buffer object. - * @end: Last page of the range within the buffer object + 1. - * - * If we're using the _PAGETABLE scan method, we may leak dirty pages - * when calling unmap_mapping_range(). This function makes sure we pick - * up all dirty pages. - */ -static void vmw_bo_dirty_pre_unmap(struct vmw_buffer_object *vbo, - pgoff_t start, pgoff_t end) -{ - struct vmw_bo_dirty *dirty = vbo->dirty; - unsigned long offset = drm_vma_node_start(&vbo->base.vma_node); - struct address_space *mapping = vbo->base.bdev->dev_mapping; - - if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end) - return; - - apply_as_wrprotect(mapping, start + offset, end - start); - apply_as_clean(mapping, start + offset, end - start, offset, - &dirty->bitmap[0], &dirty->start, &dirty->end); -} - -/** - * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo - * @vbo: The buffer object, - * @start: First page of the range within the buffer object. - * @end: Last page of the range within the buffer object + 1. - * - * This is similar to ttm_bo_unmap_virtual_locked() except it takes a subrange. - */ -void vmw_bo_dirty_unmap(struct vmw_buffer_object *vbo, - pgoff_t start, pgoff_t end) -{ - unsigned long offset = drm_vma_node_start(&vbo->base.vma_node); - struct address_space *mapping = vbo->base.bdev->dev_mapping; - - vmw_bo_dirty_pre_unmap(vbo, start, end); - unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT, - (loff_t) (end - start) << PAGE_SHIFT); -} - -/** - * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object - * @vbo: The buffer object - * - * This function registers a dirty-tracking user to a buffer object. - * A user can be for example a resource or a vma in a special user-space - * mapping. - * - * Return: Zero on success, -ENOMEM on memory allocation failure. - */ -int vmw_bo_dirty_add(struct vmw_buffer_object *vbo) -{ - struct vmw_bo_dirty *dirty = vbo->dirty; - pgoff_t num_pages = vbo->base.num_pages; - size_t size, acc_size; - int ret; - static struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - - if (dirty) { - dirty->ref_count++; - return 0; - } - - size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long); - acc_size = ttm_round_pot(size); - ret = ttm_mem_global_alloc(&ttm_mem_glob, acc_size, &ctx); - if (ret) { - VMW_DEBUG_USER("Out of graphics memory for buffer object " - "dirty tracker.\n"); - return ret; - } - dirty = kvzalloc(size, GFP_KERNEL); - if (!dirty) { - ret = -ENOMEM; - goto out_no_dirty; - } - - dirty->size = acc_size; - dirty->bitmap_size = num_pages; - dirty->start = dirty->bitmap_size; - dirty->end = 0; - dirty->ref_count = 1; - if (num_pages < PAGE_SIZE / sizeof(pte_t)) { - dirty->method = VMW_BO_DIRTY_PAGETABLE; - } else { - struct address_space *mapping = vbo->base.bdev->dev_mapping; - pgoff_t offset = drm_vma_node_start(&vbo->base.vma_node); - - dirty->method = VMW_BO_DIRTY_MKWRITE; - - /* Write-protect and then pick up already dirty bits */ - apply_as_wrprotect(mapping, offset, num_pages); - apply_as_clean(mapping, offset, num_pages, offset, - &dirty->bitmap[0], &dirty->start, &dirty->end); - } - - vbo->dirty = dirty; - - return 0; - -out_no_dirty: - ttm_mem_global_free(&ttm_mem_glob, acc_size); - return ret; -} - -/** - * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object - * @vbo: The buffer object - * - * This function releases a dirty-tracking user from a buffer object. - * If the reference count reaches zero, then the dirty-tracking object is - * freed and the pointer to it cleared. - * - * Return: Zero on success, -ENOMEM on memory allocation failure. - */ -void vmw_bo_dirty_release(struct vmw_buffer_object *vbo) -{ - struct vmw_bo_dirty *dirty = vbo->dirty; - - if (dirty && --dirty->ref_count == 0) { - size_t acc_size = dirty->size; - - kvfree(dirty); - ttm_mem_global_free(&ttm_mem_glob, acc_size); - vbo->dirty = NULL; - } -} - -/** - * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from - * its backing mob. - * @res: The resource - * - * This function will pick up all dirty ranges affecting the resource from - * it's backup mob, and call vmw_resource_dirty_update() once for each - * range. The transferred ranges will be cleared from the backing mob's - * dirty tracking. - */ -void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res) -{ - struct vmw_buffer_object *vbo = res->backup; - struct vmw_bo_dirty *dirty = vbo->dirty; - pgoff_t start, cur, end; - unsigned long res_start = res->backup_offset; - unsigned long res_end = res->backup_offset + res->backup_size; - - WARN_ON_ONCE(res_start & ~PAGE_MASK); - res_start >>= PAGE_SHIFT; - res_end = DIV_ROUND_UP(res_end, PAGE_SIZE); - - if (res_start >= dirty->end || res_end <= dirty->start) - return; - - cur = max(res_start, dirty->start); - res_end = max(res_end, dirty->end); - while (cur < res_end) { - unsigned long num; - - start = find_next_bit(&dirty->bitmap[0], res_end, cur); - if (start >= res_end) - break; - - end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1); - cur = end + 1; - num = end - start; - bitmap_clear(&dirty->bitmap[0], start, num); - vmw_resource_dirty_update(res, start, end); - } - - if (res_start <= dirty->start && res_end > dirty->start) - dirty->start = res_end; - if (res_start < dirty->end && res_end >= dirty->end) - dirty->end = res_start; -} - -/** - * vmw_bo_dirty_clear_res - Clear a resource's dirty region from - * its backing mob. - * @res: The resource - * - * This function will clear all dirty ranges affecting the resource from - * it's backup mob's dirty tracking. - */ -void vmw_bo_dirty_clear_res(struct vmw_resource *res) -{ - unsigned long res_start = res->backup_offset; - unsigned long res_end = res->backup_offset + res->backup_size; - struct vmw_buffer_object *vbo = res->backup; - struct vmw_bo_dirty *dirty = vbo->dirty; - - res_start >>= PAGE_SHIFT; - res_end = DIV_ROUND_UP(res_end, PAGE_SIZE); - - if (res_start >= dirty->end || res_end <= dirty->start) - return; - - res_start = max(res_start, dirty->start); - res_end = min(res_end, dirty->end); - bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start); - - if (res_start <= dirty->start && res_end > dirty->start) - dirty->start = res_end; - if (res_start < dirty->end && res_end >= dirty->end) - dirty->end = res_start; -} - -vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - struct ttm_buffer_object *bo = (struct ttm_buffer_object *) - vma->vm_private_data; - vm_fault_t ret; - unsigned long page_offset; - struct vmw_buffer_object *vbo = - container_of(bo, typeof(*vbo), base); - - ret = ttm_bo_vm_reserve(bo, vmf); - if (ret) - return ret; - - page_offset = vmf->pgoff - drm_vma_node_start(&bo->vma_node); - if (unlikely(page_offset >= bo->num_pages)) { - ret = VM_FAULT_SIGBUS; - goto out_unlock; - } - - if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE && - !test_bit(page_offset, &vbo->dirty->bitmap[0])) { - struct vmw_bo_dirty *dirty = vbo->dirty; - - __set_bit(page_offset, &dirty->bitmap[0]); - dirty->start = min(dirty->start, page_offset); - dirty->end = max(dirty->end, page_offset + 1); - } - -out_unlock: - reservation_object_unlock(bo->resv); - return ret; -} - -vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - struct ttm_buffer_object *bo = (struct ttm_buffer_object *) - vma->vm_private_data; - struct vmw_buffer_object *vbo = - container_of(bo, struct vmw_buffer_object, base); - pgoff_t num_prefault; - pgprot_t prot; - vm_fault_t ret; - - ret = ttm_bo_vm_reserve(bo, vmf); - if (ret) - return ret; - - num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 : - TTM_BO_VM_NUM_PREFAULT; - - if (vbo->dirty) { - pgoff_t allowed_prefault; - unsigned long page_offset; - - page_offset = vmf->pgoff - drm_vma_node_start(&bo->vma_node); - if (page_offset >= bo->num_pages || - vmw_resources_clean(vbo, page_offset, - page_offset + PAGE_SIZE, - &allowed_prefault)) { - ret = VM_FAULT_SIGBUS; - goto out_unlock; - } - - num_prefault = min(num_prefault, allowed_prefault); - } - - /* - * If we don't track dirty using the MKWRITE method, make sure - * sure the page protection is write-enabled so we don't get - * a lot of unnecessary write faults. - */ - if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE) - prot = vma->vm_page_prot; - else - prot = vm_get_page_prot(vma->vm_flags); - - ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault); - if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) - return ret; - -out_unlock: - reservation_object_unlock(bo->resv); - return ret; -} diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c index d70ee0df5c13..1d38a8b2f2ec 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c @@ -34,51 +34,6 @@ #define VMW_RES_EVICT_ERR_COUNT 10 -/** - * vmw_resource_mob_attach - Mark a resource as attached to its backing mob - * @res: The resource - */ -void vmw_resource_mob_attach(struct vmw_resource *res) -{ - struct vmw_buffer_object *backup = res->backup; - struct rb_node **new = &backup->res_tree.rb_node, *parent = NULL; - - lockdep_assert_held(&backup->base.resv->lock.base); - res->used_prio = (res->res_dirty) ? res->func->dirty_prio : - res->func->prio; - - while (*new) { - struct vmw_resource *this = - container_of(*new, struct vmw_resource, mob_node); - - parent = *new; - new = (res->backup_offset < this->backup_offset) ? - &((*new)->rb_left) : &((*new)->rb_right); - } - - rb_link_node(&res->mob_node, parent, new); - rb_insert_color(&res->mob_node, &backup->res_tree); - - vmw_bo_prio_add(backup, res->used_prio); -} - -/** - * vmw_resource_mob_detach - Mark a resource as detached from its backing mob - * @res: The resource - */ -void vmw_resource_mob_detach(struct vmw_resource *res) -{ - struct vmw_buffer_object *backup = res->backup; - - lockdep_assert_held(&backup->base.resv->lock.base); - if (vmw_resource_mob_attached(res)) { - rb_erase(&res->mob_node, &backup->res_tree); - RB_CLEAR_NODE(&res->mob_node); - vmw_bo_prio_del(backup, res->used_prio); - } -} - - struct vmw_resource *vmw_resource_reference(struct vmw_resource *res) { kref_get(&res->kref); @@ -125,7 +80,7 @@ static void vmw_resource_release(struct kref *kref) struct ttm_buffer_object *bo = &res->backup->base; ttm_bo_reserve(bo, false, false, NULL); - if (vmw_resource_mob_attached(res) && + if (!list_empty(&res->mob_head) && res->func->unbind != NULL) { struct ttm_validate_buffer val_buf; @@ -134,11 +89,7 @@ static void vmw_resource_release(struct kref *kref) res->func->unbind(res, false, &val_buf); } res->backup_dirty = false; - vmw_resource_mob_detach(res); - if (res->dirty) - res->func->dirty_free(res); - if (res->coherent) - vmw_bo_dirty_release(res->backup); + list_del_init(&res->mob_head); ttm_bo_unreserve(bo); vmw_bo_unreference(&res->backup); } @@ -220,17 +171,14 @@ int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res, res->res_free = res_free; res->dev_priv = dev_priv; res->func = func; - RB_CLEAR_NODE(&res->mob_node); INIT_LIST_HEAD(&res->lru_head); + INIT_LIST_HEAD(&res->mob_head); INIT_LIST_HEAD(&res->binding_head); res->id = -1; res->backup = NULL; res->backup_offset = 0; res->backup_dirty = false; res->res_dirty = false; - res->coherent = false; - res->used_prio = 3; - res->dirty = NULL; if (delay_id) return 0; else @@ -395,8 +343,7 @@ out_no_bo: * should be retried once resources have been freed up. */ static int vmw_resource_do_validate(struct vmw_resource *res, - struct ttm_validate_buffer *val_buf, - bool dirtying) + struct ttm_validate_buffer *val_buf) { int ret = 0; const struct vmw_res_func *func = res->func; @@ -408,47 +355,14 @@ static int vmw_resource_do_validate(struct vmw_resource *res, } if (func->bind && - ((func->needs_backup && !vmw_resource_mob_attached(res) && + ((func->needs_backup && list_empty(&res->mob_head) && val_buf->bo != NULL) || (!func->needs_backup && val_buf->bo != NULL))) { ret = func->bind(res, val_buf); if (unlikely(ret != 0)) goto out_bind_failed; if (func->needs_backup) - vmw_resource_mob_attach(res); - } - - /* - * Handle the case where the backup mob is marked coherent but - * the resource isn't. - */ - if (func->dirty_alloc && vmw_resource_mob_attached(res) && - !res->coherent) { - if (res->backup->dirty && !res->dirty) { - ret = func->dirty_alloc(res); - if (ret) - return ret; - } else if (!res->backup->dirty && res->dirty) { - func->dirty_free(res); - } - } - - /* - * Transfer the dirty regions to the resource and update - * the resource. - */ - if (res->dirty) { - if (dirtying && !res->res_dirty) { - pgoff_t start = res->backup_offset >> PAGE_SHIFT; - pgoff_t end = __KERNEL_DIV_ROUND_UP - (res->backup_offset + res->backup_size, - PAGE_SIZE); - - vmw_bo_dirty_unmap(res->backup, start, end); - } - - vmw_bo_dirty_transfer_to_res(res); - return func->dirty_sync(res); + list_add_tail(&res->mob_head, &res->backup->res_list); } return 0; @@ -488,29 +402,19 @@ void vmw_resource_unreserve(struct vmw_resource *res, if (switch_backup && new_backup != res->backup) { if (res->backup) { - vmw_resource_mob_detach(res); - if (res->coherent) - vmw_bo_dirty_release(res->backup); + lockdep_assert_held(&res->backup->base.resv->lock.base); + list_del_init(&res->mob_head); vmw_bo_unreference(&res->backup); } if (new_backup) { res->backup = vmw_bo_reference(new_backup); - - /* - * The validation code should already have added a - * dirty tracker here. - */ - WARN_ON(res->coherent && !new_backup->dirty); - - vmw_resource_mob_attach(res); + lockdep_assert_held(&new_backup->base.resv->lock.base); + list_add_tail(&res->mob_head, &new_backup->res_list); } else { res->backup = NULL; } - } else if (switch_backup && res->coherent) { - vmw_bo_dirty_release(res->backup); } - if (switch_backup) res->backup_offset = new_backup_offset; @@ -565,7 +469,7 @@ vmw_resource_check_buffer(struct ww_acquire_ctx *ticket, if (unlikely(ret != 0)) goto out_no_reserve; - if (res->func->needs_backup && !vmw_resource_mob_attached(res)) + if (res->func->needs_backup && list_empty(&res->mob_head)) return 0; backup_dirty = res->backup_dirty; @@ -670,11 +574,11 @@ static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket, return ret; if (unlikely(func->unbind != NULL && - (!func->needs_backup || vmw_resource_mob_attached(res)))) { + (!func->needs_backup || !list_empty(&res->mob_head)))) { ret = func->unbind(res, res->res_dirty, &val_buf); if (unlikely(ret != 0)) goto out_no_unbind; - vmw_resource_mob_detach(res); + list_del_init(&res->mob_head); } ret = func->destroy(res); res->backup_dirty = true; @@ -691,7 +595,6 @@ out_no_unbind: * to the device. * @res: The resource to make visible to the device. * @intr: Perform waits interruptible if possible. - * @dirtying: Pending GPU operation will dirty the resource * * On succesful return, any backup DMA buffer pointed to by @res->backup will * be reserved and validated. @@ -701,8 +604,7 @@ out_no_unbind: * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code * on failure. */ -int vmw_resource_validate(struct vmw_resource *res, bool intr, - bool dirtying) +int vmw_resource_validate(struct vmw_resource *res, bool intr) { int ret; struct vmw_resource *evict_res; @@ -719,7 +621,7 @@ int vmw_resource_validate(struct vmw_resource *res, bool intr, if (res->backup) val_buf.bo = &res->backup->base; do { - ret = vmw_resource_do_validate(res, &val_buf, dirtying); + ret = vmw_resource_do_validate(res, &val_buf); if (likely(ret != -EBUSY)) break; @@ -758,7 +660,7 @@ int vmw_resource_validate(struct vmw_resource *res, bool intr, if (unlikely(ret != 0)) goto out_no_validate; else if (!res->func->needs_backup && res->backup) { - WARN_ON_ONCE(vmw_resource_mob_attached(res)); + list_del_init(&res->mob_head); vmw_bo_unreference(&res->backup); } @@ -782,23 +684,22 @@ out_no_validate: */ void vmw_resource_unbind_list(struct vmw_buffer_object *vbo) { + + struct vmw_resource *res, *next; struct ttm_validate_buffer val_buf = { .bo = &vbo->base, .num_shared = 0 }; lockdep_assert_held(&vbo->base.resv->lock.base); - while (!RB_EMPTY_ROOT(&vbo->res_tree)) { - struct rb_node *node = vbo->res_tree.rb_node; - struct vmw_resource *res = - container_of(node, struct vmw_resource, mob_node); - - if (!WARN_ON_ONCE(!res->func->unbind)) - (void) res->func->unbind(res, res->res_dirty, &val_buf); + list_for_each_entry_safe(res, next, &vbo->res_list, mob_head) { + if (!res->func->unbind) + continue; + (void) res->func->unbind(res, res->res_dirty, &val_buf); res->backup_dirty = true; res->res_dirty = false; - vmw_resource_mob_detach(res); + list_del_init(&res->mob_head); } (void) ttm_bo_wait(&vbo->base, false, false); @@ -1019,7 +920,7 @@ int vmw_resource_pin(struct vmw_resource *res, bool interruptible) /* Do we really need to pin the MOB as well? */ vmw_bo_pin_reserved(vbo, true); } - ret = vmw_resource_validate(res, interruptible, true); + ret = vmw_resource_validate(res, interruptible); if (vbo) ttm_bo_unreserve(&vbo->base); if (ret) @@ -1079,101 +980,3 @@ enum vmw_res_type vmw_res_type(const struct vmw_resource *res) { return res->func->res_type; } - -/** - * vmw_resource_update_dirty - Update a resource's dirty tracker with a - * sequential range of touched backing store memory. - * @res: The resource. - * @start: The first page touched. - * @end: The last page touched + 1. - */ -void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start, - pgoff_t end) -{ - if (res->dirty) - res->func->dirty_range_add(res, start << PAGE_SHIFT, - end << PAGE_SHIFT); -} - -/** - * vmw_resources_clean - Clean resources intersecting a mob range - * @vbo: The mob buffer object - * @start: The mob page offset starting the range - * @end: The mob page offset ending the range - * @num_prefault: Returns how many pages including the first have been - * cleaned and are ok to prefault - */ -int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start, - pgoff_t end, pgoff_t *num_prefault) -{ - struct rb_node *cur = vbo->res_tree.rb_node; - struct vmw_resource *found = NULL; - unsigned long res_start = start << PAGE_SHIFT; - unsigned long res_end = end << PAGE_SHIFT; - unsigned long last_cleaned = 0; - - /* - * Find the resource with lowest backup_offset that intersects the - * range. - */ - while (cur) { - struct vmw_resource *cur_res = - container_of(cur, struct vmw_resource, mob_node); - - if (cur_res->backup_offset >= res_end) { - cur = cur->rb_left; - } else if (cur_res->backup_offset + cur_res->backup_size <= - res_start) { - cur = cur->rb_right; - } else { - found = cur_res; - cur = cur->rb_left; - /* Continue to look for resources with lower offsets */ - } - } - - /* - * In order of increasing backup_offset, clean dirty resorces - * intersecting the range. - */ - while (found) { - if (found->res_dirty) { - int ret; - - if (!found->func->clean) - return -EINVAL; - - ret = found->func->clean(found); - if (ret) - return ret; - - found->res_dirty = false; - } - last_cleaned = found->backup_offset + found->backup_size; - cur = rb_next(&found->mob_node); - if (!cur) - break; - - found = container_of(cur, struct vmw_resource, mob_node); - if (found->backup_offset >= res_end) - break; - } - - /* - * Set number of pages allowed prefaulting and fence the buffer object - */ - *num_prefault = 1; - if (last_cleaned > res_start) { - struct ttm_buffer_object *bo = &vbo->base; - - *num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start, - PAGE_SIZE); - vmw_bo_fence_single(bo, NULL); - if (bo->moving) - dma_fence_put(bo->moving); - bo->moving = dma_fence_get - (reservation_object_get_excl(bo->resv)); - } - - return 0; -} diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h index 3b7438b2d289..7e19eba0b0b8 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource_priv.h @@ -71,13 +71,6 @@ struct vmw_user_resource_conv { * @commit_notify: If the resource is a command buffer managed resource, * callback to notify that a define or remove command * has been committed to the device. - * @dirty_alloc: Allocate a dirty tracker. NULL if dirty-tracking is not - * supported. - * @dirty_free: Free the dirty tracker. - * @dirty_sync: Upload the dirty mob contents to the resource. - * @dirty_add_range: Add a sequential dirty range to the resource - * dirty tracker. - * @clean: Clean the resource. */ struct vmw_res_func { enum vmw_res_type res_type; @@ -85,8 +78,6 @@ struct vmw_res_func { const char *type_name; struct ttm_placement *backup_placement; bool may_evict; - u32 prio; - u32 dirty_prio; int (*create) (struct vmw_resource *res); int (*destroy) (struct vmw_resource *res); @@ -97,12 +88,6 @@ struct vmw_res_func { struct ttm_validate_buffer *val_buf); void (*commit_notify)(struct vmw_resource *res, enum vmw_cmdbuf_res_state state); - int (*dirty_alloc)(struct vmw_resource *res); - void (*dirty_free)(struct vmw_resource *res); - int (*dirty_sync)(struct vmw_resource *res); - void (*dirty_range_add)(struct vmw_resource *res, size_t start, - size_t end); - int (*clean)(struct vmw_resource *res); }; /** diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c index e139fdfd1635..d310d21f0d54 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c @@ -95,8 +95,6 @@ static const struct vmw_res_func vmw_gb_shader_func = { .res_type = vmw_res_shader, .needs_backup = true, .may_evict = true, - .prio = 3, - .dirty_prio = 3, .type_name = "guest backed shaders", .backup_placement = &vmw_mob_placement, .create = vmw_gb_shader_create, @@ -108,9 +106,7 @@ static const struct vmw_res_func vmw_gb_shader_func = { static const struct vmw_res_func vmw_dx_shader_func = { .res_type = vmw_res_shader, .needs_backup = true, - .may_evict = true, - .prio = 3, - .dirty_prio = 3, + .may_evict = false, .type_name = "dx shaders", .backup_placement = &vmw_mob_placement, .create = vmw_dx_shader_create, @@ -427,7 +423,7 @@ static int vmw_dx_shader_create(struct vmw_resource *res) WARN_ON_ONCE(!shader->committed); - if (vmw_resource_mob_attached(res)) { + if (!list_empty(&res->mob_head)) { mutex_lock(&dev_priv->binding_mutex); ret = vmw_dx_shader_unscrub(res); mutex_unlock(&dev_priv->binding_mutex); diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c index 862ca44680ca..219471903bc1 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c @@ -68,20 +68,6 @@ struct vmw_surface_offset { uint32_t bo_offset; }; -/** - * vmw_surface_dirty - Surface dirty-tracker - * @cache: Cached layout information of the surface. - * @size: Accounting size for the struct vmw_surface_dirty. - * @num_subres: Number of subresources. - * @boxes: Array of SVGA3dBoxes indicating dirty regions. One per subresource. - */ -struct vmw_surface_dirty { - struct svga3dsurface_cache cache; - size_t size; - u32 num_subres; - SVGA3dBox boxes[0]; -}; - static void vmw_user_surface_free(struct vmw_resource *res); static struct vmw_resource * vmw_user_surface_base_to_res(struct ttm_base_object *base); @@ -110,13 +96,6 @@ vmw_gb_surface_reference_internal(struct drm_device *dev, struct drm_vmw_gb_surface_ref_ext_rep *rep, struct drm_file *file_priv); -static void vmw_surface_dirty_free(struct vmw_resource *res); -static int vmw_surface_dirty_alloc(struct vmw_resource *res); -static int vmw_surface_dirty_sync(struct vmw_resource *res); -static void vmw_surface_dirty_range_add(struct vmw_resource *res, size_t start, - size_t end); -static int vmw_surface_clean(struct vmw_resource *res); - static const struct vmw_user_resource_conv user_surface_conv = { .object_type = VMW_RES_SURFACE, .base_obj_to_res = vmw_user_surface_base_to_res, @@ -133,8 +112,6 @@ static const struct vmw_res_func vmw_legacy_surface_func = { .res_type = vmw_res_surface, .needs_backup = false, .may_evict = true, - .prio = 1, - .dirty_prio = 1, .type_name = "legacy surfaces", .backup_placement = &vmw_srf_placement, .create = &vmw_legacy_srf_create, @@ -147,19 +124,12 @@ static const struct vmw_res_func vmw_gb_surface_func = { .res_type = vmw_res_surface, .needs_backup = true, .may_evict = true, - .prio = 1, - .dirty_prio = 2, .type_name = "guest backed surfaces", .backup_placement = &vmw_mob_placement, .create = vmw_gb_surface_create, .destroy = vmw_gb_surface_destroy, .bind = vmw_gb_surface_bind, - .unbind = vmw_gb_surface_unbind, - .dirty_alloc = vmw_surface_dirty_alloc, - .dirty_free = vmw_surface_dirty_free, - .dirty_sync = vmw_surface_dirty_sync, - .dirty_range_add = vmw_surface_dirty_range_add, - .clean = vmw_surface_clean, + .unbind = vmw_gb_surface_unbind }; /** @@ -667,7 +637,6 @@ static void vmw_user_surface_free(struct vmw_resource *res) struct vmw_private *dev_priv = srf->res.dev_priv; uint32_t size = user_srf->size; - WARN_ON_ONCE(res->dirty); if (user_srf->master) drm_master_put(&user_srf->master); kfree(srf->offsets); @@ -946,6 +915,12 @@ vmw_surface_handle_reference(struct vmw_private *dev_priv, if (unlikely(drm_is_render_client(file_priv))) require_exist = true; + if (READ_ONCE(vmw_fpriv(file_priv)->locked_master)) { + DRM_ERROR("Locked master refused legacy " + "surface reference.\n"); + return -EACCES; + } + handle = u_handle; } @@ -1195,16 +1170,10 @@ static int vmw_gb_surface_bind(struct vmw_resource *res, cmd2->header.id = SVGA_3D_CMD_UPDATE_GB_SURFACE; cmd2->header.size = sizeof(cmd2->body); cmd2->body.sid = res->id; + res->backup_dirty = false; } vmw_fifo_commit(dev_priv, submit_size); - if (res->backup->dirty && res->backup_dirty) { - /* We've just made a full upload. Cear dirty regions. */ - vmw_bo_dirty_clear_res(res); - } - - res->backup_dirty = false; - return 0; } @@ -1669,8 +1638,7 @@ vmw_gb_surface_define_internal(struct drm_device *dev, } } } else if (req->base.drm_surface_flags & - (drm_vmw_surface_flag_create_buffer | - drm_vmw_surface_flag_coherent)) + drm_vmw_surface_flag_create_buffer) ret = vmw_user_bo_alloc(dev_priv, tfile, res->backup_size, req->base.drm_surface_flags & @@ -1684,26 +1652,6 @@ vmw_gb_surface_define_internal(struct drm_device *dev, goto out_unlock; } - if (req->base.drm_surface_flags & drm_vmw_surface_flag_coherent) { - struct vmw_buffer_object *backup = res->backup; - - ttm_bo_reserve(&backup->base, false, false, NULL); - if (!res->func->dirty_alloc) - ret = -EINVAL; - if (!ret) - ret = vmw_bo_dirty_add(backup); - if (!ret) { - res->coherent = true; - ret = res->func->dirty_alloc(res); - } - ttm_bo_unreserve(&backup->base); - if (ret) { - vmw_resource_unreference(&res); - goto out_unlock; - } - - } - tmp = vmw_resource_reference(res); ret = ttm_prime_object_init(tfile, res->backup_size, &user_srf->prime, req->base.drm_surface_flags & @@ -1812,338 +1760,3 @@ out_bad_resource: return ret; } - -/** - * vmw_subres_dirty_add - Add a dirty region to a subresource - * @dirty: The surfaces's dirty tracker. - * @loc_start: The location corresponding to the start of the region. - * @loc_end: The location corresponding to the end of the region. - * - * As we are assuming that @loc_start and @loc_end represent a sequential - * range of backing store memory, if the region spans multiple lines then - * regardless of the x coordinate, the full lines are dirtied. - * Correspondingly if the region spans multiple z slices, then full rather - * than partial z slices are dirtied. - */ -static void vmw_subres_dirty_add(struct vmw_surface_dirty *dirty, - const struct svga3dsurface_loc *loc_start, - const struct svga3dsurface_loc *loc_end) -{ - const struct svga3dsurface_cache *cache = &dirty->cache; - SVGA3dBox *box = &dirty->boxes[loc_start->sub_resource]; - u32 mip = loc_start->sub_resource % cache->num_mip_levels; - const struct drm_vmw_size *size = &cache->mip[mip].size; - u32 box_c2 = box->z + box->d; - - if (WARN_ON(loc_start->sub_resource >= dirty->num_subres)) - return; - - if (box->d == 0 || box->z > loc_start->z) - box->z = loc_start->z; - if (box_c2 < loc_end->z) - box->d = loc_end->z - box->z; - - if (loc_start->z + 1 == loc_end->z) { - box_c2 = box->y + box->h; - if (box->h == 0 || box->y > loc_start->y) - box->y = loc_start->y; - if (box_c2 < loc_end->y) - box->h = loc_end->y - box->y; - - if (loc_start->y + 1 == loc_end->y) { - box_c2 = box->x + box->w; - if (box->w == 0 || box->x > loc_start->x) - box->x = loc_start->x; - if (box_c2 < loc_end->x) - box->w = loc_end->x - box->x; - } else { - box->x = 0; - box->w = size->width; - } - } else { - box->y = 0; - box->h = size->height; - box->x = 0; - box->w = size->width; - } -} - -/** - * vmw_subres_dirty_full - Mark a full subresource as dirty - * @dirty: The surface's dirty tracker. - * @subres: The subresource - */ -static void vmw_subres_dirty_full(struct vmw_surface_dirty *dirty, u32 subres) -{ - const struct svga3dsurface_cache *cache = &dirty->cache; - u32 mip = subres % cache->num_mip_levels; - const struct drm_vmw_size *size = &cache->mip[mip].size; - SVGA3dBox *box = &dirty->boxes[subres]; - - box->x = 0; - box->y = 0; - box->z = 0; - box->w = size->width; - box->h = size->height; - box->d = size->depth; -} - -/* - * vmw_surface_tex_dirty_add_range - The dirty_add_range callback for texture - * surfaces. - */ -static void vmw_surface_tex_dirty_range_add(struct vmw_resource *res, - size_t start, size_t end) -{ - struct vmw_surface_dirty *dirty = - (struct vmw_surface_dirty *) res->dirty; - size_t backup_end = res->backup_offset + res->backup_size; - struct svga3dsurface_loc loc1, loc2; - const struct svga3dsurface_cache *cache; - - start = max_t(size_t, start, res->backup_offset) - res->backup_offset; - end = min(end, backup_end) - res->backup_offset; - cache = &dirty->cache; - svga3dsurface_get_loc(cache, &loc1, start); - svga3dsurface_get_loc(cache, &loc2, end - 1); - svga3dsurface_inc_loc(cache, &loc2); - - if (loc1.sub_resource + 1 == loc2.sub_resource) { - /* Dirty range covers a single sub-resource */ - vmw_subres_dirty_add(dirty, &loc1, &loc2); - } else { - /* Dirty range covers multiple sub-resources */ - struct svga3dsurface_loc loc_min, loc_max; - u32 sub_res = loc1.sub_resource; - - svga3dsurface_max_loc(cache, loc1.sub_resource, &loc_max); - vmw_subres_dirty_add(dirty, &loc1, &loc_max); - svga3dsurface_min_loc(cache, loc2.sub_resource - 1, &loc_min); - vmw_subres_dirty_add(dirty, &loc_min, &loc2); - for (sub_res = loc1.sub_resource + 1; - sub_res < loc2.sub_resource - 1; ++sub_res) - vmw_subres_dirty_full(dirty, sub_res); - } -} - -/* - * vmw_surface_tex_dirty_add_range - The dirty_add_range callback for buffer - * surfaces. - */ -static void vmw_surface_buf_dirty_range_add(struct vmw_resource *res, - size_t start, size_t end) -{ - struct vmw_surface_dirty *dirty = - (struct vmw_surface_dirty *) res->dirty; - const struct svga3dsurface_cache *cache = &dirty->cache; - size_t backup_end = res->backup_offset + cache->mip_chain_bytes; - SVGA3dBox *box = &dirty->boxes[0]; - u32 box_c2; - - box->h = box->d = 1; - start = max_t(size_t, start, res->backup_offset) - res->backup_offset; - end = min(end, backup_end) - res->backup_offset; - box_c2 = box->x + box->w; - if (box->w == 0 || box->x > start) - box->x = start; - if (box_c2 < end) - box->w = end - box->x; -} - -/* - * vmw_surface_tex_dirty_add_range - The dirty_add_range callback for surfaces - */ -static void vmw_surface_dirty_range_add(struct vmw_resource *res, size_t start, - size_t end) -{ - struct vmw_surface *srf = vmw_res_to_srf(res); - - if (WARN_ON(end <= res->backup_offset || - start >= res->backup_offset + res->backup_size)) - return; - - if (srf->format == SVGA3D_BUFFER) - vmw_surface_buf_dirty_range_add(res, start, end); - else - vmw_surface_tex_dirty_range_add(res, start, end); -} - -/* - * vmw_surface_dirty_sync - The surface's dirty_sync callback. - */ -static int vmw_surface_dirty_sync(struct vmw_resource *res) -{ - struct vmw_private *dev_priv = res->dev_priv; - bool has_dx = 0; - u32 i, num_dirty; - struct vmw_surface_dirty *dirty = - (struct vmw_surface_dirty *) res->dirty; - size_t alloc_size; - const struct svga3dsurface_cache *cache = &dirty->cache; - struct { - SVGA3dCmdHeader header; - SVGA3dCmdDXUpdateSubResource body; - } *cmd1; - struct { - SVGA3dCmdHeader header; - SVGA3dCmdUpdateGBImage body; - } *cmd2; - void *cmd; - - num_dirty = 0; - for (i = 0; i < dirty->num_subres; ++i) { - const SVGA3dBox *box = &dirty->boxes[i]; - - if (box->d) - num_dirty++; - } - - if (!num_dirty) - goto out; - - alloc_size = num_dirty * ((has_dx) ? sizeof(*cmd1) : sizeof(*cmd2)); - cmd = VMW_FIFO_RESERVE(dev_priv, alloc_size); - if (!cmd) - return -ENOMEM; - - cmd1 = cmd; - cmd2 = cmd; - - for (i = 0; i < dirty->num_subres; ++i) { - const SVGA3dBox *box = &dirty->boxes[i]; - - if (!box->d) - continue; - - /* - * DX_UPDATE_SUBRESOURCE is aware of array surfaces. - * UPDATE_GB_IMAGE is not. - */ - if (has_dx) { - cmd1->header.id = SVGA_3D_CMD_DX_UPDATE_SUBRESOURCE; - cmd1->header.size = sizeof(cmd1->body); - cmd1->body.sid = res->id; - cmd1->body.subResource = i; - cmd1->body.box = *box; - cmd1++; - } else { - cmd2->header.id = SVGA_3D_CMD_UPDATE_GB_IMAGE; - cmd2->header.size = sizeof(cmd2->body); - cmd2->body.image.sid = res->id; - cmd2->body.image.face = i / cache->num_mip_levels; - cmd2->body.image.mipmap = i - - (cache->num_mip_levels * cmd2->body.image.face); - cmd2->body.box = *box; - cmd2++; - } - - } - vmw_fifo_commit(dev_priv, alloc_size); - out: - memset(&dirty->boxes[0], 0, sizeof(dirty->boxes[0]) * - dirty->num_subres); - - return 0; -} - -/* - * vmw_surface_dirty_alloc - The surface's dirty_alloc callback. - */ -static int vmw_surface_dirty_alloc(struct vmw_resource *res) -{ - struct vmw_surface *srf = vmw_res_to_srf(res); - struct vmw_surface_dirty *dirty; - u32 num_layers = 1; - u32 num_mip; - u32 num_subres; - u32 num_samples; - size_t dirty_size, acc_size; - static struct ttm_operation_ctx ctx = { - .interruptible = false, - .no_wait_gpu = false - }; - int ret; - - if (srf->array_size) - num_layers = srf->array_size; - else if (srf->flags & SVGA3D_SURFACE_CUBEMAP) - num_layers *= SVGA3D_MAX_SURFACE_FACES; - - num_mip = srf->mip_levels[0]; - if (!num_mip) - num_mip = 1; - - num_subres = num_layers * num_mip; - dirty_size = sizeof(*dirty) + num_subres * sizeof(dirty->boxes[0]); - acc_size = ttm_round_pot(dirty_size); - ret = ttm_mem_global_alloc(vmw_mem_glob(res->dev_priv), - acc_size, &ctx); - if (ret) { - VMW_DEBUG_USER("Out of graphics memory for surface " - "dirty tracker.\n"); - return ret; - } - - dirty = kvzalloc(dirty_size, GFP_KERNEL); - if (!dirty) { - ret = -ENOMEM; - goto out_no_dirty; - } - - num_samples = max_t(u32, 1, srf->multisample_count); - ret = svga3dsurface_setup_cache(&srf->base_size, srf->format, num_mip, - num_layers, num_samples, &dirty->cache); - if (ret) - goto out_no_cache; - - dirty->num_subres = num_subres; - dirty->size = acc_size; - res->dirty = (struct vmw_resource_dirty *) dirty; - - return 0; - -out_no_cache: - kvfree(dirty); -out_no_dirty: - ttm_mem_global_free(vmw_mem_glob(res->dev_priv), acc_size); - return ret; -} - -/* - * vmw_surface_dirty_free - The surface's dirty_free callback - */ -static void vmw_surface_dirty_free(struct vmw_resource *res) -{ - struct vmw_surface_dirty *dirty = - (struct vmw_surface_dirty *) res->dirty; - size_t acc_size = dirty->size; - - kvfree(dirty); - ttm_mem_global_free(vmw_mem_glob(res->dev_priv), acc_size); - res->dirty = NULL; -} - -/* - * vmw_surface_clean - The surface's clean callback - */ -static int vmw_surface_clean(struct vmw_resource *res) -{ - struct vmw_private *dev_priv = res->dev_priv; - size_t alloc_size; - struct { - SVGA3dCmdHeader header; - SVGA3dCmdReadbackGBSurface body; - } *cmd; - - alloc_size = sizeof(*cmd); - cmd = VMW_FIFO_RESERVE(dev_priv, alloc_size); - if (!cmd) - return -ENOMEM; - - cmd->header.id = SVGA_3D_CMD_READBACK_GB_SURFACE; - cmd->header.size = sizeof(cmd->body); - cmd->body.sid = res->id; - vmw_fifo_commit(dev_priv, alloc_size); - - return 0; -} diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c index 9aaf807ed73c..f611b2290a1b 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.c @@ -33,8 +33,6 @@ * struct vmw_validation_bo_node - Buffer object validation metadata. * @base: Metadata used for TTM reservation- and validation. * @hash: A hash entry used for the duplicate detection hash table. - * @coherent_count: If switching backup buffers, number of new coherent - * resources that will have this buffer as a backup buffer. * @as_mob: Validate as mob. * @cpu_blit: Validate for cpu blit access. * @@ -44,7 +42,6 @@ struct vmw_validation_bo_node { struct ttm_validate_buffer base; struct drm_hash_item hash; - unsigned int coherent_count; u32 as_mob : 1; u32 cpu_blit : 1; }; @@ -462,19 +459,6 @@ int vmw_validation_res_reserve(struct vmw_validation_context *ctx, if (ret) goto out_unreserve; } - - if (val->switching_backup && val->new_backup && - res->coherent) { - struct vmw_validation_bo_node *bo_node = - vmw_validation_find_bo_dup(ctx, - val->new_backup); - - if (WARN_ON(!bo_node)) { - ret = -EINVAL; - goto out_unreserve; - } - bo_node->coherent_count++; - } } return 0; @@ -578,9 +562,6 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr) int ret; list_for_each_entry(entry, &ctx->bo_list, base.head) { - struct vmw_buffer_object *vbo = - container_of(entry->base.bo, typeof(*vbo), base); - if (entry->cpu_blit) { struct ttm_operation_ctx ctx = { .interruptible = intr, @@ -595,27 +576,6 @@ int vmw_validation_bo_validate(struct vmw_validation_context *ctx, bool intr) } if (ret) return ret; - - /* - * Rather than having the resource code allocating the bo - * dirty tracker in resource_unreserve() where we can't fail, - * Do it here when validating the buffer object. - */ - if (entry->coherent_count) { - unsigned int coherent_count = entry->coherent_count; - - while (coherent_count) { - ret = vmw_bo_dirty_add(vbo); - if (ret) - return ret; - - coherent_count--; - } - entry->coherent_count -= coherent_count; - } - - if (vbo->dirty) - vmw_bo_dirty_scan(vbo); } return 0; } @@ -641,8 +601,7 @@ int vmw_validation_res_validate(struct vmw_validation_context *ctx, bool intr) struct vmw_resource *res = val->res; struct vmw_buffer_object *backup = res->backup; - ret = vmw_resource_validate(res, intr, val->dirty_set && - val->dirty); + ret = vmw_resource_validate(res, intr); if (ret) { if (ret != -ERESTARTSYS) DRM_ERROR("Failed to validate resource.\n"); @@ -869,34 +828,3 @@ int vmw_validation_preload_res(struct vmw_validation_context *ctx, ctx->mem_size_left += size; return 0; } - -/** - * vmw_validation_bo_backoff - Unreserve buffer objects registered with a - * validation context - * @ctx: The validation context - * - * This function unreserves the buffer objects previously reserved using - * vmw_validation_bo_reserve. It's typically used as part of an error path - */ -void vmw_validation_bo_backoff(struct vmw_validation_context *ctx) -{ - struct vmw_validation_bo_node *entry; - - /* - * Switching coherent resource backup buffers failed. - * Release corresponding buffer object dirty trackers. - */ - list_for_each_entry(entry, &ctx->bo_list, base.head) { - if (entry->coherent_count) { - unsigned int coherent_count = entry->coherent_count; - struct vmw_buffer_object *vbo = - container_of(entry->base.bo, typeof(*vbo), - base); - - while (coherent_count--) - vmw_bo_dirty_release(vbo); - } - } - - ttm_eu_backoff_reservation(&ctx->ticket, &ctx->bo_list); -} diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h index fd83e017c2a5..1d2322ad6fd5 100644 --- a/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h +++ b/drivers/gpu/drm/vmwgfx/vmwgfx_validation.h @@ -172,6 +172,20 @@ vmw_validation_bo_reserve(struct vmw_validation_context *ctx, NULL, true); } +/** + * vmw_validation_bo_backoff - Unreserve buffer objects registered with a + * validation context + * @ctx: The validation context + * + * This function unreserves the buffer objects previously reserved using + * vmw_validation_bo_reserve. It's typically used as part of an error path + */ +static inline void +vmw_validation_bo_backoff(struct vmw_validation_context *ctx) +{ + ttm_eu_backoff_reservation(&ctx->ticket, &ctx->bo_list); +} + /** * vmw_validation_bo_fence - Unreserve and fence buffer objects registered * with a validation context @@ -254,6 +268,4 @@ int vmw_validation_preload_res(struct vmw_validation_context *ctx, unsigned int size); void vmw_validation_res_set_dirty(struct vmw_validation_context *ctx, void *val_private, u32 dirty); -void vmw_validation_bo_backoff(struct vmw_validation_context *ctx); - #endif diff --git a/include/drm/ttm/ttm_bo_api.h b/include/drm/ttm/ttm_bo_api.h index 435d02f719a8..49d9cdfc58f2 100644 --- a/include/drm/ttm/ttm_bo_api.h +++ b/include/drm/ttm/ttm_bo_api.h @@ -768,14 +768,4 @@ int ttm_bo_swapout(struct ttm_bo_global *glob, struct ttm_operation_ctx *ctx); void ttm_bo_swapout_all(struct ttm_bo_device *bdev); int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo); - -/* Default number of pre-faulted pages in the TTM fault handler */ -#define TTM_BO_VM_NUM_PREFAULT 16 - -vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, - struct vm_fault *vmf); - -vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, - pgprot_t prot, - pgoff_t num_prefault); #endif diff --git a/include/drm/ttm/ttm_bo_driver.h b/include/drm/ttm/ttm_bo_driver.h index a2d810a2504d..c9b8ba492f24 100644 --- a/include/drm/ttm/ttm_bo_driver.h +++ b/include/drm/ttm/ttm_bo_driver.h @@ -442,9 +442,6 @@ extern struct ttm_bo_global { * @driver: Pointer to a struct ttm_bo_driver struct setup by the driver. * @man: An array of mem_type_managers. * @vma_manager: Address space manager - * @vm_ops: Pointer to the struct vm_operations_struct used for this - * device's VM operations. The driver may override this before the first - * mmap() call. * lru_lock: Spinlock that protects the buffer+device lru lists and * ddestroy lists. * @dev_mapping: A pointer to the struct address_space representing the @@ -463,7 +460,6 @@ struct ttm_bo_device { struct ttm_bo_global *glob; struct ttm_bo_driver *driver; struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; - const struct vm_operations_struct *vm_ops; /* * Protected by internal locks. @@ -492,8 +488,6 @@ struct ttm_bo_device { bool no_retry; }; -extern const struct vm_operations_struct ttm_bo_vm_ops; - /** * struct ttm_lru_bulk_move_pos * diff --git a/include/linux/mm.h b/include/linux/mm.h index 798cdda9560e..dd0b5f4e1e45 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2686,24 +2686,7 @@ typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); -struct pfn_range_apply; -typedef int (*pter_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, - struct pfn_range_apply *closure); -struct pfn_range_apply { - struct mm_struct *mm; - pter_fn_t ptefn; - unsigned int alloc; -}; -extern int apply_to_pfn_range(struct pfn_range_apply *closure, - unsigned long address, unsigned long size); -unsigned long apply_as_wrprotect(struct address_space *mapping, - pgoff_t first_index, pgoff_t nr); -unsigned long apply_as_clean(struct address_space *mapping, - pgoff_t first_index, pgoff_t nr, - pgoff_t bitmap_pgoff, - unsigned long *bitmap, - pgoff_t *start, - pgoff_t *end); + #ifdef CONFIG_PAGE_POISONING extern bool page_poisoning_enabled(void); extern void kernel_poison_pages(struct page *page, int numpages, int enable); diff --git a/include/uapi/drm/vmwgfx_drm.h b/include/uapi/drm/vmwgfx_drm.h index 02cab33f2f25..399f58317cff 100644 --- a/include/uapi/drm/vmwgfx_drm.h +++ b/include/uapi/drm/vmwgfx_drm.h @@ -891,13 +891,11 @@ struct drm_vmw_shader_arg { * surface. * @drm_vmw_surface_flag_create_buffer: Create a backup buffer if none is * given. - * @drm_vmw_surface_flag_coherent: Back surface with coherent memory. */ enum drm_vmw_surface_flags { drm_vmw_surface_flag_shareable = (1 << 0), drm_vmw_surface_flag_scanout = (1 << 1), - drm_vmw_surface_flag_create_buffer = (1 << 2), - drm_vmw_surface_flag_coherent = (1 << 3), + drm_vmw_surface_flag_create_buffer = (1 << 2) }; /** diff --git a/mm/Kconfig b/mm/Kconfig index 5006d0e6a5c7..f0c76ba47695 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -765,7 +765,4 @@ config GUP_BENCHMARK config ARCH_HAS_PTE_SPECIAL bool -config AS_DIRTY_HELPERS - bool - endmenu diff --git a/mm/Makefile b/mm/Makefile index f5d412bbc2f7..ac5e5ba78874 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -104,4 +104,3 @@ obj-$(CONFIG_HARDENED_USERCOPY) += usercopy.o obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o obj-$(CONFIG_HMM) += hmm.o obj-$(CONFIG_MEMFD_CREATE) += memfd.o -obj-$(CONFIG_AS_DIRTY_HELPERS) += as_dirty_helpers.o diff --git a/mm/as_dirty_helpers.c b/mm/as_dirty_helpers.c deleted file mode 100644 index f600e31534fb..000000000000 --- a/mm/as_dirty_helpers.c +++ /dev/null @@ -1,300 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#include -#include -#include -#include -#include -#include -#include - -/** - * struct apply_as - Closure structure for apply_as_range - * @base: struct pfn_range_apply we derive from - * @start: Address of first modified pte - * @end: Address of last modified pte + 1 - * @total: Total number of modified ptes - * @vma: Pointer to the struct vm_area_struct we're currently operating on - */ -struct apply_as { - struct pfn_range_apply base; - unsigned long start; - unsigned long end; - unsigned long total; - struct vm_area_struct *vma; -}; - -/** - * apply_pt_wrprotect - Leaf pte callback to write-protect a pte - * @pte: Pointer to the pte - * @token: Page table token, see apply_to_pfn_range() - * @addr: The virtual page address - * @closure: Pointer to a struct pfn_range_apply embedded in a - * struct apply_as - * - * The function write-protects a pte and records the range in - * virtual address space of touched ptes for efficient range TLB flushes. - * - * Return: Always zero. - */ -static int apply_pt_wrprotect(pte_t *pte, pgtable_t token, - unsigned long addr, - struct pfn_range_apply *closure) -{ - struct apply_as *aas = container_of(closure, typeof(*aas), base); - pte_t ptent = *pte; - - if (pte_write(ptent)) { - pte_t old_pte = ptep_modify_prot_start(aas->vma, addr, pte); - - ptent = pte_wrprotect(old_pte); - ptep_modify_prot_commit(aas->vma, addr, pte, old_pte, ptent); - aas->total++; - aas->start = min(aas->start, addr); - aas->end = max(aas->end, addr + PAGE_SIZE); - } - - return 0; -} - -/** - * struct apply_as_clean - Closure structure for apply_as_clean - * @base: struct apply_as we derive from - * @bitmap_pgoff: Address_space Page offset of the first bit in @bitmap - * @bitmap: Bitmap with one bit for each page offset in the address_space range - * covered. - * @start: Address_space page offset of first modified pte relative - * to @bitmap_pgoff - * @end: Address_space page offset of last modified pte relative - * to @bitmap_pgoff - */ -struct apply_as_clean { - struct apply_as base; - pgoff_t bitmap_pgoff; - unsigned long *bitmap; - pgoff_t start; - pgoff_t end; -}; - -/** - * apply_pt_clean - Leaf pte callback to clean a pte - * @pte: Pointer to the pte - * @token: Page table token, see apply_to_pfn_range() - * @addr: The virtual page address - * @closure: Pointer to a struct pfn_range_apply embedded in a - * struct apply_as_clean - * - * The function cleans a pte and records the range in - * virtual address space of touched ptes for efficient TLB flushes. - * It also records dirty ptes in a bitmap representing page offsets - * in the address_space, as well as the first and last of the bits - * touched. - * - * Return: Always zero. - */ -static int apply_pt_clean(pte_t *pte, pgtable_t token, - unsigned long addr, - struct pfn_range_apply *closure) -{ - struct apply_as *aas = container_of(closure, typeof(*aas), base); - struct apply_as_clean *clean = container_of(aas, typeof(*clean), base); - pte_t ptent = *pte; - - if (pte_dirty(ptent)) { - pgoff_t pgoff = ((addr - aas->vma->vm_start) >> PAGE_SHIFT) + - aas->vma->vm_pgoff - clean->bitmap_pgoff; - pte_t old_pte = ptep_modify_prot_start(aas->vma, addr, pte); - - ptent = pte_mkclean(old_pte); - ptep_modify_prot_commit(aas->vma, addr, pte, old_pte, ptent); - - aas->total++; - aas->start = min(aas->start, addr); - aas->end = max(aas->end, addr + PAGE_SIZE); - - __set_bit(pgoff, clean->bitmap); - clean->start = min(clean->start, pgoff); - clean->end = max(clean->end, pgoff + 1); - } - - return 0; -} - -/** - * apply_as_range - Apply a pte callback to all PTEs pointing into a range - * of an address_space. - * @mapping: Pointer to the struct address_space - * @aas: Closure structure - * @first_index: First page offset in the address_space - * @nr: Number of incremental page offsets to cover - * - * Return: Number of ptes touched. Note that this number might be larger - * than @nr if there are overlapping vmas - */ -static unsigned long apply_as_range(struct address_space *mapping, - struct apply_as *aas, - pgoff_t first_index, pgoff_t nr) -{ - struct vm_area_struct *vma; - pgoff_t vba, vea, cba, cea; - unsigned long start_addr, end_addr; - struct mmu_notifier_range range; - - i_mmap_lock_read(mapping); - vma_interval_tree_foreach(vma, &mapping->i_mmap, first_index, - first_index + nr - 1) { - unsigned long vm_flags = READ_ONCE(vma->vm_flags); - - /* - * We can only do advisory flag tests below, since we can't - * require the vm's mmap_sem to be held to protect the flags. - * Therefore, callers that strictly depend on specific mmap - * flags to remain constant throughout the operation must - * either ensure those flags are immutable for all relevant - * vmas or can't use this function. Fixing this properly would - * require the vma::vm_flags to be protected by a separate - * lock taken after the i_mmap_lock - */ - - /* Skip non-applicable VMAs */ - if ((vm_flags & (VM_SHARED | VM_WRITE)) != - (VM_SHARED | VM_WRITE)) - continue; - - /* Warn on and skip VMAs whose flags indicate illegal usage */ - if (WARN_ON((vm_flags & (VM_HUGETLB | VM_IO)) != VM_IO)) - continue; - - /* Clip to the vma */ - vba = vma->vm_pgoff; - vea = vba + vma_pages(vma); - cba = first_index; - cba = max(cba, vba); - cea = first_index + nr; - cea = min(cea, vea); - - /* Translate to virtual address */ - start_addr = ((cba - vba) << PAGE_SHIFT) + vma->vm_start; - end_addr = ((cea - vba) << PAGE_SHIFT) + vma->vm_start; - if (start_addr >= end_addr) - continue; - - aas->base.mm = vma->vm_mm; - aas->vma = vma; - aas->start = end_addr; - aas->end = start_addr; - - mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0, - vma, vma->vm_mm, start_addr, end_addr); - mmu_notifier_invalidate_range_start(&range); - - /* Needed when we only change protection? */ - flush_cache_range(vma, start_addr, end_addr); - - /* - * We're not using tlb_gather_mmu() since typically - * only a small subrange of PTEs are affected. - */ - inc_tlb_flush_pending(vma->vm_mm); - - /* Should not error since aas->base.alloc == 0 */ - WARN_ON(apply_to_pfn_range(&aas->base, start_addr, - end_addr - start_addr)); - if (aas->end > aas->start) - flush_tlb_range(vma, aas->start, aas->end); - - mmu_notifier_invalidate_range_end(&range); - dec_tlb_flush_pending(vma->vm_mm); - } - i_mmap_unlock_read(mapping); - - return aas->total; -} - -/** - * apply_as_wrprotect - Write-protect all ptes in an address_space range - * @mapping: The address_space we want to write protect - * @first_index: The first page offset in the range - * @nr: Number of incremental page offsets to cover - * - * WARNING: This function should only be used for address spaces whose - * vmas are marked VM_IO and that do not contain huge pages. - * To avoid interference with COW'd pages, vmas not marked VM_SHARED are - * simply skipped. - * - * Return: The number of ptes actually write-protected. Note that - * already write-protected ptes are not counted. - */ -unsigned long apply_as_wrprotect(struct address_space *mapping, - pgoff_t first_index, pgoff_t nr) -{ - struct apply_as aas = { - .base = { - .alloc = 0, - .ptefn = apply_pt_wrprotect, - }, - .total = 0, - }; - - return apply_as_range(mapping, &aas, first_index, nr); -} -EXPORT_SYMBOL_GPL(apply_as_wrprotect); - -/** - * apply_as_clean - Clean all ptes in an address_space range - * @mapping: The address_space we want to clean - * @first_index: The first page offset in the range - * @nr: Number of incremental page offsets to cover - * @bitmap_pgoff: The page offset of the first bit in @bitmap - * @bitmap: Pointer to a bitmap of at least @nr bits. The bitmap needs to - * cover the whole range @first_index..@first_index + @nr. - * @start: Pointer to number of the first set bit in @bitmap. - * is modified as new bits are set by the function. - * @end: Pointer to the number of the last set bit in @bitmap. - * none set. The value is modified as new bits are set by the function. - * - * Note: When this function returns there is no guarantee that a CPU has - * not already dirtied new ptes. However it will not clean any ptes not - * reported in the bitmap. - * - * If a caller needs to make sure all dirty ptes are picked up and none - * additional are added, it first needs to write-protect the address-space - * range and make sure new writers are blocked in page_mkwrite() or - * pfn_mkwrite(). And then after a TLB flush following the write-protection - * pick up all dirty bits. - * - * WARNING: This function should only be used for address spaces whose - * vmas are marked VM_IO and that do not contain huge pages. - * To avoid interference with COW'd pages, vmas not marked VM_SHARED are - * simply skipped. - * - * Return: The number of dirty ptes actually cleaned. - */ -unsigned long apply_as_clean(struct address_space *mapping, - pgoff_t first_index, pgoff_t nr, - pgoff_t bitmap_pgoff, - unsigned long *bitmap, - pgoff_t *start, - pgoff_t *end) -{ - bool none_set = (*start >= *end); - struct apply_as_clean clean = { - .base = { - .base = { - .alloc = 0, - .ptefn = apply_pt_clean, - }, - .total = 0, - }, - .bitmap_pgoff = bitmap_pgoff, - .bitmap = bitmap, - .start = none_set ? nr : *start, - .end = none_set ? 0 : *end, - }; - unsigned long ret = apply_as_range(mapping, &clean.base, first_index, - nr); - - *start = clean.start; - *end = clean.end; - return ret; -} -EXPORT_SYMBOL_GPL(apply_as_clean); diff --git a/mm/memory.c b/mm/memory.c index 462aa47f8878..ddf20bd0c317 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2032,17 +2032,18 @@ int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long } EXPORT_SYMBOL(vm_iomap_memory); -static int apply_to_pte_range(struct pfn_range_apply *closure, pmd_t *pmd, - unsigned long addr, unsigned long end) +static int apply_to_pte_range(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, unsigned long end, + pte_fn_t fn, void *data) { pte_t *pte; int err; pgtable_t token; spinlock_t *uninitialized_var(ptl); - pte = (closure->mm == &init_mm) ? + pte = (mm == &init_mm) ? pte_alloc_kernel(pmd, addr) : - pte_alloc_map_lock(closure->mm, pmd, addr, &ptl); + pte_alloc_map_lock(mm, pmd, addr, &ptl); if (!pte) return -ENOMEM; @@ -2053,109 +2054,86 @@ static int apply_to_pte_range(struct pfn_range_apply *closure, pmd_t *pmd, token = pmd_pgtable(*pmd); do { - err = closure->ptefn(pte++, token, addr, closure); + err = fn(pte++, token, addr, data); if (err) break; } while (addr += PAGE_SIZE, addr != end); arch_leave_lazy_mmu_mode(); - if (closure->mm != &init_mm) + if (mm != &init_mm) pte_unmap_unlock(pte-1, ptl); return err; } -static int apply_to_pmd_range(struct pfn_range_apply *closure, pud_t *pud, - unsigned long addr, unsigned long end) +static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, + unsigned long addr, unsigned long end, + pte_fn_t fn, void *data) { pmd_t *pmd; unsigned long next; - int err = 0; + int err; BUG_ON(pud_huge(*pud)); - pmd = pmd_alloc(closure->mm, pud, addr); + pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; - do { next = pmd_addr_end(addr, end); - if (!closure->alloc && pmd_none_or_clear_bad(pmd)) - continue; - err = apply_to_pte_range(closure, pmd, addr, next); + err = apply_to_pte_range(mm, pmd, addr, next, fn, data); if (err) break; } while (pmd++, addr = next, addr != end); return err; } -static int apply_to_pud_range(struct pfn_range_apply *closure, p4d_t *p4d, - unsigned long addr, unsigned long end) +static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, + unsigned long addr, unsigned long end, + pte_fn_t fn, void *data) { pud_t *pud; unsigned long next; - int err = 0; + int err; - pud = pud_alloc(closure->mm, p4d, addr); + pud = pud_alloc(mm, p4d, addr); if (!pud) return -ENOMEM; - do { next = pud_addr_end(addr, end); - if (!closure->alloc && pud_none_or_clear_bad(pud)) - continue; - err = apply_to_pmd_range(closure, pud, addr, next); + err = apply_to_pmd_range(mm, pud, addr, next, fn, data); if (err) break; } while (pud++, addr = next, addr != end); return err; } -static int apply_to_p4d_range(struct pfn_range_apply *closure, pgd_t *pgd, - unsigned long addr, unsigned long end) +static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd, + unsigned long addr, unsigned long end, + pte_fn_t fn, void *data) { p4d_t *p4d; unsigned long next; - int err = 0; + int err; - p4d = p4d_alloc(closure->mm, pgd, addr); + p4d = p4d_alloc(mm, pgd, addr); if (!p4d) return -ENOMEM; - do { next = p4d_addr_end(addr, end); - if (!closure->alloc && p4d_none_or_clear_bad(p4d)) - continue; - err = apply_to_pud_range(closure, p4d, addr, next); + err = apply_to_pud_range(mm, p4d, addr, next, fn, data); if (err) break; } while (p4d++, addr = next, addr != end); return err; } -/** - * apply_to_pfn_range - Scan a region of virtual memory, calling a provided - * function on each leaf page table entry - * @closure: Details about how to scan and what function to apply - * @addr: Start virtual address - * @size: Size of the region - * - * If @closure->alloc is set to 1, the function will fill in the page table - * as necessary. Otherwise it will skip non-present parts. - * Note: The caller must ensure that the range does not contain huge pages. - * The caller must also assure that the proper mmu_notifier functions are - * called before and after the call to apply_to_pfn_range. - * - * WARNING: Do not use this function unless you know exactly what you are - * doing. It is lacking support for huge pages and transparent huge pages. - * - * Return: Zero on success. If the provided function returns a non-zero status, - * the page table walk will terminate and that status will be returned. - * If @closure->alloc is set to 1, then this function may also return memory - * allocation errors arising from allocating page table memory. +/* + * Scan a region of virtual memory, filling in page tables as necessary + * and calling a provided function on each leaf page table. */ -int apply_to_pfn_range(struct pfn_range_apply *closure, - unsigned long addr, unsigned long size) +int apply_to_page_range(struct mm_struct *mm, unsigned long addr, + unsigned long size, pte_fn_t fn, void *data) { pgd_t *pgd; unsigned long next; @@ -2165,65 +2143,16 @@ int apply_to_pfn_range(struct pfn_range_apply *closure, if (WARN_ON(addr >= end)) return -EINVAL; - pgd = pgd_offset(closure->mm, addr); + pgd = pgd_offset(mm, addr); do { next = pgd_addr_end(addr, end); - if (!closure->alloc && pgd_none_or_clear_bad(pgd)) - continue; - err = apply_to_p4d_range(closure, pgd, addr, next); + err = apply_to_p4d_range(mm, pgd, addr, next, fn, data); if (err) break; } while (pgd++, addr = next, addr != end); return err; } - -/** - * struct page_range_apply - Closure structure for apply_to_page_range() - * @pter: The base closure structure we derive from - * @fn: The leaf pte function to call - * @data: The leaf pte function closure - */ -struct page_range_apply { - struct pfn_range_apply pter; - pte_fn_t fn; - void *data; -}; - -/* - * Callback wrapper to enable use of apply_to_pfn_range for - * the apply_to_page_range interface - */ -static int apply_to_page_range_wrapper(pte_t *pte, pgtable_t token, - unsigned long addr, - struct pfn_range_apply *pter) -{ - struct page_range_apply *pra = - container_of(pter, typeof(*pra), pter); - - return pra->fn(pte, token, addr, pra->data); -} - -/* - * Scan a region of virtual memory, filling in page tables as necessary - * and calling a provided function on each leaf page table. - * - * WARNING: Do not use this function unless you know exactly what you are - * doing. It is lacking support for huge pages and transparent huge pages. - */ -int apply_to_page_range(struct mm_struct *mm, unsigned long addr, - unsigned long size, pte_fn_t fn, void *data) -{ - struct page_range_apply pra = { - .pter = {.mm = mm, - .alloc = 1, - .ptefn = apply_to_page_range_wrapper }, - .fn = fn, - .data = data - }; - - return apply_to_pfn_range(&pra.pter, addr, size); -} EXPORT_SYMBOL_GPL(apply_to_page_range); /* @@ -2309,7 +2238,7 @@ static vm_fault_t do_page_mkwrite(struct vm_fault *vmf) ret = vmf->vma->vm_ops->page_mkwrite(vmf); /* Restore original flags so that caller is not surprised */ vmf->flags = old_flags; - if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY))) + if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE))) return ret; if (unlikely(!(ret & VM_FAULT_LOCKED))) { lock_page(page); @@ -2586,7 +2515,7 @@ static vm_fault_t wp_pfn_shared(struct vm_fault *vmf) pte_unmap_unlock(vmf->pte, vmf->ptl); vmf->flags |= FAULT_FLAG_MKWRITE; ret = vma->vm_ops->pfn_mkwrite(vmf); - if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)) + if (ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)) return ret; return finish_mkwrite_fault(vmf); } @@ -2607,8 +2536,7 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf) pte_unmap_unlock(vmf->pte, vmf->ptl); tmp = do_page_mkwrite(vmf); if (unlikely(!tmp || (tmp & - (VM_FAULT_ERROR | VM_FAULT_NOPAGE | - VM_FAULT_RETRY)))) { + (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { put_page(vmf->page); return tmp; } @@ -3673,8 +3601,7 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf) unlock_page(vmf->page); tmp = do_page_mkwrite(vmf); if (unlikely(!tmp || - (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | - VM_FAULT_RETRY)))) { + (tmp & (VM_FAULT_ERROR | VM_FAULT_NOPAGE)))) { put_page(vmf->page); return tmp; } -- cgit v1.2.3-59-g8ed1b