aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c')
-rw-r--r--drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c95
1 files changed, 77 insertions, 18 deletions
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index ec0d62a16e53..a3b933967171 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -109,7 +109,7 @@ static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
{
return dqm->dev->device_info->num_sdma_engines
- * KFD_SDMA_QUEUES_PER_ENGINE;
+ * dqm->dev->device_info->num_sdma_queues_per_engine;
}
void program_sh_mem_settings(struct device_queue_manager *dqm,
@@ -358,8 +358,8 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
struct queue *q,
struct qcm_process_device *qpd)
{
- int retval;
struct mqd_manager *mqd_mgr;
+ int retval;
mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
if (!mqd_mgr)
@@ -387,8 +387,12 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm,
if (!q->properties.is_active)
return 0;
- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
- &q->properties, q->process->mm);
+ if (WARN(q->process->mm != current->mm,
+ "should only run in user thread"))
+ retval = -EFAULT;
+ else
+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
+ &q->properties, current->mm);
if (retval)
goto out_uninit_mqd;
@@ -545,9 +549,15 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q)
retval = map_queues_cpsch(dqm);
else if (q->properties.is_active &&
(q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
- q->properties.type == KFD_QUEUE_TYPE_SDMA))
- retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
- &q->properties, q->process->mm);
+ q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
+ if (WARN(q->process->mm != current->mm,
+ "should only run in user thread"))
+ retval = -EFAULT;
+ else
+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd,
+ q->pipe, q->queue,
+ &q->properties, current->mm);
+ }
out_unlock:
dqm_unlock(dqm);
@@ -653,10 +663,11 @@ out:
static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
{
+ struct mm_struct *mm = NULL;
struct queue *q;
struct mqd_manager *mqd_mgr;
struct kfd_process_device *pdd;
- uint32_t pd_base;
+ uint64_t pd_base;
int retval = 0;
pdd = qpd_to_pdd(qpd);
@@ -676,7 +687,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
/* Update PD Base in QPD */
qpd->page_table_base = pd_base;
- pr_debug("Updated PD address to 0x%08x\n", pd_base);
+ pr_debug("Updated PD address to 0x%llx\n", pd_base);
if (!list_empty(&qpd->queues_list)) {
dqm->dev->kfd2kgd->set_vm_context_page_table_base(
@@ -686,6 +697,15 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
kfd_flush_tlb(pdd);
}
+ /* Take a safe reference to the mm_struct, which may otherwise
+ * disappear even while the kfd_process is still referenced.
+ */
+ mm = get_task_mm(pdd->process->lead_thread);
+ if (!mm) {
+ retval = -EFAULT;
+ goto out;
+ }
+
/* activate all active queues on the qpd */
list_for_each_entry(q, &qpd->queues_list, list) {
if (!q->properties.is_evicted)
@@ -700,14 +720,15 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm,
q->properties.is_evicted = false;
q->properties.is_active = true;
retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
- q->queue, &q->properties,
- q->process->mm);
+ q->queue, &q->properties, mm);
if (retval)
goto out;
dqm->queue_count++;
}
qpd->evicted = 0;
out:
+ if (mm)
+ mmput(mm);
dqm_unlock(dqm);
return retval;
}
@@ -717,7 +738,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
{
struct queue *q;
struct kfd_process_device *pdd;
- uint32_t pd_base;
+ uint64_t pd_base;
int retval = 0;
pdd = qpd_to_pdd(qpd);
@@ -737,7 +758,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm,
/* Update PD Base in QPD */
qpd->page_table_base = pd_base;
- pr_debug("Updated PD address to 0x%08x\n", pd_base);
+ pr_debug("Updated PD address to 0x%llx\n", pd_base);
/* activate all active queues on the qpd */
list_for_each_entry(q, &qpd->queues_list, list) {
@@ -761,7 +782,7 @@ static int register_process(struct device_queue_manager *dqm,
{
struct device_process_node *n;
struct kfd_process_device *pdd;
- uint32_t pd_base;
+ uint64_t pd_base;
int retval;
n = kzalloc(sizeof(*n), GFP_KERNEL);
@@ -779,6 +800,7 @@ static int register_process(struct device_queue_manager *dqm,
/* Update PD Base in QPD */
qpd->page_table_base = pd_base;
+ pr_debug("Updated PD address to 0x%llx\n", pd_base);
retval = dqm->asic_ops.update_qpd(dqm, qpd);
@@ -1342,9 +1364,6 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm,
{
int retval;
struct mqd_manager *mqd_mgr;
- bool preempt_all_queues;
-
- preempt_all_queues = false;
retval = 0;
@@ -1528,6 +1547,41 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm,
return retval;
}
+static int get_wave_state(struct device_queue_manager *dqm,
+ struct queue *q,
+ void __user *ctl_stack,
+ u32 *ctl_stack_used_size,
+ u32 *save_area_used_size)
+{
+ struct mqd_manager *mqd;
+ int r;
+
+ dqm_lock(dqm);
+
+ if (q->properties.type != KFD_QUEUE_TYPE_COMPUTE ||
+ q->properties.is_active || !q->device->cwsr_enabled) {
+ r = -EINVAL;
+ goto dqm_unlock;
+ }
+
+ mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
+ if (!mqd) {
+ r = -ENOMEM;
+ goto dqm_unlock;
+ }
+
+ if (!mqd->get_wave_state) {
+ r = -EINVAL;
+ goto dqm_unlock;
+ }
+
+ r = mqd->get_wave_state(mqd, q->mqd, ctl_stack, ctl_stack_used_size,
+ save_area_used_size);
+
+dqm_unlock:
+ dqm_unlock(dqm);
+ return r;
+}
static int process_termination_cpsch(struct device_queue_manager *dqm,
struct qcm_process_device *qpd)
@@ -1649,6 +1703,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
dqm->ops.process_termination = process_termination_cpsch;
dqm->ops.evict_process_queues = evict_process_queues_cpsch;
dqm->ops.restore_process_queues = restore_process_queues_cpsch;
+ dqm->ops.get_wave_state = get_wave_state;
break;
case KFD_SCHED_POLICY_NO_HWS:
/* initialize dqm for no cp scheduling */
@@ -1668,6 +1723,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
dqm->ops.evict_process_queues = evict_process_queues_nocpsch;
dqm->ops.restore_process_queues =
restore_process_queues_nocpsch;
+ dqm->ops.get_wave_state = get_wave_state;
break;
default:
pr_err("Invalid scheduling policy %d\n", dqm->sched_policy);
@@ -1695,6 +1751,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_dev *dev)
break;
case CHIP_VEGA10:
+ case CHIP_VEGA20:
case CHIP_RAVEN:
device_queue_manager_init_v9(&dqm->asic_ops);
break;
@@ -1806,7 +1863,9 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data)
}
for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) {
- for (queue = 0; queue < KFD_SDMA_QUEUES_PER_ENGINE; queue++) {
+ for (queue = 0;
+ queue < dqm->dev->device_info->num_sdma_queues_per_engine;
+ queue++) {
r = dqm->dev->kfd2kgd->hqd_sdma_dump(
dqm->dev->kgd, pipe, queue, &dump, &n_regs);
if (r)
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.