1 files changed, 144 insertions, 38 deletions

diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
index 30e7b7196dab..04e6f6f9b742 100644
--- a/drivers/gpu/drm/panfrost/panfrost_job.c
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -20,12 +20,22 @@
 #include "panfrost_gpu.h"
 #include "panfrost_mmu.h"
 
+#define JOB_TIMEOUT_MS 500
+
 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg))
 #define job_read(dev, reg) readl(dev->iomem + (reg))
 
+enum panfrost_queue_status {
+	PANFROST_QUEUE_STATUS_ACTIVE,
+	PANFROST_QUEUE_STATUS_STOPPED,
+	PANFROST_QUEUE_STATUS_STARTING,
+	PANFROST_QUEUE_STATUS_FAULT_PENDING,
+};
+
 struct panfrost_queue_state {
 	struct drm_gpu_scheduler sched;
-
+	atomic_t status;
+	struct mutex lock;
 	u64 fence_context;
 	u64 emit_seqno;
 };
@@ -369,13 +379,64 @@ void panfrost_job_enable_interrupts(struct panfrost_device *pfdev)
 	job_write(pfdev, JOB_INT_MASK, irq_mask);
 }
 
+static bool panfrost_scheduler_stop(struct panfrost_queue_state *queue,
+				    struct drm_sched_job *bad)
+{
+	enum panfrost_queue_status old_status;
+	bool stopped = false;
+
+	mutex_lock(&queue->lock);
+	old_status = atomic_xchg(&queue->status,
+				 PANFROST_QUEUE_STATUS_STOPPED);
+	if (old_status == PANFROST_QUEUE_STATUS_STOPPED)
+		goto out;
+
+	WARN_ON(old_status != PANFROST_QUEUE_STATUS_ACTIVE);
+	drm_sched_stop(&queue->sched, bad);
+	if (bad)
+		drm_sched_increase_karma(bad);
+
+	stopped = true;
+
+	/*
+	 * Set the timeout to max so the timer doesn't get started
+	 * when we return from the timeout handler (restored in
+	 * panfrost_scheduler_start()).
+	 */
+	queue->sched.timeout = MAX_SCHEDULE_TIMEOUT;
+
+out:
+	mutex_unlock(&queue->lock);
+
+	return stopped;
+}
+
+static void panfrost_scheduler_start(struct panfrost_queue_state *queue)
+{
+	enum panfrost_queue_status old_status;
+
+	mutex_lock(&queue->lock);
+	old_status = atomic_xchg(&queue->status,
+				 PANFROST_QUEUE_STATUS_STARTING);
+	WARN_ON(old_status != PANFROST_QUEUE_STATUS_STOPPED);
+
+	/* Restore the original timeout before starting the scheduler. */
+	queue->sched.timeout = msecs_to_jiffies(JOB_TIMEOUT_MS);
+	drm_sched_resubmit_jobs(&queue->sched);
+	drm_sched_start(&queue->sched, true);
+	old_status = atomic_xchg(&queue->status,
+				 PANFROST_QUEUE_STATUS_ACTIVE);
+	if (old_status == PANFROST_QUEUE_STATUS_FAULT_PENDING)
+		drm_sched_fault(&queue->sched);
+
+	mutex_unlock(&queue->lock);
+}
+
 static void panfrost_job_timedout(struct drm_sched_job *sched_job)
 {
 	struct panfrost_job *job = to_panfrost_job(sched_job);
 	struct panfrost_device *pfdev = job->pfdev;
 	int js = panfrost_job_get_slot(job);
-	unsigned long flags;
-	int i;
 
 	/*
 	 * If the GPU managed to complete this jobs fence, the timeout is
@@ -392,40 +453,13 @@ static void panfrost_job_timedout(struct drm_sched_job *sched_job)
 		job_read(pfdev, JS_TAIL_LO(js)),
 		sched_job);
 
-	if (!mutex_trylock(&pfdev->reset_lock))
+	/* Scheduler is already stopped, nothing to do. */
+	if (!panfrost_scheduler_stop(&pfdev->js->queue[js], sched_job))
 		return;
 
-	for (i = 0; i < NUM_JOB_SLOTS; i++) {
-		struct drm_gpu_scheduler *sched = &pfdev->js->queue[i].sched;
-
-		drm_sched_stop(sched, sched_job);
-		if (js != i)
-			/* Ensure any timeouts on other slots have finished */
-			cancel_delayed_work_sync(&sched->work_tdr);
-	}
-
-	drm_sched_increase_karma(sched_job);
-
-	spin_lock_irqsave(&pfdev->js->job_lock, flags);
-	for (i = 0; i < NUM_JOB_SLOTS; i++) {
-		if (pfdev->jobs[i]) {
-			pm_runtime_put_noidle(pfdev->dev);
-			panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
-			pfdev->jobs[i] = NULL;
-		}
-	}
-	spin_unlock_irqrestore(&pfdev->js->job_lock, flags);
-
-	panfrost_device_reset(pfdev);
-
-	for (i = 0; i < NUM_JOB_SLOTS; i++)
-		drm_sched_resubmit_jobs(&pfdev->js->queue[i].sched);
-
-	/* restart scheduler after GPU is usable again */
-	for (i = 0; i < NUM_JOB_SLOTS; i++)
-		drm_sched_start(&pfdev->js->queue[i].sched, true);
-
-	mutex_unlock(&pfdev->reset_lock);
+	/* Schedule a reset if there's no reset in progress. */
+	if (!atomic_xchg(&pfdev->reset.pending, 1))
+		schedule_work(&pfdev->reset.work);
 }
 
 static const struct drm_sched_backend_ops panfrost_sched_ops = {
@@ -457,6 +491,8 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
 		job_write(pfdev, JOB_INT_CLEAR, mask);
 
 		if (status & JOB_INT_MASK_ERR(j)) {
+			enum panfrost_queue_status old_status;
+
 			job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_NOP);
 
 			dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x",
@@ -465,7 +501,18 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
 				job_read(pfdev, JS_HEAD_LO(j)),
 				job_read(pfdev, JS_TAIL_LO(j)));
 
-			drm_sched_fault(&pfdev->js->queue[j].sched);
+			/*
+			 * When the queue is being restarted we don't report
+			 * faults directly to avoid races between the timeout
+			 * and reset handlers. panfrost_scheduler_start() will
+			 * call drm_sched_fault() after the queue has been
+			 * started if status == FAULT_PENDING.
+			 */
+			old_status = atomic_cmpxchg(&pfdev->js->queue[j].status,
+						    PANFROST_QUEUE_STATUS_STARTING,
+						    PANFROST_QUEUE_STATUS_FAULT_PENDING);
+			if (old_status == PANFROST_QUEUE_STATUS_ACTIVE)
+				drm_sched_fault(&pfdev->js->queue[j].sched);
 		}
 
 		if (status & JOB_INT_MASK_DONE(j)) {
@@ -492,11 +539,66 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static void panfrost_reset(struct work_struct *work)
+{
+	struct panfrost_device *pfdev = container_of(work,
+						     struct panfrost_device,
+						     reset.work);
+	unsigned long flags;
+	unsigned int i;
+	bool cookie;
+
+	cookie = dma_fence_begin_signalling();
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		/*
+		 * We want pending timeouts to be handled before we attempt
+		 * to stop the scheduler. If we don't do that and the timeout
+		 * handler is in flight, it might have removed the bad job
+		 * from the list, and we'll lose this job if the reset handler
+		 * enters the critical section in panfrost_scheduler_stop()
+		 * before the timeout handler.
+		 *
+		 * Timeout is set to MAX_SCHEDULE_TIMEOUT - 1 because we need
+		 * something big enough to make sure the timer will not expire
+		 * before we manage to stop the scheduler, but we can't use
+		 * MAX_SCHEDULE_TIMEOUT because drm_sched_get_cleanup_job()
+		 * considers that as 'timer is not running' and will dequeue
+		 * the job without making sure the timeout handler is not
+		 * running.
+		 */
+		pfdev->js->queue[i].sched.timeout = MAX_SCHEDULE_TIMEOUT - 1;
+		cancel_delayed_work_sync(&pfdev->js->queue[i].sched.work_tdr);
+		panfrost_scheduler_stop(&pfdev->js->queue[i], NULL);
+	}
+
+	/* All timers have been stopped, we can safely reset the pending state. */
+	atomic_set(&pfdev->reset.pending, 0);
+
+	spin_lock_irqsave(&pfdev->js->job_lock, flags);
+	for (i = 0; i < NUM_JOB_SLOTS; i++) {
+		if (pfdev->jobs[i]) {
+			pm_runtime_put_noidle(pfdev->dev);
+			panfrost_devfreq_record_idle(&pfdev->pfdevfreq);
+			pfdev->jobs[i] = NULL;
+		}
+	}
+	spin_unlock_irqrestore(&pfdev->js->job_lock, flags);
+
+	panfrost_device_reset(pfdev);
+
+	for (i = 0; i < NUM_JOB_SLOTS; i++)
+		panfrost_scheduler_start(&pfdev->js->queue[i]);
+
+	dma_fence_end_signalling(cookie);
+}
+
 int panfrost_job_init(struct panfrost_device *pfdev)
 {
 	struct panfrost_job_slot *js;
 	int ret, j, irq;
 
+	INIT_WORK(&pfdev->reset.work, panfrost_reset);
+
 	pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL);
 	if (!js)
 		return -ENOMEM;
@@ -515,11 +617,13 @@ int panfrost_job_init(struct panfrost_device *pfdev)
 	}
 
 	for (j = 0; j < NUM_JOB_SLOTS; j++) {
+		mutex_init(&js->queue[j].lock);
+
 		js->queue[j].fence_context = dma_fence_context_alloc(1);
 
 		ret = drm_sched_init(&js->queue[j].sched,
 				     &panfrost_sched_ops,
-				     1, 0, msecs_to_jiffies(500),
+				     1, 0, msecs_to_jiffies(JOB_TIMEOUT_MS),
 				     "pan_js");
 		if (ret) {
 			dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret);
@@ -545,8 +649,10 @@ void panfrost_job_fini(struct panfrost_device *pfdev)
 
 	job_write(pfdev, JOB_INT_MASK, 0);
 
-	for (j = 0; j < NUM_JOB_SLOTS; j++)
+	for (j = 0; j < NUM_JOB_SLOTS; j++) {
 		drm_sched_fini(&js->queue[j].sched);
+		mutex_destroy(&js->queue[j].lock);
+	}
 
 }