aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/gpu/drm/i915/i915_gem.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/i915_gem.c')
-rw-r--r--drivers/gpu/drm/i915/i915_gem.c781
1 files changed, 412 insertions, 369 deletions
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index dabc08987b5e..aad26851cee3 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -32,14 +32,13 @@
#include "i915_vgpu.h"
#include "i915_trace.h"
#include "intel_drv.h"
+#include "intel_mocs.h"
#include <linux/shmem_fs.h>
#include <linux/slab.h>
#include <linux/swap.h>
#include <linux/pci.h>
#include <linux/dma-buf.h>
-#define RQ_BUG_ON(expr)
-
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
static void
@@ -85,9 +84,7 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
{
int ret;
-#define EXIT_COND (!i915_reset_in_progress(error) || \
- i915_terminally_wedged(error))
- if (EXIT_COND)
+ if (!i915_reset_in_progress(error))
return 0;
/*
@@ -96,17 +93,16 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
* we should simply try to bail out and fail as gracefully as possible.
*/
ret = wait_event_interruptible_timeout(error->reset_queue,
- EXIT_COND,
+ !i915_reset_in_progress(error),
10*HZ);
if (ret == 0) {
DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
return -EIO;
} else if (ret < 0) {
return ret;
+ } else {
+ return 0;
}
-#undef EXIT_COND
-
- return 0;
}
int i915_mutex_lock_interruptible(struct drm_device *dev)
@@ -130,9 +126,9 @@ int
i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
struct drm_file *file)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct drm_i915_gem_get_aperture *args = data;
- struct i915_gtt *ggtt = &dev_priv->gtt;
struct i915_vma *vma;
size_t pinned;
@@ -146,7 +142,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
pinned += vma->node.size;
mutex_unlock(&dev->struct_mutex);
- args->aper_size = dev_priv->gtt.base.total;
+ args->aper_size = ggtt->base.total;
args->aper_available_size = args->aper_size - pinned;
return 0;
@@ -211,11 +207,10 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
BUG_ON(obj->madv == __I915_MADV_PURGED);
ret = i915_gem_object_set_to_cpu_domain(obj, true);
- if (ret) {
+ if (WARN_ON(ret)) {
/* In the event of a disaster, abandon all caches and
* hope for the best.
*/
- WARN_ON(ret != -EIO);
obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
}
@@ -324,7 +319,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
{
struct drm_device *dev = obj->base.dev;
void *vaddr = obj->phys_handle->vaddr + args->offset;
- char __user *user_data = to_user_ptr(args->data_ptr);
+ char __user *user_data = u64_to_user_ptr(args->data_ptr);
int ret = 0;
/* We manually control the domain here and pretend that it
@@ -605,7 +600,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
int needs_clflush = 0;
struct sg_page_iter sg_iter;
- user_data = to_user_ptr(args->data_ptr);
+ user_data = u64_to_user_ptr(args->data_ptr);
remain = args->size;
obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
@@ -692,7 +687,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
return 0;
if (!access_ok(VERIFY_WRITE,
- to_user_ptr(args->data_ptr),
+ u64_to_user_ptr(args->data_ptr),
args->size))
return -EFAULT;
@@ -700,7 +695,7 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -765,7 +760,8 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
struct drm_i915_gem_pwrite *args,
struct drm_file *file)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
ssize_t remain;
loff_t offset, page_base;
char __user *user_data;
@@ -783,7 +779,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
if (ret)
goto out_unpin;
- user_data = to_user_ptr(args->data_ptr);
+ user_data = u64_to_user_ptr(args->data_ptr);
remain = args->size;
offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
@@ -807,7 +803,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
* source page isn't available. Return the error and we'll
* retry in the slow path.
*/
- if (fast_user_write(dev_priv->gtt.mappable, page_base,
+ if (fast_user_write(ggtt->mappable, page_base,
page_offset, user_data, page_length)) {
ret = -EFAULT;
goto out_flush;
@@ -907,7 +903,7 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
int needs_clflush_before = 0;
struct sg_page_iter sg_iter;
- user_data = to_user_ptr(args->data_ptr);
+ user_data = u64_to_user_ptr(args->data_ptr);
remain = args->size;
obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
@@ -1036,12 +1032,12 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
return 0;
if (!access_ok(VERIFY_READ,
- to_user_ptr(args->data_ptr),
+ u64_to_user_ptr(args->data_ptr),
args->size))
return -EFAULT;
if (likely(!i915.prefault_disable)) {
- ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
+ ret = fault_in_multipages_readable(u64_to_user_ptr(args->data_ptr),
args->size);
if (ret)
return -EFAULT;
@@ -1053,7 +1049,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
if (ret)
goto put_rpm;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -1109,27 +1105,19 @@ put_rpm:
return ret;
}
-int
-i915_gem_check_wedge(struct i915_gpu_error *error,
- bool interruptible)
+static int
+i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
{
- if (i915_reset_in_progress(error)) {
+ if (__i915_terminally_wedged(reset_counter))
+ return -EIO;
+
+ if (__i915_reset_in_progress(reset_counter)) {
/* Non-interruptible callers can't handle -EAGAIN, hence return
* -EIO unconditionally for these. */
if (!interruptible)
return -EIO;
- /* Recovery complete, but the reset failed ... */
- if (i915_terminally_wedged(error))
- return -EIO;
-
- /*
- * Check if GPU Reset is in progress - we need intel_ring_begin
- * to work properly to reinit the hw state while the gpu is
- * still marked as reset-in-progress. Handle this with a flag.
- */
- if (!error->reload_in_reset)
- return -EAGAIN;
+ return -EAGAIN;
}
return 0;
@@ -1141,9 +1129,9 @@ static void fake_irq(unsigned long data)
}
static bool missed_irq(struct drm_i915_private *dev_priv,
- struct intel_engine_cs *ring)
+ struct intel_engine_cs *engine)
{
- return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
+ return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings);
}
static unsigned long local_clock_us(unsigned *cpu)
@@ -1193,7 +1181,7 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
* takes to sleep on a request, on the order of a microsecond.
*/
- if (req->ring->irq_refcount)
+ if (req->engine->irq_refcount)
return -EBUSY;
/* Only spin if we know the GPU is processing this request */
@@ -1223,7 +1211,6 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
/**
* __i915_wait_request - wait until execution of request has finished
* @req: duh!
- * @reset_counter: reset sequence associated with the given request
* @interruptible: do an interruptible wait (normally yes)
* @timeout: in - how long to wait (NULL forever); out - how much time remaining
*
@@ -1238,16 +1225,15 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
* errno with remaining time filled in timeout argument.
*/
int __i915_wait_request(struct drm_i915_gem_request *req,
- unsigned reset_counter,
bool interruptible,
s64 *timeout,
struct intel_rps_client *rps)
{
- struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
- struct drm_device *dev = ring->dev;
+ struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
+ struct drm_device *dev = engine->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
const bool irq_test_in_progress =
- ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
+ ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine);
int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
DEFINE_WAIT(wait);
unsigned long timeout_expire;
@@ -1288,7 +1274,7 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
if (ret == 0)
goto out;
- if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
+ if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) {
ret = -ENODEV;
goto out;
}
@@ -1296,16 +1282,17 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
for (;;) {
struct timer_list timer;
- prepare_to_wait(&ring->irq_queue, &wait, state);
+ prepare_to_wait(&engine->irq_queue, &wait, state);
/* We need to check whether any gpu reset happened in between
- * the caller grabbing the seqno and now ... */
- if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
- /* ... but upgrade the -EAGAIN to an -EIO if the gpu
- * is truely gone. */
- ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
- if (ret == 0)
- ret = -EAGAIN;
+ * the request being submitted and now. If a reset has occurred,
+ * the request is effectively complete (we either are in the
+ * process of or have discarded the rendering and completely
+ * reset the GPU. The results of the request are lost and we
+ * are free to continue on with the original operation.
+ */
+ if (req->reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
+ ret = 0;
break;
}
@@ -1325,11 +1312,11 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
}
timer.function = NULL;
- if (timeout || missed_irq(dev_priv, ring)) {
+ if (timeout || missed_irq(dev_priv, engine)) {
unsigned long expire;
setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
- expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
+ expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire;
mod_timer(&timer, expire);
}
@@ -1341,9 +1328,9 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
}
}
if (!irq_test_in_progress)
- ring->irq_put(ring);
+ engine->irq_put(engine);
- finish_wait(&ring->irq_queue, &wait);
+ finish_wait(&engine->irq_queue, &wait);
out:
trace_i915_gem_request_wait_end(req);
@@ -1370,7 +1357,6 @@ out:
int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
struct drm_file *file)
{
- struct drm_i915_private *dev_private;
struct drm_i915_file_private *file_priv;
WARN_ON(!req || !file || req->file_priv);
@@ -1381,7 +1367,6 @@ int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
if (req->file_priv)
return -EINVAL;
- dev_private = req->ring->dev->dev_private;
file_priv = file->driver_priv;
spin_lock(&file_priv->mm.lock);
@@ -1434,7 +1419,7 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
static void
__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
{
- struct intel_engine_cs *engine = req->ring;
+ struct intel_engine_cs *engine = req->engine;
struct drm_i915_gem_request *tmp;
lockdep_assert_held(&engine->dev->struct_mutex);
@@ -1459,30 +1444,22 @@ __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
int
i915_wait_request(struct drm_i915_gem_request *req)
{
- struct drm_device *dev;
- struct drm_i915_private *dev_priv;
+ struct drm_i915_private *dev_priv = req->i915;
bool interruptible;
int ret;
- BUG_ON(req == NULL);
-
- dev = req->ring->dev;
- dev_priv = dev->dev_private;
interruptible = dev_priv->mm.interruptible;
- BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+ BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
- ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
+ ret = __i915_wait_request(req, interruptible, NULL, NULL);
if (ret)
return ret;
- ret = __i915_wait_request(req,
- atomic_read(&dev_priv->gpu_error.reset_counter),
- interruptible, NULL, NULL);
- if (ret)
- return ret;
+ /* If the GPU hung, we want to keep the requests to find the guilty. */
+ if (req->reset_counter == i915_reset_counter(&dev_priv->gpu_error))
+ __i915_gem_request_retire__upto(req);
- __i915_gem_request_retire__upto(req);
return 0;
}
@@ -1505,14 +1482,14 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
if (ret)
return ret;
- i = obj->last_write_req->ring->id;
+ i = obj->last_write_req->engine->id;
if (obj->last_read_req[i] == obj->last_write_req)
i915_gem_object_retire__read(obj, i);
else
i915_gem_object_retire__write(obj);
}
} else {
- for (i = 0; i < I915_NUM_RINGS; i++) {
+ for (i = 0; i < I915_NUM_ENGINES; i++) {
if (obj->last_read_req[i] == NULL)
continue;
@@ -1522,7 +1499,7 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
i915_gem_object_retire__read(obj, i);
}
- RQ_BUG_ON(obj->active);
+ GEM_BUG_ON(obj->active);
}
return 0;
@@ -1532,14 +1509,15 @@ static void
i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
struct drm_i915_gem_request *req)
{
- int ring = req->ring->id;
+ int ring = req->engine->id;
if (obj->last_read_req[ring] == req)
i915_gem_object_retire__read(obj, ring);
else if (obj->last_write_req == req)
i915_gem_object_retire__write(obj);
- __i915_gem_request_retire__upto(req);
+ if (req->reset_counter == i915_reset_counter(&req->i915->gpu_error))
+ __i915_gem_request_retire__upto(req);
}
/* A nonblocking variant of the above wait. This is a highly dangerous routine
@@ -1552,8 +1530,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
{
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
- struct drm_i915_gem_request *requests[I915_NUM_RINGS];
- unsigned reset_counter;
+ struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
int ret, i, n = 0;
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
@@ -1562,12 +1539,6 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
if (!obj->active)
return 0;
- ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
- if (ret)
- return ret;
-
- reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
-
if (readonly) {
struct drm_i915_gem_request *req;
@@ -1577,7 +1548,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
requests[n++] = i915_gem_request_reference(req);
} else {
- for (i = 0; i < I915_NUM_RINGS; i++) {
+ for (i = 0; i < I915_NUM_ENGINES; i++) {
struct drm_i915_gem_request *req;
req = obj->last_read_req[i];
@@ -1589,9 +1560,9 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
}
mutex_unlock(&dev->struct_mutex);
+ ret = 0;
for (i = 0; ret == 0 && i < n; i++)
- ret = __i915_wait_request(requests[i], reset_counter, true,
- NULL, rps);
+ ret = __i915_wait_request(requests[i], true, NULL, rps);
mutex_lock(&dev->struct_mutex);
for (i = 0; i < n; i++) {
@@ -1640,7 +1611,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -1688,7 +1659,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -1732,10 +1703,10 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
if (args->flags & ~(I915_MMAP_WC))
return -EINVAL;
- if (args->flags & I915_MMAP_WC && !cpu_has_pat)
+ if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
return -ENODEV;
- obj = drm_gem_object_lookup(dev, file, args->handle);
+ obj = drm_gem_object_lookup(file, args->handle);
if (obj == NULL)
return -ENOENT;
@@ -1754,7 +1725,10 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
- down_write(&mm->mmap_sem);
+ if (down_write_killable(&mm->mmap_sem)) {
+ drm_gem_object_unreference_unlocked(obj);
+ return -EINTR;
+ }
vma = find_vma(mm, addr);
if (vma)
vma->vm_page_prot =
@@ -1792,7 +1766,8 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
struct drm_device *dev = obj->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct i915_ggtt_view view = i915_ggtt_view_normal;
pgoff_t page_offset;
unsigned long pfn;
@@ -1827,7 +1802,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
}
/* Use a partial view if the object is bigger than the aperture. */
- if (obj->base.size >= dev_priv->gtt.mappable_end &&
+ if (obj->base.size >= ggtt->mappable_end &&
obj->tiling_mode == I915_TILING_NONE) {
static const unsigned int chunk_size = 256; // 1 MiB
@@ -1855,7 +1830,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
goto unpin;
/* Finally, remap it using the new GTT offset */
- pfn = dev_priv->gtt.mappable_base +
+ pfn = ggtt->mappable_base +
i915_gem_obj_ggtt_offset_view(obj, &view);
pfn >>= PAGE_SHIFT;
@@ -1964,11 +1939,27 @@ out:
void
i915_gem_release_mmap(struct drm_i915_gem_object *obj)
{
+ /* Serialisation between user GTT access and our code depends upon
+ * revoking the CPU's PTE whilst the mutex is held. The next user
+ * pagefault then has to wait until we release the mutex.
+ */
+ lockdep_assert_held(&obj->base.dev->struct_mutex);
+
if (!obj->fault_mappable)
return;
drm_vma_node_unmap(&obj->base.vma_node,
obj->base.dev->anon_inode->i_mapping);
+
+ /* Ensure that the CPU's PTE are revoked and there are not outstanding
+ * memory transactions from userspace before we return. The TLB
+ * flushing implied above by changing the PTE above *should* be
+ * sufficient, an extra barrier here just provides us with a bit
+ * of paranoid documentation about our requirement to serialise
+ * memory writes before touching registers / GSM.
+ */
+ wmb();
+
obj->fault_mappable = false;
}
@@ -2033,9 +2024,6 @@ static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
int ret;
- if (drm_vma_node_has_offset(&obj->base.vma_node))
- return 0;
-
dev_priv->mm.shrinker_no_lock_stealing = true;
ret = drm_gem_create_mmap_offset(&obj->base);
@@ -2084,7 +2072,7 @@ i915_gem_mmap_gtt(struct drm_file *file,
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -2180,11 +2168,10 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
BUG_ON(obj->madv == __I915_MADV_PURGED);
ret = i915_gem_object_set_to_cpu_domain(obj, true);
- if (ret) {
+ if (WARN_ON(ret)) {
/* In the event of a disaster, abandon all caches and
* hope for the best.
*/
- WARN_ON(ret != -EIO);
i915_gem_clflush_object(obj, true);
obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
}
@@ -2232,6 +2219,14 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
* lists early. */
list_del(&obj->global_list);
+ if (obj->mapping) {
+ if (is_vmalloc_addr(obj->mapping))
+ vunmap(obj->mapping);
+ else
+ kunmap(kmap_to_page(obj->mapping));
+ obj->mapping = NULL;
+ }
+
ops->put_pages(obj);
obj->pages = NULL;
@@ -2400,21 +2395,64 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
return 0;
}
+void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
+{
+ int ret;
+
+ lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+ ret = i915_gem_object_get_pages(obj);
+ if (ret)
+ return ERR_PTR(ret);
+
+ i915_gem_object_pin_pages(obj);
+
+ if (obj->mapping == NULL) {
+ struct page **pages;
+
+ pages = NULL;
+ if (obj->base.size == PAGE_SIZE)
+ obj->mapping = kmap(sg_page(obj->pages->sgl));
+ else
+ pages = drm_malloc_gfp(obj->base.size >> PAGE_SHIFT,
+ sizeof(*pages),
+ GFP_TEMPORARY);
+ if (pages != NULL) {
+ struct sg_page_iter sg_iter;
+ int n;
+
+ n = 0;
+ for_each_sg_page(obj->pages->sgl, &sg_iter,
+ obj->pages->nents, 0)
+ pages[n++] = sg_page_iter_page(&sg_iter);
+
+ obj->mapping = vmap(pages, n, 0, PAGE_KERNEL);
+ drm_free_large(pages);
+ }
+ if (obj->mapping == NULL) {
+ i915_gem_object_unpin_pages(obj);
+ return ERR_PTR(-ENOMEM);
+ }
+ }
+
+ return obj->mapping;
+}
+
void i915_vma_move_to_active(struct i915_vma *vma,
struct drm_i915_gem_request *req)
{
struct drm_i915_gem_object *obj = vma->obj;
- struct intel_engine_cs *ring;
+ struct intel_engine_cs *engine;
- ring = i915_gem_request_get_ring(req);
+ engine = i915_gem_request_get_engine(req);
/* Add a reference if we're newly entering the active list. */
if (obj->active == 0)
drm_gem_object_reference(&obj->base);
- obj->active |= intel_ring_flag(ring);
+ obj->active |= intel_engine_flag(engine);
- list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
- i915_gem_request_assign(&obj->last_read_req[ring->id], req);
+ list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
+ i915_gem_request_assign(&obj->last_read_req[engine->id], req);
list_move_tail(&vma->vm_link, &vma->vm->active_list);
}
@@ -2422,8 +2460,8 @@ void i915_vma_move_to_active(struct i915_vma *vma,
static void
i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
{
- RQ_BUG_ON(obj->last_write_req == NULL);
- RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
+ GEM_BUG_ON(obj->last_write_req == NULL);
+ GEM_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
i915_gem_request_assign(&obj->last_write_req, NULL);
intel_fb_obj_flush(obj, true, ORIGIN_CS);
@@ -2434,13 +2472,13 @@ i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
{
struct i915_vma *vma;
- RQ_BUG_ON(obj->last_read_req[ring] == NULL);
- RQ_BUG_ON(!(obj->active & (1 << ring)));
+ GEM_BUG_ON(obj->last_read_req[ring] == NULL);
+ GEM_BUG_ON(!(obj->active & (1 << ring)));
- list_del_init(&obj->ring_list[ring]);
+ list_del_init(&obj->engine_list[ring]);
i915_gem_request_assign(&obj->last_read_req[ring], NULL);
- if (obj->last_write_req && obj->last_write_req->ring->id == ring)
+ if (obj->last_write_req && obj->last_write_req->engine->id == ring)
i915_gem_object_retire__write(obj);
obj->active &= ~(1 << ring);
@@ -2467,24 +2505,20 @@ static int
i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
- int ret, i, j;
+ struct intel_engine_cs *engine;
+ int ret;
/* Carefully retire all requests without writing to the rings */
- for_each_ring(ring, dev_priv, i) {
- ret = intel_ring_idle(ring);
+ for_each_engine(engine, dev_priv) {
+ ret = intel_engine_idle(engine);
if (ret)
return ret;
}
i915_gem_retire_requests(dev);
/* Finally reset hw state */
- for_each_ring(ring, dev_priv, i) {
- intel_ring_init_seqno(ring, seqno);
-
- for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
- ring->semaphore.sync_seqno[j] = 0;
- }
+ for_each_engine(engine, dev_priv)
+ intel_ring_init_seqno(engine, seqno);
return 0;
}
@@ -2542,7 +2576,7 @@ void __i915_add_request(struct drm_i915_gem_request *request,
struct drm_i915_gem_object *obj,
bool flush_caches)
{
- struct intel_engine_cs *ring;
+ struct intel_engine_cs *engine;
struct drm_i915_private *dev_priv;
struct intel_ringbuffer *ringbuf;
u32 request_start;
@@ -2551,8 +2585,8 @@ void __i915_add_request(struct drm_i915_gem_request *request,
if (WARN_ON(request == NULL))
return;
- ring = request->ring;
- dev_priv = ring->dev->dev_private;
+ engine = request->engine;
+ dev_priv = request->i915;
ringbuf = request->ringbuf;
/*
@@ -2579,6 +2613,28 @@ void __i915_add_request(struct drm_i915_gem_request *request,
WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
}
+ trace_i915_gem_request_add(request);
+
+ request->head = request_start;
+
+ /* Whilst this request exists, batch_obj will be on the
+ * active_list, and so will hold the active reference. Only when this
+ * request is retired will the the batch_obj be moved onto the
+ * inactive_list and lose its active reference. Hence we do not need
+ * to explicitly hold another reference here.
+ */
+ request->batch_obj = obj;
+
+ /* Seal the request and mark it as pending execution. Note that
+ * we may inspect this state, without holding any locks, during
+ * hangcheck. Hence we apply the barrier to ensure that we do not
+ * see a more recent value in the hws than we are tracking.
+ */
+ request->emitted_jiffies = jiffies;
+ request->previous_seqno = engine->last_submitted_seqno;
+ smp_store_mb(engine->last_submitted_seqno, request->seqno);
+ list_add_tail(&request->list, &engine->request_list);
+
/* Record the position of the start of the request so that
* should we detect the updated seqno part-way through the
* GPU processing the request, we never over-estimate the
@@ -2587,33 +2643,16 @@ void __i915_add_request(struct drm_i915_gem_request *request,
request->postfix = intel_ring_get_tail(ringbuf);
if (i915.enable_execlists)
- ret = ring->emit_request(request);
+ ret = engine->emit_request(request);
else {
- ret = ring->add_request(request);
+ ret = engine->add_request(request);
request->tail = intel_ring_get_tail(ringbuf);
}
/* Not allowed to fail! */
WARN(ret, "emit|add_request failed: %d!\n", ret);
- request->head = request_start;
-
- /* Whilst this request exists, batch_obj will be on the
- * active_list, and so will hold the active reference. Only when this
- * request is retired will the the batch_obj be moved onto the
- * inactive_list and lose its active reference. Hence we do not need
- * to explicitly hold another reference here.
- */
- request->batch_obj = obj;
-
- request->emitted_jiffies = jiffies;
- request->previous_seqno = ring->last_submitted_seqno;
- ring->last_submitted_seqno = request->seqno;
- list_add_tail(&request->list, &ring->request_list);
-
- trace_i915_gem_request_add(request);
-
- i915_queue_hangcheck(ring->dev);
+ i915_queue_hangcheck(engine->dev);
queue_delayed_work(dev_priv->wq,
&dev_priv->mm.retire_work,
@@ -2680,7 +2719,7 @@ void i915_gem_request_free(struct kref *req_ref)
if (ctx) {
if (i915.enable_execlists && ctx != req->i915->kernel_context)
- intel_lr_context_unpin(ctx, req->ring);
+ intel_lr_context_unpin(ctx, req->engine);
i915_gem_context_unreference(ctx);
}
@@ -2689,11 +2728,12 @@ void i915_gem_request_free(struct kref *req_ref)
}
static inline int
-__i915_gem_request_alloc(struct intel_engine_cs *ring,
+__i915_gem_request_alloc(struct intel_engine_cs *engine,
struct intel_context *ctx,
struct drm_i915_gem_request **req_out)
{
- struct drm_i915_private *dev_priv = to_i915(ring->dev);
+ struct drm_i915_private *dev_priv = to_i915(engine->dev);
+ unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
struct drm_i915_gem_request *req;
int ret;
@@ -2702,17 +2742,26 @@ __i915_gem_request_alloc(struct intel_engine_cs *ring,
*req_out = NULL;
+ /* ABI: Before userspace accesses the GPU (e.g. execbuffer), report
+ * EIO if the GPU is already wedged, or EAGAIN to drop the struct_mutex
+ * and restart.
+ */
+ ret = i915_gem_check_wedge(reset_counter, dev_priv->mm.interruptible);
+ if (ret)
+ return ret;
+
req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
if (req == NULL)
return -ENOMEM;
- ret = i915_gem_get_seqno(ring->dev, &req->seqno);
+ ret = i915_gem_get_seqno(engine->dev, &req->seqno);
if (ret)
goto err;
kref_init(&req->ref);
req->i915 = dev_priv;
- req->ring = ring;
+ req->engine = engine;
+ req->reset_counter = reset_counter;
req->ctx = ctx;
i915_gem_context_reference(req->ctx);
@@ -2742,7 +2791,8 @@ __i915_gem_request_alloc(struct intel_engine_cs *ring,
* fully prepared. Thus it can be cleaned up using the proper
* free code.
*/
- i915_gem_request_cancel(req);
+ intel_ring_reserved_space_cancel(req->ringbuf);
+ i915_gem_request_unreference(req);
return ret;
}
@@ -2779,19 +2829,12 @@ i915_gem_request_alloc(struct intel_engine_cs *engine,
return err ? ERR_PTR(err) : req;
}
-void i915_gem_request_cancel(struct drm_i915_gem_request *req)
-{
- intel_ring_reserved_space_cancel(req->ringbuf);
-
- i915_gem_request_unreference(req);
-}
-
struct drm_i915_gem_request *
-i915_gem_find_active_request(struct intel_engine_cs *ring)
+i915_gem_find_active_request(struct intel_engine_cs *engine)
{
struct drm_i915_gem_request *request;
- list_for_each_entry(request, &ring->request_list, list) {
+ list_for_each_entry(request, &engine->request_list, list) {
if (i915_gem_request_completed(request, false))
continue;
@@ -2801,38 +2844,38 @@ i915_gem_find_active_request(struct intel_engine_cs *ring)
return NULL;
}
-static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
- struct intel_engine_cs *ring)
+static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv,
+ struct intel_engine_cs *engine)
{
struct drm_i915_gem_request *request;
bool ring_hung;
- request = i915_gem_find_active_request(ring);
+ request = i915_gem_find_active_request(engine);
if (request == NULL)
return;
- ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
+ ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
i915_set_reset_status(dev_priv, request->ctx, ring_hung);
- list_for_each_entry_continue(request, &ring->request_list, list)
+ list_for_each_entry_continue(request, &engine->request_list, list)
i915_set_reset_status(dev_priv, request->ctx, false);
}
-static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
- struct intel_engine_cs *ring)
+static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
+ struct intel_engine_cs *engine)
{
struct intel_ringbuffer *buffer;
- while (!list_empty(&ring->active_list)) {
+ while (!list_empty(&engine->active_list)) {
struct drm_i915_gem_object *obj;
- obj = list_first_entry(&ring->active_list,
+ obj = list_first_entry(&engine->active_list,
struct drm_i915_gem_object,
- ring_list[ring->id]);
+ engine_list[engine->id]);
- i915_gem_object_retire__read(obj, ring->id);
+ i915_gem_object_retire__read(obj, engine->id);
}
/*
@@ -2842,14 +2885,16 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
*/
if (i915.enable_execlists) {
- spin_lock_irq(&ring->execlist_lock);
+ /* Ensure irq handler finishes or is cancelled. */
+ tasklet_kill(&engine->irq_tasklet);
+ spin_lock_bh(&engine->execlist_lock);
/* list_splice_tail_init checks for empty lists */
- list_splice_tail_init(&ring->execlist_queue,
- &ring->execlist_retired_req_list);
+ list_splice_tail_init(&engine->execlist_queue,
+ &engine->execlist_retired_req_list);
+ spin_unlock_bh(&engine->execlist_lock);
- spin_unlock_irq(&ring->execlist_lock);
- intel_execlists_retire_requests(ring);
+ intel_execlists_retire_requests(engine);
}
/*
@@ -2859,10 +2904,10 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
* implicit references on things like e.g. ppgtt address spaces through
* the request.
*/
- while (!list_empty(&ring->request_list)) {
+ while (!list_empty(&engine->request_list)) {
struct drm_i915_gem_request *request;
- request = list_first_entry(&ring->request_list,
+ request = list_first_entry(&engine->request_list,
struct drm_i915_gem_request,
list);
@@ -2876,28 +2921,29 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
* upon reset is less than when we start. Do one more pass over
* all the ringbuffers to reset last_retired_head.
*/
- list_for_each_entry(buffer, &ring->buffers, link) {
+ list_for_each_entry(buffer, &engine->buffers, link) {
buffer->last_retired_head = buffer->tail;
intel_ring_update_space(buffer);
}
+
+ intel_ring_init_seqno(engine, engine->last_submitted_seqno);
}
void i915_gem_reset(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
- int i;
+ struct intel_engine_cs *engine;
/*
* Before we free the objects from the requests, we need to inspect
* them for finding the guilty party. As the requests only borrow
* their reference to the objects, the inspection must be done first.
*/
- for_each_ring(ring, dev_priv, i)
- i915_gem_reset_ring_status(dev_priv, ring);
+ for_each_engine(engine, dev_priv)
+ i915_gem_reset_engine_status(dev_priv, engine);
- for_each_ring(ring, dev_priv, i)
- i915_gem_reset_ring_cleanup(dev_priv, ring);
+ for_each_engine(engine, dev_priv)
+ i915_gem_reset_engine_cleanup(dev_priv, engine);
i915_gem_context_reset(dev);
@@ -2910,19 +2956,19 @@ void i915_gem_reset(struct drm_device *dev)
* This function clears the request list as sequence numbers are passed.
*/
void
-i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
+i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
{
- WARN_ON(i915_verify_lists(ring->dev));
+ WARN_ON(i915_verify_lists(engine->dev));
/* Retire requests first as we use it above for the early return.
* If we retire requests last, we may use a later seqno and so clear
* the requests lists without clearing the active list, leading to
* confusion.
*/
- while (!list_empty(&ring->request_list)) {
+ while (!list_empty(&engine->request_list)) {
struct drm_i915_gem_request *request;
- request = list_first_entry(&ring->request_list,
+ request = list_first_entry(&engine->request_list,
struct drm_i915_gem_request,
list);
@@ -2936,45 +2982,44 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
* by the ringbuffer to the flushing/inactive lists as appropriate,
* before we free the context associated with the requests.
*/
- while (!list_empty(&ring->active_list)) {
+ while (!list_empty(&engine->active_list)) {
struct drm_i915_gem_object *obj;
- obj = list_first_entry(&ring->active_list,
- struct drm_i915_gem_object,
- ring_list[ring->id]);
+ obj = list_first_entry(&engine->active_list,
+ struct drm_i915_gem_object,
+ engine_list[engine->id]);
- if (!list_empty(&obj->last_read_req[ring->id]->list))
+ if (!list_empty(&obj->last_read_req[engine->id]->list))
break;
- i915_gem_object_retire__read(obj, ring->id);
+ i915_gem_object_retire__read(obj, engine->id);
}
- if (unlikely(ring->trace_irq_req &&
- i915_gem_request_completed(ring->trace_irq_req, true))) {
- ring->irq_put(ring);
- i915_gem_request_assign(&ring->trace_irq_req, NULL);
+ if (unlikely(engine->trace_irq_req &&
+ i915_gem_request_completed(engine->trace_irq_req, true))) {
+ engine->irq_put(engine);
+ i915_gem_request_assign(&engine->trace_irq_req, NULL);
}
- WARN_ON(i915_verify_lists(ring->dev));
+ WARN_ON(i915_verify_lists(engine->dev));
}
bool
i915_gem_retire_requests(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
+ struct intel_engine_cs *engine;
bool idle = true;
- int i;
- for_each_ring(ring, dev_priv, i) {
- i915_gem_retire_requests_ring(ring);
- idle &= list_empty(&ring->request_list);
+ for_each_engine(engine, dev_priv) {
+ i915_gem_retire_requests_ring(engine);
+ idle &= list_empty(&engine->request_list);
if (i915.enable_execlists) {
- spin_lock_irq(&ring->execlist_lock);
- idle &= list_empty(&ring->execlist_queue);
- spin_unlock_irq(&ring->execlist_lock);
+ spin_lock_bh(&engine->execlist_lock);
+ idle &= list_empty(&engine->execlist_queue);
+ spin_unlock_bh(&engine->execlist_lock);
- intel_execlists_retire_requests(ring);
+ intel_execlists_retire_requests(engine);
}
}
@@ -3011,25 +3056,21 @@ i915_gem_idle_work_handler(struct work_struct *work)
struct drm_i915_private *dev_priv =
container_of(work, typeof(*dev_priv), mm.idle_work.work);
struct drm_device *dev = dev_priv->dev;
- struct intel_engine_cs *ring;
- int i;
+ struct intel_engine_cs *engine;
- for_each_ring(ring, dev_priv, i)
- if (!list_empty(&ring->request_list))
+ for_each_engine(engine, dev_priv)
+ if (!list_empty(&engine->request_list))
return;
/* we probably should sync with hangcheck here, using cancel_work_sync.
- * Also locking seems to be fubar here, ring->request_list is protected
+ * Also locking seems to be fubar here, engine->request_list is protected
* by dev->struct_mutex. */
intel_mark_idle(dev);
if (mutex_trylock(&dev->struct_mutex)) {
- struct intel_engine_cs *ring;
- int i;
-
- for_each_ring(ring, dev_priv, i)
- i915_gem_batch_pool_fini(&ring->batch_pool);
+ for_each_engine(engine, dev_priv)
+ i915_gem_batch_pool_fini(&engine->batch_pool);
mutex_unlock(&dev->struct_mutex);
}
@@ -3048,7 +3089,7 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
if (!obj->active)
return 0;
- for (i = 0; i < I915_NUM_RINGS; i++) {
+ for (i = 0; i < I915_NUM_ENGINES; i++) {
struct drm_i915_gem_request *req;
req = obj->last_read_req[i];
@@ -3093,11 +3134,9 @@ retire:
int
i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_wait *args = data;
struct drm_i915_gem_object *obj;
- struct drm_i915_gem_request *req[I915_NUM_RINGS];
- unsigned reset_counter;
+ struct drm_i915_gem_request *req[I915_NUM_ENGINES];
int i, n = 0;
int ret;
@@ -3108,7 +3147,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->bo_handle));
if (&obj->base == NULL) {
mutex_unlock(&dev->struct_mutex);
return -ENOENT;
@@ -3131,9 +3170,8 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
}
drm_gem_object_unreference(&obj->base);
- reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- for (i = 0; i < I915_NUM_RINGS; i++) {
+ for (i = 0; i < I915_NUM_ENGINES; i++) {
if (obj->last_read_req[i] == NULL)
continue;
@@ -3144,7 +3182,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
for (i = 0; i < n; i++) {
if (ret == 0)
- ret = __i915_wait_request(req[i], reset_counter, true,
+ ret = __i915_wait_request(req[i], true,
args->timeout_ns > 0 ? &args->timeout_ns : NULL,
to_rps_client(file));
i915_gem_request_unreference__unlocked(req[i]);
@@ -3166,7 +3204,7 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj,
struct intel_engine_cs *from;
int ret;
- from = i915_gem_request_get_ring(from_req);
+ from = i915_gem_request_get_engine(from_req);
if (to == from)
return 0;
@@ -3176,7 +3214,6 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj,
if (!i915_semaphore_is_enabled(obj->base.dev)) {
struct drm_i915_private *i915 = to_i915(obj->base.dev);
ret = __i915_wait_request(from_req,
- atomic_read(&i915->gpu_error.reset_counter),
i915->mm.interruptible,
NULL,
&i915->rps.semaphores);
@@ -3260,7 +3297,7 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj,
struct drm_i915_gem_request **to_req)
{
const bool readonly = obj->base.pending_write_domain == 0;
- struct drm_i915_gem_request *req[I915_NUM_RINGS];
+ struct drm_i915_gem_request *req[I915_NUM_ENGINES];
int ret, i, n;
if (!obj->active)
@@ -3274,7 +3311,7 @@ i915_gem_object_sync(struct drm_i915_gem_object *obj,
if (obj->last_write_req)
req[n++] = obj->last_write_req;
} else {
- for (i = 0; i < I915_NUM_RINGS; i++)
+ for (i = 0; i < I915_NUM_ENGINES; i++)
if (obj->last_read_req[i])
req[n++] = obj->last_read_req[i];
}
@@ -3297,9 +3334,6 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
return;
- /* Wait for any direct GTT access to complete */
- mb();
-
old_read_domains = obj->base.read_domains;
old_write_domain = obj->base.write_domain;
@@ -3391,28 +3425,25 @@ int __i915_vma_unbind_no_wait(struct i915_vma *vma)
int i915_gpu_idle(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
- int ret, i;
+ struct intel_engine_cs *engine;
+ int ret;
/* Flush everything onto the inactive list. */
- for_each_ring(ring, dev_priv, i) {
+ for_each_engine(engine, dev_priv) {
if (!i915.enable_execlists) {
struct drm_i915_gem_request *req;
- req = i915_gem_request_alloc(ring, NULL);
+ req = i915_gem_request_alloc(engine, NULL);
if (IS_ERR(req))
return PTR_ERR(req);
ret = i915_switch_context(req);
- if (ret) {
- i915_gem_request_cancel(req);
- return ret;
- }
-
i915_add_request_no_flush(req);
+ if (ret)
+ return ret;
}
- ret = intel_ring_idle(ring);
+ ret = intel_engine_idle(engine);
if (ret)
return ret;
}
@@ -3466,7 +3497,8 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
uint64_t flags)
{
struct drm_device *dev = obj->base.dev;
- struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
u32 fence_alignment, unfenced_alignment;
u32 search_flag, alloc_flag;
u64 start, end;
@@ -3513,7 +3545,7 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
end = vm->total;
if (flags & PIN_MAPPABLE)
- end = min_t(u64, end, dev_priv->gtt.mappable_end);
+ end = min_t(u64, end, ggtt->mappable_end);
if (flags & PIN_ZONE_4G)
end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
@@ -3720,6 +3752,9 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
int
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
{
+ struct drm_device *dev = obj->base.dev;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
uint32_t old_write_domain, old_read_domains;
struct i915_vma *vma;
int ret;
@@ -3774,7 +3809,7 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
vma = i915_gem_obj_to_ggtt(obj);
if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
list_move_tail(&vma->vm_link,
- &to_i915(obj->base.dev)->gtt.base.inactive_list);
+ &ggtt->base.inactive_list);
return 0;
}
@@ -3906,7 +3941,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
struct drm_i915_gem_caching *args = data;
struct drm_i915_gem_object *obj;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL)
return -ENOENT;
@@ -3949,7 +3984,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
* cacheline, whereas normally such cachelines would get
* invalidated.
*/
- if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+ if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
return -ENODEV;
level = I915_CACHE_LLC;
@@ -3967,7 +4002,7 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
if (ret)
goto rpm_put;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -4128,16 +4163,15 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
struct drm_i915_file_private *file_priv = file->driver_priv;
unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
struct drm_i915_gem_request *request, *target = NULL;
- unsigned reset_counter;
int ret;
ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
if (ret)
return ret;
- ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
- if (ret)
- return ret;
+ /* ABI: return -EIO if already wedged */
+ if (i915_terminally_wedged(&dev_priv->gpu_error))
+ return -EIO;
spin_lock(&file_priv->mm.lock);
list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
@@ -4153,7 +4187,6 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
target = request;
}
- reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
if (target)
i915_gem_request_reference(target);
spin_unlock(&file_priv->mm.lock);
@@ -4161,7 +4194,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
if (target == NULL)
return 0;
- ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
+ ret = __i915_wait_request(target, true, NULL, NULL);
if (ret == 0)
queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
@@ -4211,7 +4244,7 @@ void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
(vma->node.start & (fence_alignment - 1)) == 0);
mappable = (vma->node.start + fence_size <=
- to_i915(obj->base.dev)->gtt.mappable_end);
+ to_i915(obj->base.dev)->ggtt.mappable_end);
obj->map_and_fenceable = mappable && fenceable;
}
@@ -4243,9 +4276,6 @@ i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
i915_gem_obj_to_vma(obj, vm);
- if (IS_ERR(vma))
- return PTR_ERR(vma);
-
if (vma) {
if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
return -EBUSY;
@@ -4308,10 +4338,13 @@ i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
uint32_t alignment,
uint64_t flags)
{
- if (WARN_ONCE(!view, "no view specified"))
- return -EINVAL;
+ struct drm_device *dev = obj->base.dev;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
- return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
+ BUG_ON(!view);
+
+ return i915_gem_object_do_pin(obj, &ggtt->base, view,
alignment, flags | PIN_GLOBAL);
}
@@ -4321,7 +4354,6 @@ i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
{
struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
- BUG_ON(!vma);
WARN_ON(vma->pin_count == 0);
WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
@@ -4340,7 +4372,7 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -4359,15 +4391,15 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
if (obj->active) {
int i;
- for (i = 0; i < I915_NUM_RINGS; i++) {
+ for (i = 0; i < I915_NUM_ENGINES; i++) {
struct drm_i915_gem_request *req;
req = obj->last_read_req[i];
if (req)
- args->busy |= 1 << (16 + req->ring->exec_id);
+ args->busy |= 1 << (16 + req->engine->exec_id);
}
if (obj->last_write_req)
- args->busy |= obj->last_write_req->ring->exec_id;
+ args->busy |= obj->last_write_req->engine->exec_id;
}
unref:
@@ -4405,7 +4437,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
if (ret)
return ret;
- obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
+ obj = to_intel_bo(drm_gem_object_lookup(file_priv, args->handle));
if (&obj->base == NULL) {
ret = -ENOENT;
goto unlock;
@@ -4447,8 +4479,8 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
int i;
INIT_LIST_HEAD(&obj->global_list);
- for (i = 0; i < I915_NUM_RINGS; i++)
- INIT_LIST_HEAD(&obj->ring_list[i]);
+ for (i = 0; i < I915_NUM_ENGINES; i++)
+ INIT_LIST_HEAD(&obj->engine_list[i]);
INIT_LIST_HEAD(&obj->obj_exec_link);
INIT_LIST_HEAD(&obj->vma_list);
INIT_LIST_HEAD(&obj->batch_pool_link);
@@ -4623,14 +4655,15 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
const struct i915_ggtt_view *view)
{
- struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
+ struct drm_device *dev = obj->base.dev;
+ struct drm_i915_private *dev_priv = to_i915(dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct i915_vma *vma;
- if (WARN_ONCE(!view, "no view specified"))
- return ERR_PTR(-EINVAL);
+ BUG_ON(!view);
list_for_each_entry(vma, &obj->vma_list, obj_link)
- if (vma->vm == ggtt &&
+ if (vma->vm == &ggtt->base &&
i915_ggtt_view_equal(&vma->ggtt_view, view))
return vma;
return NULL;
@@ -4653,14 +4686,13 @@ void i915_gem_vma_destroy(struct i915_vma *vma)
}
static void
-i915_gem_stop_ringbuffers(struct drm_device *dev)
+i915_gem_stop_engines(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
- int i;
+ struct intel_engine_cs *engine;
- for_each_ring(ring, dev_priv, i)
- dev_priv->gt.stop_ring(ring);
+ for_each_engine(engine, dev_priv)
+ dev_priv->gt.stop_engine(engine);
}
int
@@ -4676,7 +4708,7 @@ i915_gem_suspend(struct drm_device *dev)
i915_gem_retire_requests(dev);
- i915_gem_stop_ringbuffers(dev);
+ i915_gem_stop_engines(dev);
mutex_unlock(&dev->struct_mutex);
cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
@@ -4697,8 +4729,8 @@ err:
int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
{
- struct intel_engine_cs *ring = req->ring;
- struct drm_device *dev = ring->dev;
+ struct intel_engine_cs *engine = req->engine;
+ struct drm_device *dev = engine->dev;
struct drm_i915_private *dev_priv = dev->dev_private;
u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
int i, ret;
@@ -4716,12 +4748,12 @@ int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
* at initialization time.
*/
for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
- intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
- intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
- intel_ring_emit(ring, remap_info[i]);
+ intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
+ intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
+ intel_ring_emit(engine, remap_info[i]);
}
- intel_ring_advance(ring);
+ intel_ring_advance(engine);
return ret;
}
@@ -4778,7 +4810,7 @@ static void init_unused_rings(struct drm_device *dev)
}
}
-int i915_gem_init_rings(struct drm_device *dev)
+int i915_gem_init_engines(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
int ret;
@@ -4814,13 +4846,13 @@ int i915_gem_init_rings(struct drm_device *dev)
return 0;
cleanup_vebox_ring:
- intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
+ intel_cleanup_engine(&dev_priv->engine[VECS]);
cleanup_blt_ring:
- intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
+ intel_cleanup_engine(&dev_priv->engine[BCS]);
cleanup_bsd_ring:
- intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
+ intel_cleanup_engine(&dev_priv->engine[VCS]);
cleanup_render_ring:
- intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
+ intel_cleanup_engine(&dev_priv->engine[RCS]);
return ret;
}
@@ -4829,16 +4861,13 @@ int
i915_gem_init_hw(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
- int ret, i, j;
-
- if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
- return -EIO;
+ struct intel_engine_cs *engine;
+ int ret, j;
/* Double layer security blanket, see i915_gem_init() */
intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
- if (dev_priv->ellc_size)
+ if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
if (IS_HASWELL(dev))
@@ -4876,12 +4905,14 @@ i915_gem_init_hw(struct drm_device *dev)
}
/* Need to do basic initialisation of all rings first: */
- for_each_ring(ring, dev_priv, i) {
- ret = ring->init_hw(ring);
+ for_each_engine(engine, dev_priv) {
+ ret = engine->init_hw(engine);
if (ret)
goto out;
}
+ intel_mocs_init_l3cc_table(dev);
+
/* We can't enable contexts until all firmware is loaded */
if (HAS_GUC_UCODE(dev)) {
ret = intel_guc_ucode_load(dev);
@@ -4901,38 +4932,39 @@ i915_gem_init_hw(struct drm_device *dev)
goto out;
/* Now it is safe to go back round and do everything else: */
- for_each_ring(ring, dev_priv, i) {
+ for_each_engine(engine, dev_priv) {
struct drm_i915_gem_request *req;
- req = i915_gem_request_alloc(ring, NULL);
+ req = i915_gem_request_alloc(engine, NULL);
if (IS_ERR(req)) {
ret = PTR_ERR(req);
- i915_gem_cleanup_ringbuffer(dev);
- goto out;
+ break;
}
- if (ring->id == RCS) {
- for (j = 0; j < NUM_L3_SLICES(dev); j++)
- i915_gem_l3_remap(req, j);
+ if (engine->id == RCS) {
+ for (j = 0; j < NUM_L3_SLICES(dev); j++) {
+ ret = i915_gem_l3_remap(req, j);
+ if (ret)
+ goto err_request;
+ }
}
ret = i915_ppgtt_init_ring(req);
- if (ret && ret != -EIO) {
- DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
- i915_gem_request_cancel(req);
- i915_gem_cleanup_ringbuffer(dev);
- goto out;
- }
+ if (ret)
+ goto err_request;
ret = i915_gem_context_enable(req);
- if (ret && ret != -EIO) {
- DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
- i915_gem_request_cancel(req);
- i915_gem_cleanup_ringbuffer(dev);
- goto out;
- }
+ if (ret)
+ goto err_request;
+err_request:
i915_add_request_no_flush(req);
+ if (ret) {
+ DRM_ERROR("Failed to enable %s, error=%d\n",
+ engine->name, ret);
+ i915_gem_cleanup_engines(dev);
+ break;
+ }
}
out:
@@ -4952,14 +4984,14 @@ int i915_gem_init(struct drm_device *dev)
if (!i915.enable_execlists) {
dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
- dev_priv->gt.init_rings = i915_gem_init_rings;
- dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
- dev_priv->gt.stop_ring = intel_stop_ring_buffer;
+ dev_priv->gt.init_engines = i915_gem_init_engines;
+ dev_priv->gt.cleanup_engine = intel_cleanup_engine;
+ dev_priv->gt.stop_engine = intel_stop_engine;
} else {
dev_priv->gt.execbuf_submit = intel_execlists_submission;
- dev_priv->gt.init_rings = intel_logical_rings_init;
- dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
- dev_priv->gt.stop_ring = intel_logical_ring_stop;
+ dev_priv->gt.init_engines = intel_logical_rings_init;
+ dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
+ dev_priv->gt.stop_engine = intel_logical_ring_stop;
}
/* This is just a security blanket to placate dragons.
@@ -4974,13 +5006,13 @@ int i915_gem_init(struct drm_device *dev)
if (ret)
goto out_unlock;
- i915_gem_init_global_gtt(dev);
+ i915_gem_init_ggtt(dev);
ret = i915_gem_context_init(dev);
if (ret)
goto out_unlock;
- ret = dev_priv->gt.init_rings(dev);
+ ret = dev_priv->gt.init_engines(dev);
if (ret)
goto out_unlock;
@@ -5003,29 +5035,52 @@ out_unlock:
}
void
-i915_gem_cleanup_ringbuffer(struct drm_device *dev)
+i915_gem_cleanup_engines(struct drm_device *dev)
{
struct drm_i915_private *dev_priv = dev->dev_private;
- struct intel_engine_cs *ring;
- int i;
+ struct intel_engine_cs *engine;
- for_each_ring(ring, dev_priv, i)
- dev_priv->gt.cleanup_ring(ring);
+ for_each_engine(engine, dev_priv)
+ dev_priv->gt.cleanup_engine(engine);
- if (i915.enable_execlists)
- /*
- * Neither the BIOS, ourselves or any other kernel
- * expects the system to be in execlists mode on startup,
- * so we need to reset the GPU back to legacy mode.
- */
- intel_gpu_reset(dev);
+ if (i915.enable_execlists)
+ /*
+ * Neither the BIOS, ourselves or any other kernel
+ * expects the system to be in execlists mode on startup,
+ * so we need to reset the GPU back to legacy mode.
+ */
+ intel_gpu_reset(dev, ALL_ENGINES);
}
static void
-init_ring_lists(struct intel_engine_cs *ring)
+init_engine_lists(struct intel_engine_cs *engine)
{
- INIT_LIST_HEAD(&ring->active_list);
- INIT_LIST_HEAD(&ring->request_list);
+ INIT_LIST_HEAD(&engine->active_list);
+ INIT_LIST_HEAD(&engine->request_list);
+}
+
+void
+i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
+{
+ struct drm_device *dev = dev_priv->dev;
+
+ if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
+ !IS_CHERRYVIEW(dev_priv))
+ dev_priv->num_fence_regs = 32;
+ else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
+ IS_I945GM(dev_priv) || IS_G33(dev_priv))
+ dev_priv->num_fence_regs = 16;
+ else
+ dev_priv->num_fence_regs = 8;
+
+ if (intel_vgpu_active(dev))
+ dev_priv->num_fence_regs =
+ I915_READ(vgtif_reg(avail_rs.fence_num));
+
+ /* Initialize fence registers to zero */
+ i915_gem_restore_fences(dev);
+
+ i915_gem_detect_bit_6_swizzle(dev);
}
void
@@ -5055,8 +5110,8 @@ i915_gem_load_init(struct drm_device *dev)
INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
INIT_LIST_HEAD(&dev_priv->mm.bound_list);
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
- for (i = 0; i < I915_NUM_RINGS; i++)
- init_ring_lists(&dev_priv->ring[i]);
+ for (i = 0; i < I915_NUM_ENGINES; i++)
+ init_engine_lists(&dev_priv->engine[i]);
for (i = 0; i < I915_MAX_NUM_FENCES; i++)
INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
@@ -5067,17 +5122,6 @@ i915_gem_load_init(struct drm_device *dev)
dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
- if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
- dev_priv->num_fence_regs = 32;
- else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
- dev_priv->num_fence_regs = 16;
- else
- dev_priv->num_fence_regs = 8;
-
- if (intel_vgpu_active(dev))
- dev_priv->num_fence_regs =
- I915_READ(vgtif_reg(avail_rs.fence_num));
-
/*
* Set initial sequence number for requests.
* Using this number allows the wraparound to happen early,
@@ -5086,11 +5130,8 @@ i915_gem_load_init(struct drm_device *dev)
dev_priv->next_seqno = ((u32)~0 - 0x1100);
dev_priv->last_seqno = ((u32)~0 - 0x1101);
- /* Initialize fence registers to zero */
INIT_LIST_HEAD(&dev_priv->mm.fence_list);
- i915_gem_restore_fences(dev);
- i915_gem_detect_bit_6_swizzle(dev);
init_waitqueue_head(&dev_priv->pending_flip_queue);
dev_priv->mm.interruptible = true;
@@ -5213,11 +5254,12 @@ u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
const struct i915_ggtt_view *view)
{
- struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
+ struct drm_i915_private *dev_priv = to_i915(o->base.dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct i915_vma *vma;
list_for_each_entry(vma, &o->vma_list, obj_link)
- if (vma->vm == ggtt &&
+ if (vma->vm == &ggtt->base &&
i915_ggtt_view_equal(&vma->ggtt_view, view))
return vma->node.start;
@@ -5244,11 +5286,12 @@ bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
const struct i915_ggtt_view *view)
{
- struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
+ struct drm_i915_private *dev_priv = to_i915(o->base.dev);
+ struct i915_ggtt *ggtt = &dev_priv->ggtt;
struct i915_vma *vma;
list_for_each_entry(vma, &o->vma_list, obj_link)
- if (vma->vm == ggtt &&
+ if (vma->vm == &ggtt->base &&
i915_ggtt_view_equal(&vma->ggtt_view, view) &&
drm_mm_node_allocated(&vma->node))
return true;
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.