// SPDX-License-Identifier: MIT /* * Copyright © 2016-2019 Intel Corporation */ #include #include #include #include #include "i915_drv.h" #include "intel_guc_ct.h" #include "gt/intel_gt.h" static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct) { return container_of(ct, struct intel_guc, ct); } static inline struct intel_gt *ct_to_gt(struct intel_guc_ct *ct) { return guc_to_gt(ct_to_guc(ct)); } static inline struct drm_i915_private *ct_to_i915(struct intel_guc_ct *ct) { return ct_to_gt(ct)->i915; } static inline struct drm_device *ct_to_drm(struct intel_guc_ct *ct) { return &ct_to_i915(ct)->drm; } #define CT_ERROR(_ct, _fmt, ...) \ drm_err(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__) #ifdef CONFIG_DRM_I915_DEBUG_GUC #define CT_DEBUG(_ct, _fmt, ...) \ drm_dbg(ct_to_drm(_ct), "CT: " _fmt, ##__VA_ARGS__) #else #define CT_DEBUG(...) do { } while (0) #endif #define CT_PROBE_ERROR(_ct, _fmt, ...) \ i915_probe_error(ct_to_i915(ct), "CT: " _fmt, ##__VA_ARGS__) /** * DOC: CTB Blob * * We allocate single blob to hold both CTB descriptors and buffers: * * +--------+-----------------------------------------------+------+ * | offset | contents | size | * +========+===============================================+======+ * | 0x0000 | H2G `CTB Descriptor`_ (send) | | * +--------+-----------------------------------------------+ 4K | * | 0x0800 | G2H `CTB Descriptor`_ (recv) | | * +--------+-----------------------------------------------+------+ * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K | * | | | | * +--------+-----------------------------------------------+------+ * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K | * | + n*4K | | | * +--------+-----------------------------------------------+------+ * * Size of each `CT Buffer`_ must be multiple of 4K. * We don't expect too many messages in flight at any time, unless we are * using the GuC submission. In that case each request requires a minimum * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this * enough space to avoid backpressure on the driver. We increase the size * of the receive buffer (relative to the send) to ensure a G2H response * CTB has a landing spot. */ #define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) #define CTB_H2G_BUFFER_SIZE (SZ_4K) #define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) #define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) struct ct_request { struct list_head link; u32 fence; u32 status; u32 response_len; u32 *response_buf; }; struct ct_incoming_msg { struct list_head link; u32 size; u32 msg[]; }; enum { CTB_SEND = 0, CTB_RECV = 1 }; enum { CTB_OWNER_HOST = 0 }; static void ct_receive_tasklet_func(struct tasklet_struct *t); static void ct_incoming_request_worker_func(struct work_struct *w); /** * intel_guc_ct_init_early - Initialize CT state without requiring device access * @ct: pointer to CT struct */ void intel_guc_ct_init_early(struct intel_guc_ct *ct) { spin_lock_init(&ct->ctbs.send.lock); spin_lock_init(&ct->ctbs.recv.lock); spin_lock_init(&ct->requests.lock); INIT_LIST_HEAD(&ct->requests.pending); INIT_LIST_HEAD(&ct->requests.incoming); INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func); tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func); init_waitqueue_head(&ct->wq); } static inline const char *guc_ct_buffer_type_to_str(u32 type) { switch (type) { case GUC_CTB_TYPE_HOST2GUC: return "SEND"; case GUC_CTB_TYPE_GUC2HOST: return "RECV"; default: return ""; } } static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc) { memset(desc, 0, sizeof(*desc)); } static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb) { u32 space; ctb->broken = false; ctb->tail = 0; ctb->head = 0; space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space; atomic_set(&ctb->space, space); guc_ct_buffer_desc_init(ctb->desc); } static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb, struct guc_ct_buffer_desc *desc, u32 *cmds, u32 size_in_bytes, u32 resv_space) { GEM_BUG_ON(size_in_bytes % 4); ctb->desc = desc; ctb->cmds = cmds; ctb->size = size_in_bytes / 4; ctb->resv_space = resv_space / 4; guc_ct_buffer_reset(ctb); } static int guc_action_register_ct_buffer(struct intel_guc *guc, u32 type, u32 desc_addr, u32 buff_addr, u32 size) { u32 request[HOST2GUC_REGISTER_CTB_REQUEST_MSG_LEN] = { FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_REGISTER_CTB), FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_SIZE, size / SZ_4K - 1) | FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_1_TYPE, type), FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_2_DESC_ADDR, desc_addr), FIELD_PREP(HOST2GUC_REGISTER_CTB_REQUEST_MSG_3_BUFF_ADDR, buff_addr), }; int ret; GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); GEM_BUG_ON(size % SZ_4K); /* CT registration must go over MMIO */ ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); return ret > 0 ? -EPROTO : ret; } static int ct_register_buffer(struct intel_guc_ct *ct, u32 type, u32 desc_addr, u32 buff_addr, u32 size) { int err; err = i915_inject_probe_error(guc_to_gt(ct_to_guc(ct))->i915, -ENXIO); if (unlikely(err)) return err; err = guc_action_register_ct_buffer(ct_to_guc(ct), type, desc_addr, buff_addr, size); if (unlikely(err)) CT_ERROR(ct, "Failed to register %s buffer (%pe)\n", guc_ct_buffer_type_to_str(type), ERR_PTR(err)); return err; } static int guc_action_deregister_ct_buffer(struct intel_guc *guc, u32 type) { u32 request[HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_LEN] = { FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_DEREGISTER_CTB), FIELD_PREP(HOST2GUC_DEREGISTER_CTB_REQUEST_MSG_1_TYPE, type), }; int ret; GEM_BUG_ON(type != GUC_CTB_TYPE_HOST2GUC && type != GUC_CTB_TYPE_GUC2HOST); /* CT deregistration must go over MMIO */ ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0); return ret > 0 ? -EPROTO : ret; } static int ct_deregister_buffer(struct intel_guc_ct *ct, u32 type) { int err = guc_action_deregister_ct_buffer(ct_to_guc(ct), type); if (unlikely(err)) CT_ERROR(ct, "Failed to deregister %s buffer (%pe)\n", guc_ct_buffer_type_to_str(type), ERR_PTR(err)); return err; } /** * intel_guc_ct_init - Init buffer-based communication * @ct: pointer to CT struct * * Allocate memory required for buffer-based communication. * * Return: 0 on success, a negative errno code on failure. */ int intel_guc_ct_init(struct intel_guc_ct *ct) { struct intel_guc *guc = ct_to_guc(ct); struct guc_ct_buffer_desc *desc; u32 blob_size; u32 cmds_size; u32 resv_space; void *blob; u32 *cmds; int err; err = i915_inject_probe_error(guc_to_gt(guc)->i915, -ENXIO); if (err) return err; GEM_BUG_ON(ct->vma); blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE; err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob); if (unlikely(err)) { CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n", blob_size, ERR_PTR(err)); return err; } CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size); /* store pointers to desc and cmds for send ctb */ desc = blob; cmds = blob + 2 * CTB_DESC_SIZE; cmds_size = CTB_H2G_BUFFER_SIZE; resv_space = 0; CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send", ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, resv_space); guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space); /* store pointers to desc and cmds for recv ctb */ desc = blob + CTB_DESC_SIZE; cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE; cmds_size = CTB_G2H_BUFFER_SIZE; resv_space = G2H_ROOM_BUFFER_SIZE; CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv", ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size, resv_space); guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space); return 0; } /** * intel_guc_ct_fini - Fini buffer-based communication * @ct: pointer to CT struct * * Deallocate memory required for buffer-based communication. */ void intel_guc_ct_fini(struct intel_guc_ct *ct) { GEM_BUG_ON(ct->enabled); tasklet_kill(&ct->receive_tasklet); i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP); memset(ct, 0, sizeof(*ct)); } /** * intel_guc_ct_enable - Enable buffer based command transport. * @ct: pointer to CT struct * * Return: 0 on success, a negative errno code on failure. */ int intel_guc_ct_enable(struct intel_guc_ct *ct) { struct intel_guc *guc = ct_to_guc(ct); u32 base, desc, cmds; void *blob; int err; GEM_BUG_ON(ct->enabled); /* vma should be already allocated and map'ed */ GEM_BUG_ON(!ct->vma); GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj)); base = intel_guc_ggtt_offset(guc, ct->vma); /* blob should start with send descriptor */ blob = __px_vaddr(ct->vma->obj); GEM_BUG_ON(blob != ct->ctbs.send.desc); /* (re)initialize descriptors */ guc_ct_buffer_reset(&ct->ctbs.send); guc_ct_buffer_reset(&ct->ctbs.recv); /* * Register both CT buffers starting with RECV buffer. * Descriptors are in first half of the blob. */ desc = base + ptrdiff(ct->ctbs.recv.desc, blob); cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob); err = ct_register_buffer(ct, GUC_CTB_TYPE_GUC2HOST, desc, cmds, ct->ctbs.recv.size * 4); if (unlikely(err)) goto err_out; desc = base + ptrdiff(ct->ctbs.send.desc, blob); cmds = base + ptrdiff(ct->ctbs.send.cmds, blob); err = ct_register_buffer(ct, GUC_CTB_TYPE_HOST2GUC, desc, cmds, ct->ctbs.send.size * 4); if (unlikely(err)) goto err_deregister; ct->enabled = true; ct->stall_time = KTIME_MAX; return 0; err_deregister: ct_deregister_buffer(ct, GUC_CTB_TYPE_GUC2HOST); err_out: CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err)); return err; } /** * intel_guc_ct_disable - Disable buffer based command transport. * @ct: pointer to CT struct */ void intel_guc_ct_disable(struct intel_guc_ct *ct) { struct intel_guc *guc = ct_to_guc(ct); GEM_BUG_ON(!ct->enabled); ct->enabled = false; if (intel_guc_is_fw_running(guc)) { ct_deregister_buffer(ct, GUC_CTB_TYPE_HOST2GUC); ct_deregister_buffer(ct, GUC_CTB_TYPE_GUC2HOST); } } static u32 ct_get_next_fence(struct intel_guc_ct *ct) { /* For now it's trivial */ return ++ct->requests.last_fence; } static int ct_write(struct intel_guc_ct *ct, const u32 *action, u32 len /* in dwords */, u32 fence, u32 flags) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; struct guc_ct_buffer_desc *desc = ctb->desc; u32 tail = ctb->tail; u32 size = ctb->size; u32 header; u32 hxg; u32 type; u32 *cmds = ctb->cmds; unsigned int i; if (unlikely(desc->status)) goto corrupted; GEM_BUG_ON(tail > size); #ifdef CONFIG_DRM_I915_DEBUG_GUC if (unlikely(tail != READ_ONCE(desc->tail))) { CT_ERROR(ct, "Tail was modified %u != %u\n", desc->tail, tail); desc->status |= GUC_CTB_STATUS_MISMATCH; goto corrupted; } if (unlikely(READ_ONCE(desc->head) >= size)) { CT_ERROR(ct, "Invalid head offset %u >= %u)\n", desc->head, size); desc->status |= GUC_CTB_STATUS_OVERFLOW; goto corrupted; } #endif /* * dw0: CT header (including fence) * dw1: HXG header (including action code) * dw2+: action data */ header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence); type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_EVENT : GUC_HXG_TYPE_REQUEST; hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) | FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | GUC_HXG_EVENT_MSG_0_DATA0, action[0]); CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n", tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]); cmds[tail] = header; tail = (tail + 1) % size; cmds[tail] = hxg; tail = (tail + 1) % size; for (i = 1; i < len; i++) { cmds[tail] = action[i]; tail = (tail + 1) % size; } GEM_BUG_ON(tail > size); /* * make sure H2G buffer update and LRC tail update (if this triggering a * submission) are visible before updating the descriptor tail */ intel_guc_write_barrier(ct_to_guc(ct)); /* update local copies */ ctb->tail = tail; GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN); atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space); /* now update descriptor */ WRITE_ONCE(desc->tail, tail); return 0; corrupted: CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", desc->head, desc->tail, desc->status); ctb->broken = true; return -EPIPE; } /** * wait_for_ct_request_update - Wait for CT request state update. * @req: pointer to pending request * @status: placeholder for status * * For each sent request, GuC shall send back CT response message. * Our message handler will update status of tracked request once * response message with given fence is received. Wait here and * check for valid response status value. * * Return: * * 0 response received (status is valid) * * -ETIMEDOUT no response within hardcoded timeout */ static int wait_for_ct_request_update(struct ct_request *req, u32 *status) { int err; /* * Fast commands should complete in less than 10us, so sample quickly * up to that length of time, then switch to a slower sleep-wait loop. * No GuC command should ever take longer than 10ms but many GuC * commands can be inflight at time, so use a 1s timeout on the slower * sleep-wait loop. */ #define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10 #define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000 #define done \ (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \ GUC_HXG_ORIGIN_GUC) err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS); if (err) err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS); #undef done *status = req->status; return err; } #define GUC_CTB_TIMEOUT_MS 1500 static inline bool ct_deadlocked(struct intel_guc_ct *ct) { long timeout = GUC_CTB_TIMEOUT_MS; bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout; if (unlikely(ret)) { struct guc_ct_buffer_desc *send = ct->ctbs.send.desc; struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc; CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n", ktime_ms_delta(ktime_get(), ct->stall_time), send->status, recv->status); CT_ERROR(ct, "H2G Space: %u (Bytes)\n", atomic_read(&ct->ctbs.send.space) * 4); CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head); CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail); CT_ERROR(ct, "G2H Space: %u (Bytes)\n", atomic_read(&ct->ctbs.recv.space) * 4); CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head); CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail); ct->ctbs.send.broken = true; } return ret; } static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; /* * We leave a certain amount of space in the G2H CTB buffer for * unexpected G2H CTBs (e.g. logging, engine hang, etc...) */ return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw; } static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw) { lockdep_assert_held(&ct->ctbs.send.lock); GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw)); if (g2h_len_dw) atomic_sub(g2h_len_dw, &ct->ctbs.recv.space); } static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw) { atomic_add(g2h_len_dw, &ct->ctbs.recv.space); } static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; struct guc_ct_buffer_desc *desc = ctb->desc; u32 head; u32 space; if (atomic_read(&ctb->space) >= len_dw) return true; head = READ_ONCE(desc->head); if (unlikely(head > ctb->size)) { CT_ERROR(ct, "Invalid head offset %u >= %u)\n", head, ctb->size); desc->status |= GUC_CTB_STATUS_OVERFLOW; ctb->broken = true; return false; } space = CIRC_SPACE(ctb->tail, head, ctb->size); atomic_set(&ctb->space, space); return space >= len_dw; } static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw) { bool h2g = h2g_has_room(ct, h2g_dw); bool g2h = g2h_has_room(ct, g2h_dw); lockdep_assert_held(&ct->ctbs.send.lock); if (unlikely(!h2g || !g2h)) { if (ct->stall_time == KTIME_MAX) ct->stall_time = ktime_get(); /* Be paranoid and kick G2H tasklet to free credits */ if (!g2h) tasklet_hi_schedule(&ct->receive_tasklet); if (unlikely(ct_deadlocked(ct))) return -EPIPE; else return -EBUSY; } ct->stall_time = KTIME_MAX; return 0; } #define G2H_LEN_DW(f) ({ \ typeof(f) f_ = (f); \ FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \ FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \ GUC_CTB_HXG_MSG_MIN_LEN : 0; \ }) static int ct_send_nb(struct intel_guc_ct *ct, const u32 *action, u32 len, u32 flags) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; unsigned long spin_flags; u32 g2h_len_dw = G2H_LEN_DW(flags); u32 fence; int ret; spin_lock_irqsave(&ctb->lock, spin_flags); ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw); if (unlikely(ret)) goto out; fence = ct_get_next_fence(ct); ret = ct_write(ct, action, len, fence, flags); if (unlikely(ret)) goto out; g2h_reserve_space(ct, g2h_len_dw); intel_guc_notify(ct_to_guc(ct)); out: spin_unlock_irqrestore(&ctb->lock, spin_flags); return ret; } static int ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, u32 *response_buf, u32 response_buf_size, u32 *status) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.send; struct ct_request request; unsigned long flags; unsigned int sleep_period_ms = 1; u32 fence; int err; GEM_BUG_ON(!ct->enabled); GEM_BUG_ON(!len); GEM_BUG_ON(len & ~GUC_CT_MSG_LEN_MASK); GEM_BUG_ON(!response_buf && response_buf_size); might_sleep(); /* * We use a lazy spin wait loop here as we believe that if the CT * buffers are sized correctly the flow control condition should be * rare. Reserving the maximum size in the G2H credits as we don't know * how big the response is going to be. */ retry: spin_lock_irqsave(&ctb->lock, flags); if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) || !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) { if (ct->stall_time == KTIME_MAX) ct->stall_time = ktime_get(); spin_unlock_irqrestore(&ctb->lock, flags); if (unlikely(ct_deadlocked(ct))) return -EPIPE; if (msleep_interruptible(sleep_period_ms)) return -EINTR; sleep_period_ms = sleep_period_ms << 1; goto retry; } ct->stall_time = KTIME_MAX; fence = ct_get_next_fence(ct); request.fence = fence; request.status = 0; request.response_len = response_buf_size; request.response_buf = response_buf; spin_lock(&ct->requests.lock); list_add_tail(&request.link, &ct->requests.pending); spin_unlock(&ct->requests.lock); err = ct_write(ct, action, len, fence, 0); g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); spin_unlock_irqrestore(&ctb->lock, flags); if (unlikely(err)) goto unlink; intel_guc_notify(ct_to_guc(ct)); err = wait_for_ct_request_update(&request, status); g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); if (unlikely(err)) { CT_ERROR(ct, "No response for request %#x (fence %u)\n", action[0], request.fence); goto unlink; } if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) { err = -EIO; goto unlink; } if (response_buf) { /* There shall be no data in the status */ WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status)); /* Return actual response len */ err = request.response_len; } else { /* There shall be no response payload */ WARN_ON(request.response_len); /* Return data decoded from the status dword */ err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status); } unlink: spin_lock_irqsave(&ct->requests.lock, flags); list_del(&request.link); spin_unlock_irqrestore(&ct->requests.lock, flags); return err; } /* * Command Transport (CT) buffer based GuC send function. */ int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len, u32 *response_buf, u32 response_buf_size, u32 flags) { u32 status = ~0; /* undefined */ int ret; if (unlikely(!ct->enabled)) { struct intel_guc *guc = ct_to_guc(ct); struct intel_uc *uc = container_of(guc, struct intel_uc, guc); WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action); return -ENODEV; } if (unlikely(ct->ctbs.send.broken)) return -EPIPE; if (flags & INTEL_GUC_CT_SEND_NB) return ct_send_nb(ct, action, len, flags); ret = ct_send(ct, action, len, response_buf, response_buf_size, &status); if (unlikely(ret < 0)) { CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n", action[0], ERR_PTR(ret), status); } else if (unlikely(ret)) { CT_DEBUG(ct, "send action %#x returned %d (%#x)\n", action[0], ret, ret); } return ret; } static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords) { struct ct_incoming_msg *msg; msg = kmalloc(sizeof(*msg) + sizeof(u32) * num_dwords, GFP_ATOMIC); if (msg) msg->size = num_dwords; return msg; } static void ct_free_msg(struct ct_incoming_msg *msg) { kfree(msg); } /* * Return: number available remaining dwords to read (0 if empty) * or a negative error code on failure */ static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg) { struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv; struct guc_ct_buffer_desc *desc = ctb->desc; u32 head = ctb->head; u32 tail = READ_ONCE(desc->tail); u32 size = ctb->size; u32 *cmds = ctb->cmds; s32 available; unsigned int len; unsigned int i; u32 header; if (unlikely(ctb->broken)) return -EPIPE; if (unlikely(desc->status)) goto corrupted; GEM_BUG_ON(head > size); #ifdef CONFIG_DRM_I915_DEBUG_GUC if (unlikely(head != READ_ONCE(desc->head))) { CT_ERROR(ct, "Head was modified %u != %u\n", desc->head, head); desc->status |= GUC_CTB_STATUS_MISMATCH; goto corrupted; } #endif if (unlikely(tail >= size)) { CT_ERROR(ct, "Invalid tail offset %u >= %u)\n", tail, size); desc->status |= GUC_CTB_STATUS_OVERFLOW; goto corrupted; } /* tail == head condition indicates empty */ available = tail - head; if (unlikely(available == 0)) { *msg = NULL; return 0; } /* beware of buffer wrap case */ if (unlikely(available < 0)) available += size; CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size); GEM_BUG_ON(available < 0); header = cmds[head]; head = (head + 1) % size; /* message len with header */ len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN; if (unlikely(len > (u32)available)) { CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n", 4, &header, 4 * (head + available - 1 > size ? size - head : available - 1), &cmds[head], 4 * (head + available - 1 > size ? available - 1 - size + head : 0), &cmds[0]); desc->status |= GUC_CTB_STATUS_UNDERFLOW; goto corrupted; } *msg = ct_alloc_msg(len); if (!*msg) { CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n", 4, &header, 4 * (head + available - 1 > size ? size - head : available - 1), &cmds[head], 4 * (head + available - 1 > size ? available - 1 - size + head : 0), &cmds[0]); return available; } (*msg)->msg[0] = header; for (i = 1; i < len; i++) { (*msg)->msg[i] = cmds[head]; head = (head + 1) % size; } CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg); /* update local copies */ ctb->head = head; /* now update descriptor */ WRITE_ONCE(desc->head, head); return available - len; corrupted: CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n", desc->head, desc->tail, desc->status); ctb->broken = true; return -EPIPE; } static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response) { u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]); u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]); const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN]; const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN]; u32 datalen = len - GUC_HXG_MSG_MIN_LEN; struct ct_request *req; unsigned long flags; bool found = false; int err = 0; GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN); GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC); GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS && FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE); CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]); spin_lock_irqsave(&ct->requests.lock, flags); list_for_each_entry(req, &ct->requests.pending, link) { if (unlikely(fence != req->fence)) { CT_DEBUG(ct, "request %u awaits response\n", req->fence); continue; } if (unlikely(datalen > req->response_len)) { CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n", req->fence, datalen, req->response_len); datalen = min(datalen, req->response_len); err = -EMSGSIZE; } if (datalen) memcpy(req->response_buf, data, 4 * datalen); req->response_len = datalen; WRITE_ONCE(req->status, hxg[0]); found = true; break; } if (!found) { CT_ERROR(ct, "Unsolicited response (fence %u)\n", fence); CT_ERROR(ct, "Could not find fence=%u, last_fence=%u\n", fence, ct->requests.last_fence); list_for_each_entry(req, &ct->requests.pending, link) CT_ERROR(ct, "request %u awaits response\n", req->fence); err = -ENOKEY; } spin_unlock_irqrestore(&ct->requests.lock, flags); if (unlikely(err)) return err; ct_free_msg(response); return 0; } static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request) { struct intel_guc *guc = ct_to_guc(ct); const u32 *hxg; const u32 *payload; u32 hxg_len, action, len; int ret; hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; hxg_len = request->size - GUC_CTB_MSG_MIN_LEN; payload = &hxg[GUC_HXG_MSG_MIN_LEN]; action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); len = hxg_len - GUC_HXG_MSG_MIN_LEN; CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload); switch (action) { case INTEL_GUC_ACTION_DEFAULT: ret = intel_guc_to_host_process_recv_msg(guc, payload, len); break; case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: ret = intel_guc_deregister_done_process_msg(guc, payload, len); break; case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: ret = intel_guc_sched_done_process_msg(guc, payload, len); break; case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: ret = intel_guc_context_reset_process_msg(guc, payload, len); break; case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: ret = intel_guc_engine_failure_process_msg(guc, payload, len); break; default: ret = -EOPNOTSUPP; break; } if (unlikely(ret)) { CT_ERROR(ct, "Failed to process request %04x (%pe)\n", action, ERR_PTR(ret)); return ret; } ct_free_msg(request); return 0; } static bool ct_process_incoming_requests(struct intel_guc_ct *ct) { unsigned long flags; struct ct_incoming_msg *request; bool done; int err; spin_lock_irqsave(&ct->requests.lock, flags); request = list_first_entry_or_null(&ct->requests.incoming, struct ct_incoming_msg, link); if (request) list_del(&request->link); done = !!list_empty(&ct->requests.incoming); spin_unlock_irqrestore(&ct->requests.lock, flags); if (!request) return true; err = ct_process_request(ct, request); if (unlikely(err)) { CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", ERR_PTR(err), 4 * request->size, request->msg); ct_free_msg(request); } return done; } static void ct_incoming_request_worker_func(struct work_struct *w) { struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, requests.worker); bool done; do { done = ct_process_incoming_requests(ct); } while (!done); } static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request) { const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN]; u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]); unsigned long flags; GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT); /* * Adjusting the space must be done in IRQ or deadlock can occur as the * CTB processing in the below workqueue can send CTBs which creates a * circular dependency if the space was returned there. */ switch (action) { case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE: g2h_release_space(ct, request->size); } spin_lock_irqsave(&ct->requests.lock, flags); list_add_tail(&request->link, &ct->requests.incoming); spin_unlock_irqrestore(&ct->requests.lock, flags); queue_work(system_unbound_wq, &ct->requests.worker); return 0; } static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) { u32 origin, type; u32 *hxg; int err; if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN)) return -EBADMSG; hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN]; origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]); if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { err = -EPROTO; goto failed; } type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]); switch (type) { case GUC_HXG_TYPE_EVENT: err = ct_handle_event(ct, msg); break; case GUC_HXG_TYPE_RESPONSE_SUCCESS: case GUC_HXG_TYPE_RESPONSE_FAILURE: err = ct_handle_response(ct, msg); break; default: err = -EOPNOTSUPP; } if (unlikely(err)) { failed: CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n", ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg); } return err; } static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg) { u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]); int err; if (format == GUC_CTB_FORMAT_HXG) err = ct_handle_hxg(ct, msg); else err = -EOPNOTSUPP; if (unlikely(err)) { CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n", ERR_PTR(err), 4 * msg->size, msg->msg); ct_free_msg(msg); } } /* * Return: number available remaining dwords to read (0 if empty) * or a negative error code on failure */ static int ct_receive(struct intel_guc_ct *ct) { struct ct_incoming_msg *msg = NULL; unsigned long flags; int ret; spin_lock_irqsave(&ct->ctbs.recv.lock, flags); ret = ct_read(ct, &msg); spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags); if (ret < 0) return ret; if (msg) ct_handle_msg(ct, msg); return ret; } static void ct_try_receive_message(struct intel_guc_ct *ct) { int ret; if (GEM_WARN_ON(!ct->enabled)) return; ret = ct_receive(ct); if (ret > 0) tasklet_hi_schedule(&ct->receive_tasklet); } static void ct_receive_tasklet_func(struct tasklet_struct *t) { struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet); ct_try_receive_message(ct); } /* * When we're communicating with the GuC over CT, GuC uses events * to notify us about new messages being posted on the RECV buffer. */ void intel_guc_ct_event_handler(struct intel_guc_ct *ct) { if (unlikely(!ct->enabled)) { WARN(1, "Unexpected GuC event received while CT disabled!\n"); return; } ct_try_receive_message(ct); } void intel_guc_ct_print_info(struct intel_guc_ct *ct, struct drm_printer *p) { drm_printf(p, "CT %s\n", enableddisabled(ct->enabled)); if (!ct->enabled) return; drm_printf(p, "H2G Space: %u\n", atomic_read(&ct->ctbs.send.space) * 4); drm_printf(p, "Head: %u\n", ct->ctbs.send.desc->head); drm_printf(p, "Tail: %u\n", ct->ctbs.send.desc->tail); drm_printf(p, "G2H Space: %u\n", atomic_read(&ct->ctbs.recv.space) * 4); drm_printf(p, "Head: %u\n", ct->ctbs.recv.desc->head); drm_printf(p, "Tail: %u\n", ct->ctbs.recv.desc->tail); }