diff options
Diffstat (limited to 'drivers/usb/dwc3/gadget.c')
| -rw-r--r-- | drivers/usb/dwc3/gadget.c | 773 |
1 files changed, 361 insertions, 412 deletions
diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 0dedf8a799f4..69bf137aab37 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -27,6 +27,9 @@ #include "gadget.h" #include "io.h" +#define DWC3_ALIGN_FRAME(d) (((d)->frame_number + (d)->interval) \ + & ~((d)->interval - 1)) + /** * dwc3_gadget_set_test_mode - enables usb2 test modes * @dwc: pointer to our context structure @@ -375,6 +378,7 @@ int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd, switch (DWC3_DEPCMD_CMD(cmd)) { case DWC3_DEPCMD_STARTTRANSFER: dep->flags |= DWC3_EP_TRANSFER_STARTED; + dwc3_gadget_ep_get_transfer_index(dep); break; case DWC3_DEPCMD_ENDTRANSFER: dep->flags &= ~DWC3_EP_TRANSFER_STARTED; @@ -455,7 +459,17 @@ static void dwc3_free_trb_pool(struct dwc3_ep *dep) dep->trb_pool_dma = 0; } -static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep); +static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) +{ + struct dwc3_gadget_ep_cmd_params params; + + memset(¶ms, 0x00, sizeof(params)); + + params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); + + return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, + ¶ms); +} /** * dwc3_gadget_start_config - configure ep resources @@ -491,9 +505,10 @@ static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep); * triggered only when called for EP0-out, which always happens first, and which * should only happen in one of the above conditions. */ -static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep) +static int dwc3_gadget_start_config(struct dwc3_ep *dep) { struct dwc3_gadget_ep_cmd_params params; + struct dwc3 *dwc; u32 cmd; int i; int ret; @@ -503,6 +518,7 @@ static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep) memset(¶ms, 0x00, sizeof(params)); cmd = DWC3_DEPCMD_DEPSTARTCFG; + dwc = dep->dwc; ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); if (ret) @@ -514,7 +530,7 @@ static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep) if (!dep) continue; - ret = dwc3_gadget_set_xfer_resource(dwc, dep); + ret = dwc3_gadget_set_xfer_resource(dep); if (ret) return ret; } @@ -522,16 +538,12 @@ static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep) return 0; } -static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep, - bool modify, bool restore) +static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) { const struct usb_ss_ep_comp_descriptor *comp_desc; const struct usb_endpoint_descriptor *desc; struct dwc3_gadget_ep_cmd_params params; - - if (dev_WARN_ONCE(dwc->dev, modify && restore, - "Can't modify and restore\n")) - return -EINVAL; + struct dwc3 *dwc = dep->dwc; comp_desc = dep->endpoint.comp_desc; desc = dep->endpoint.desc; @@ -547,14 +559,9 @@ static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep, params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); } - if (modify) { - params.param0 |= DWC3_DEPCFG_ACTION_MODIFY; - } else if (restore) { - params.param0 |= DWC3_DEPCFG_ACTION_RESTORE; + params.param0 |= action; + if (action == DWC3_DEPCFG_ACTION_RESTORE) params.param2 |= dep->saved_state; - } else { - params.param0 |= DWC3_DEPCFG_ACTION_INIT; - } if (usb_endpoint_xfer_control(desc)) params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN; @@ -594,29 +601,15 @@ static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep, return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); } -static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep) -{ - struct dwc3_gadget_ep_cmd_params params; - - memset(¶ms, 0x00, sizeof(params)); - - params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); - - return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, - ¶ms); -} - /** * __dwc3_gadget_ep_enable - initializes a hw endpoint * @dep: endpoint to be initialized - * @modify: if true, modify existing endpoint configuration - * @restore: if true, restore endpoint configuration from scratch buffer + * @action: one of INIT, MODIFY or RESTORE * * Caller should take care of locking. Execute all necessary commands to * initialize a HW endpoint so it can be used by a gadget driver. */ -static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, - bool modify, bool restore) +static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) { const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; struct dwc3 *dwc = dep->dwc; @@ -625,12 +618,12 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, int ret; if (!(dep->flags & DWC3_EP_ENABLED)) { - ret = dwc3_gadget_start_config(dwc, dep); + ret = dwc3_gadget_start_config(dep); if (ret) return ret; } - ret = dwc3_gadget_set_ep_config(dwc, dep, modify, restore); + ret = dwc3_gadget_set_ep_config(dep, action); if (ret) return ret; @@ -671,7 +664,8 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, * Issue StartTransfer here with no-op TRB so we can always rely on No * Response Update Transfer command. */ - if (usb_endpoint_xfer_bulk(desc)) { + if (usb_endpoint_xfer_bulk(desc) || + usb_endpoint_xfer_int(desc)) { struct dwc3_gadget_ep_cmd_params params; struct dwc3_trb *trb; dma_addr_t trb_dma; @@ -689,26 +683,20 @@ static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); if (ret < 0) return ret; - - dep->flags |= DWC3_EP_BUSY; - - dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep); - WARN_ON_ONCE(!dep->resource_index); } - out: trace_dwc3_gadget_ep_enable(dep); return 0; } -static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force); +static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force); static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep) { struct dwc3_request *req; - dwc3_stop_active_transfer(dwc, dep->number, true); + dwc3_stop_active_transfer(dep, true); /* - giveback all requests to gadget driver */ while (!list_empty(&dep->started_list)) { @@ -806,7 +794,7 @@ static int dwc3_gadget_ep_enable(struct usb_ep *ep, return 0; spin_lock_irqsave(&dwc->lock, flags); - ret = __dwc3_gadget_ep_enable(dep, false, false); + ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); spin_unlock_irqrestore(&dwc->lock, flags); return ret; @@ -840,7 +828,7 @@ static int dwc3_gadget_ep_disable(struct usb_ep *ep) } static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, - gfp_t gfp_flags) + gfp_t gfp_flags) { struct dwc3_request *req; struct dwc3_ep *dep = to_dwc3_ep(ep); @@ -849,11 +837,10 @@ static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, if (!req) return NULL; + req->direction = dep->direction; req->epnum = dep->number; req->dep = dep; - dep->allocated_requests++; - trace_dwc3_alloc_request(req); return &req->request; @@ -863,14 +850,58 @@ static void dwc3_gadget_ep_free_request(struct usb_ep *ep, struct usb_request *request) { struct dwc3_request *req = to_dwc3_request(request); - struct dwc3_ep *dep = to_dwc3_ep(ep); - dep->allocated_requests--; trace_dwc3_free_request(req); kfree(req); } -static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep); +/** + * dwc3_ep_prev_trb - returns the previous TRB in the ring + * @dep: The endpoint with the TRB ring + * @index: The index of the current TRB in the ring + * + * Returns the TRB prior to the one pointed to by the index. If the + * index is 0, we will wrap backwards, skip the link TRB, and return + * the one just before that. + */ +static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) +{ + u8 tmp = index; + + if (!tmp) + tmp = DWC3_TRB_NUM - 1; + + return &dep->trb_pool[tmp - 1]; +} + +static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) +{ + struct dwc3_trb *tmp; + u8 trbs_left; + + /* + * If enqueue & dequeue are equal than it is either full or empty. + * + * One way to know for sure is if the TRB right before us has HWO bit + * set or not. If it has, then we're definitely full and can't fit any + * more transfers in our ring. + */ + if (dep->trb_enqueue == dep->trb_dequeue) { + tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue); + if (tmp->ctrl & DWC3_TRB_CTRL_HWO) + return 0; + + return DWC3_TRB_NUM - 1; + } + + trbs_left = dep->trb_dequeue - dep->trb_enqueue; + trbs_left &= (DWC3_TRB_NUM - 1); + + if (dep->trb_dequeue < dep->trb_enqueue) + trbs_left--; + + return trbs_left; +} static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb, dma_addr_t dma, unsigned length, unsigned chain, unsigned node, @@ -985,11 +1016,19 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_request *req, unsigned chain, unsigned node) { struct dwc3_trb *trb; - unsigned length = req->request.length; + unsigned int length; + dma_addr_t dma; unsigned stream_id = req->request.stream_id; unsigned short_not_ok = req->request.short_not_ok; unsigned no_interrupt = req->request.no_interrupt; - dma_addr_t dma = req->request.dma; + + if (req->request.num_sgs > 0) { + length = sg_dma_len(req->start_sg); + dma = sg_dma_address(req->start_sg); + } else { + length = req->request.length; + dma = req->request.dma; + } trb = &dep->trb_pool[dep->trb_enqueue]; @@ -997,69 +1036,23 @@ static void dwc3_prepare_one_trb(struct dwc3_ep *dep, dwc3_gadget_move_started_request(req); req->trb = trb; req->trb_dma = dwc3_trb_dma_offset(dep, trb); - dep->queued_requests++; } __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node, stream_id, short_not_ok, no_interrupt); } -/** - * dwc3_ep_prev_trb - returns the previous TRB in the ring - * @dep: The endpoint with the TRB ring - * @index: The index of the current TRB in the ring - * - * Returns the TRB prior to the one pointed to by the index. If the - * index is 0, we will wrap backwards, skip the link TRB, and return - * the one just before that. - */ -static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) -{ - u8 tmp = index; - - if (!tmp) - tmp = DWC3_TRB_NUM - 1; - - return &dep->trb_pool[tmp - 1]; -} - -static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) -{ - struct dwc3_trb *tmp; - u8 trbs_left; - - /* - * If enqueue & dequeue are equal than it is either full or empty. - * - * One way to know for sure is if the TRB right before us has HWO bit - * set or not. If it has, then we're definitely full and can't fit any - * more transfers in our ring. - */ - if (dep->trb_enqueue == dep->trb_dequeue) { - tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue); - if (tmp->ctrl & DWC3_TRB_CTRL_HWO) - return 0; - - return DWC3_TRB_NUM - 1; - } - - trbs_left = dep->trb_dequeue - dep->trb_enqueue; - trbs_left &= (DWC3_TRB_NUM - 1); - - if (dep->trb_dequeue < dep->trb_enqueue) - trbs_left--; - - return trbs_left; -} - static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep, struct dwc3_request *req) { - struct scatterlist *sg = req->sg; + struct scatterlist *sg = req->start_sg; struct scatterlist *s; int i; - for_each_sg(sg, s, req->num_pending_sgs, i) { + unsigned int remaining = req->request.num_mapped_sgs + - req->num_queued_sgs; + + for_each_sg(sg, s, remaining, i) { unsigned int length = req->request.length; unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); unsigned int rem = length % maxp; @@ -1088,6 +1081,18 @@ static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep, dwc3_prepare_one_trb(dep, req, chain, i); } + /* + * There can be a situation where all sgs in sglist are not + * queued because of insufficient trb number. To handle this + * case, update start_sg to next sg to be queued, so that + * we have free trbs we can continue queuing from where we + * previously stopped + */ + if (chain) + req->start_sg = sg_next(s); + + req->num_queued_sgs++; + if (!dwc3_calc_trbs_left(dep)) break; } @@ -1178,6 +1183,8 @@ static void dwc3_prepare_trbs(struct dwc3_ep *dep) return; req->sg = req->request.sg; + req->start_sg = req->sg; + req->num_queued_sgs = 0; req->num_pending_sgs = req->request.num_mapped_sgs; if (req->num_pending_sgs > 0) @@ -1201,7 +1208,7 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) if (!dwc3_calc_trbs_left(dep)) return 0; - starting = !(dep->flags & DWC3_EP_BUSY); + starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); dwc3_prepare_trbs(dep); req = next_request(&dep->started_list); @@ -1233,18 +1240,10 @@ static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) */ if (req->trb) memset(req->trb, 0, sizeof(struct dwc3_trb)); - dep->queued_requests--; dwc3_gadget_del_and_unmap_request(dep, req, ret); return ret; } - dep->flags |= DWC3_EP_BUSY; - - if (starting) { - dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep); - WARN_ON_ONCE(!dep->resource_index); - } - return 0; } @@ -1256,35 +1255,19 @@ static int __dwc3_gadget_get_frame(struct dwc3 *dwc) return DWC3_DSTS_SOFFN(reg); } -static void __dwc3_gadget_start_isoc(struct dwc3 *dwc, - struct dwc3_ep *dep, u32 cur_uf) +static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep) { if (list_empty(&dep->pending_list)) { - dev_info(dwc->dev, "%s: ran out of requests\n", + dev_info(dep->dwc->dev, "%s: ran out of requests\n", dep->name); dep->flags |= DWC3_EP_PENDING_REQUEST; return; } - /* - * Schedule the first trb for one interval in the future or at - * least 4 microframes. - */ - dep->frame_number = cur_uf + max_t(u32, 4, dep->interval); + dep->frame_number = DWC3_ALIGN_FRAME(dep); __dwc3_gadget_kick_transfer(dep); } -static void dwc3_gadget_start_isoc(struct dwc3 *dwc, - struct dwc3_ep *dep, const struct dwc3_event_depevt *event) -{ - u32 cur_uf, mask; - - mask = ~(dep->interval - 1); - cur_uf = event->parameters & mask; - - __dwc3_gadget_start_isoc(dwc, dep, cur_uf); -} - static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) { struct dwc3 *dwc = dep->dwc; @@ -1303,8 +1286,6 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) req->request.actual = 0; req->request.status = -EINPROGRESS; - req->direction = dep->direction; - req->epnum = dep->number; trace_dwc3_ep_queue(req); @@ -1319,28 +1300,18 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) * errors which will force us issue EndTransfer command. */ if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { - if ((dep->flags & DWC3_EP_PENDING_REQUEST)) { - if (dep->flags & DWC3_EP_TRANSFER_STARTED) { - dwc3_stop_active_transfer(dwc, dep->number, true); - dep->flags = DWC3_EP_ENABLED; - } else { - u32 cur_uf; + if (!(dep->flags & DWC3_EP_PENDING_REQUEST) && + !(dep->flags & DWC3_EP_TRANSFER_STARTED)) + return 0; - cur_uf = __dwc3_gadget_get_frame(dwc); - __dwc3_gadget_start_isoc(dwc, dep, cur_uf); - dep->flags &= ~DWC3_EP_PENDING_REQUEST; + if ((dep->flags & DWC3_EP_PENDING_REQUEST)) { + if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { + __dwc3_gadget_start_isoc(dep); + return 0; } - return 0; } - - if ((dep->flags & DWC3_EP_BUSY) && - !(dep->flags & DWC3_EP_MISSED_ISOC)) - goto out; - - return 0; } -out: return __dwc3_gadget_kick_transfer(dep); } @@ -1390,7 +1361,7 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, } if (r == req) { /* wait until it is processed */ - dwc3_stop_active_transfer(dwc, dep->number, true); + dwc3_stop_active_transfer(dep, true); /* * If request was already started, this means we had to @@ -1463,7 +1434,7 @@ static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, out1: /* giveback the request */ - dep->queued_requests--; + dwc3_gadget_giveback(dep, req, -ECONNRESET); out0: @@ -1878,14 +1849,14 @@ static int __dwc3_gadget_start(struct dwc3 *dwc) dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); dep = dwc->eps[0]; - ret = __dwc3_gadget_ep_enable(dep, false, false); + ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); if (ret) { dev_err(dwc->dev, "failed to enable %s\n", dep->name); goto err0; } dep = dwc->eps[1]; - ret = __dwc3_gadget_ep_enable(dep, false, false); + ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); if (ret) { dev_err(dwc->dev, "failed to enable %s\n", dep->name); goto err1; @@ -2082,113 +2053,142 @@ static const struct usb_gadget_ops dwc3_gadget_ops = { /* -------------------------------------------------------------------------- */ -static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) +static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) { - struct dwc3_ep *dep; - u8 epnum; + struct dwc3 *dwc = dep->dwc; - INIT_LIST_HEAD(&dwc->gadget.ep_list); + usb_ep_set_maxpacket_limit(&dep->endpoint, 512); + dep->endpoint.maxburst = 1; + dep->endpoint.ops = &dwc3_gadget_ep0_ops; + if (!dep->direction) + dwc->gadget.ep0 = &dep->endpoint; - for (epnum = 0; epnum < total; epnum++) { - bool direction = epnum & 1; - u8 num = epnum >> 1; + dep->endpoint.caps.type_control = true; - dep = kzalloc(sizeof(*dep), GFP_KERNEL); - if (!dep) - return -ENOMEM; + return 0; +} - dep->dwc = dwc; - dep->number = epnum; - dep->direction = direction; - dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); - dwc->eps[epnum] = dep; +static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) +{ + struct dwc3 *dwc = dep->dwc; + int mdwidth; + int kbytes; + int size; - snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, - direction ? "in" : "out"); + mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0); + /* MDWIDTH is represented in bits, we need it in bytes */ + mdwidth /= 8; - dep->endpoint.name = dep->name; + size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); + if (dwc3_is_usb31(dwc)) + size = DWC31_GTXFIFOSIZ_TXFDEF(size); + else + size = DWC3_GTXFIFOSIZ_TXFDEF(size); - if (!(dep->number > 1)) { - dep->endpoint.desc = &dwc3_gadget_ep0_desc; - dep->endpoint.comp_desc = NULL; - } + /* FIFO Depth is in MDWDITH bytes. Multiply */ + size *= mdwidth; - spin_lock_init(&dep->lock); - - if (num == 0) { - usb_ep_set_maxpacket_limit(&dep->endpoint, 512); - dep->endpoint.maxburst = 1; - dep->endpoint.ops = &dwc3_gadget_ep0_ops; - if (!direction) - dwc->gadget.ep0 = &dep->endpoint; - } else if (direction) { - int mdwidth; - int kbytes; - int size; - int ret; - - mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0); - /* MDWIDTH is represented in bits, we need it in bytes */ - mdwidth /= 8; - - size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num)); - if (dwc3_is_usb31(dwc)) - size = DWC31_GTXFIFOSIZ_TXFDEF(size); - else - size = DWC3_GTXFIFOSIZ_TXFDEF(size); + kbytes = size / 1024; + if (kbytes == 0) + kbytes = 1; - /* FIFO Depth is in MDWDITH bytes. Multiply */ - size *= mdwidth; + /* + * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for + * internal overhead. We don't really know how these are used, + * but documentation say it exists. + */ + size -= mdwidth * (kbytes + 1); + size /= kbytes; - kbytes = size / 1024; - if (kbytes == 0) - kbytes = 1; + usb_ep_set_maxpacket_limit(&dep->endpoint, size); - /* - * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for - * internal overhead. We don't really know how these are used, - * but documentation say it exists. - */ - size -= mdwidth * (kbytes + 1); - size /= kbytes; + dep->endpoint.max_streams = 15; + dep->endpoint.ops = &dwc3_gadget_ep_ops; + list_add_tail(&dep->endpoint.ep_list, + &dwc->gadget.ep_list); + dep->endpoint.caps.type_iso = true; + dep->endpoint.caps.type_bulk = true; + dep->endpoint.caps.type_int = true; - usb_ep_set_maxpacket_limit(&dep->endpoint, size); + return dwc3_alloc_trb_pool(dep); +} - dep->endpoint.max_streams = 15; - dep->endpoint.ops = &dwc3_gadget_ep_ops; - list_add_tail(&dep->endpoint.ep_list, - &dwc->gadget.ep_list); +static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) +{ + struct dwc3 *dwc = dep->dwc; - ret = dwc3_alloc_trb_pool(dep); - if (ret) - return ret; - } else { - int ret; + usb_ep_set_maxpacket_limit(&dep->endpoint, 1024); + dep->endpoint.max_streams = 15; + dep->endpoint.ops = &dwc3_gadget_ep_ops; + list_add_tail(&dep->endpoint.ep_list, + &dwc->gadget.ep_list); + dep->endpoint.caps.type_iso = true; + dep->endpoint.caps.type_bulk = true; + dep->endpoint.caps.type_int = true; - usb_ep_set_maxpacket_limit(&dep->endpoint, 1024); - dep->endpoint.max_streams = 15; - dep->endpoint.ops = &dwc3_gadget_ep_ops; - list_add_tail(&dep->endpoint.ep_list, - &dwc->gadget.ep_list); + return dwc3_alloc_trb_pool(dep); +} - ret = dwc3_alloc_trb_pool(dep); - if (ret) - return ret; - } +static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum) +{ + struct dwc3_ep *dep; + bool direction = epnum & 1; + int ret; + u8 num = epnum >> 1; - if (num == 0) { - dep->endpoint.caps.type_control = true; - } else { - dep->endpoint.caps.type_iso = true; - dep->endpoint.caps.type_bulk = true; - dep->endpoint.caps.type_int = true; - } + dep = kzalloc(sizeof(*dep), GFP_KERNEL); + if (!dep) + return -ENOMEM; - dep->endpoint.caps.dir_in = direction; - dep->endpoint.caps.dir_out = !direction; + dep->dwc = dwc; + dep->number = epnum; + dep->direction = direction; + dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); + dwc->eps[epnum] = dep; - INIT_LIST_HEAD(&dep->pending_list); - INIT_LIST_HEAD(&dep->started_list); + snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, + direction ? "in" : "out"); + + dep->endpoint.name = dep->name; + + if (!(dep->number > 1)) { + dep->endpoint.desc = &dwc3_gadget_ep0_desc; + dep->endpoint.comp_desc = NULL; + } + + spin_lock_init(&dep->lock); + + if (num == 0) + ret = dwc3_gadget_init_control_endpoint(dep); + else if (direction) + ret = dwc3_gadget_init_in_endpoint(dep); + else + ret = dwc3_gadget_init_out_endpoint(dep); + + if (ret) + return ret; + + dep->endpoint.caps.dir_in = direction; + dep->endpoint.caps.dir_out = !direction; + + INIT_LIST_HEAD(&dep->pending_list); + INIT_LIST_HEAD(&dep->started_list); + + return 0; +} + +static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) +{ + u8 epnum; + + INIT_LIST_HEAD(&dwc->gadget.ep_list); + + for (epnum = 0; epnum < total; epnum++) { + int ret; + + ret = dwc3_gadget_init_endpoint(dwc, epnum); + if (ret) + return ret; } return 0; @@ -2223,20 +2223,14 @@ static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) /* -------------------------------------------------------------------------- */ -static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep, +static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, struct dwc3_request *req, struct dwc3_trb *trb, - const struct dwc3_event_depevt *event, int status, - int chain) + const struct dwc3_event_depevt *event, int status, int chain) { unsigned int count; - unsigned int s_pkt = 0; - unsigned int trb_status; dwc3_ep_inc_deq(dep); - if (req->trb == trb) - dep->queued_requests--; - trace_dwc3_complete_trb(dep, trb); /* @@ -2268,159 +2262,140 @@ static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep, if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) return 1; - if (dep->direction) { - if (count) { - trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size); - if (trb_status == DWC3_TRBSTS_MISSED_ISOC) { - /* - * If missed isoc occurred and there is - * no request queued then issue END - * TRANSFER, so that core generates - * next xfernotready and we will issue - * a fresh START TRANSFER. - * If there are still queued request - * then wait, do not issue either END - * or UPDATE TRANSFER, just attach next - * request in pending_list during - * giveback.If any future queued request - * is successfully transferred then we - * will issue UPDATE TRANSFER for all - * request in the pending_list. - */ - dep->flags |= DWC3_EP_MISSED_ISOC; - } else { - dev_err(dwc->dev, "incomplete IN transfer %s\n", - dep->name); - status = -ECONNRESET; - } - } else { - dep->flags &= ~DWC3_EP_MISSED_ISOC; - } - } else { - if (count && (event->status & DEPEVT_STATUS_SHORT)) - s_pkt = 1; - } - - if (s_pkt && !chain) + if (event->status & DEPEVT_STATUS_SHORT && !chain) return 1; - if ((event->status & DEPEVT_STATUS_IOC) && - (trb->ctrl & DWC3_TRB_CTRL_IOC)) + if (event->status & DEPEVT_STATUS_IOC) return 1; return 0; } -static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep, - const struct dwc3_event_depevt *event, int status) +static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, + struct dwc3_request *req, const struct dwc3_event_depevt *event, + int status) { - struct dwc3_request *req, *n; - struct dwc3_trb *trb; - bool ioc = false; - int ret = 0; + struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; + struct scatterlist *sg = req->sg; + struct scatterlist *s; + unsigned int pending = req->num_pending_sgs; + unsigned int i; + int ret = 0; - list_for_each_entry_safe(req, n, &dep->started_list, list) { - unsigned length; - int chain; + for_each_sg(sg, s, pending, i) { + trb = &dep->trb_pool[dep->trb_dequeue]; - length = req->request.length; - chain = req->num_pending_sgs > 0; - if (chain) { - struct scatterlist *sg = req->sg; - struct scatterlist *s; - unsigned int pending = req->num_pending_sgs; - unsigned int i; - - for_each_sg(sg, s, pending, i) { - trb = &dep->trb_pool[dep->trb_dequeue]; - - if (trb->ctrl & DWC3_TRB_CTRL_HWO) - break; - - req->sg = sg_next(s); - req->num_pending_sgs--; - - ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb, - event, status, chain); - if (ret) - break; - } - } else { - trb = &dep->trb_pool[dep->trb_dequeue]; - ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb, - event, status, chain); - } + if (trb->ctrl & DWC3_TRB_CTRL_HWO) + break; - if (req->unaligned || req->zero) { - trb = &dep->trb_pool[dep->trb_dequeue]; - ret = __dwc3_cleanup_done_trbs(dwc, dep, req, trb, - event, status, false); - req->unaligned = false; - req->zero = false; - } + req->sg = sg_next(s); + req->num_pending_sgs--; - req->request.actual = length - req->remaining; + ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, + trb, event, status, true); + if (ret) + break; + } + + return ret; +} - if ((req->request.actual < length) && req->num_pending_sgs) - return __dwc3_gadget_kick_transfer(dep); +static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, + struct dwc3_request *req, const struct dwc3_event_depevt *event, + int status) +{ + struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; - dwc3_gadget_giveback(dep, req, status); + return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, + event, status, false); +} - if (ret) { - if ((event->status & DEPEVT_STATUS_IOC) && - (trb->ctrl & DWC3_TRB_CTRL_IOC)) - ioc = true; - break; - } +static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req) +{ + return req->request.actual == req->request.length; +} + +static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, + const struct dwc3_event_depevt *event, + struct dwc3_request *req, int status) +{ + int ret; + + if (req->num_pending_sgs) + ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, + status); + else + ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, + status); + + if (req->unaligned || req->zero) { + ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, + status); + req->unaligned = false; + req->zero = false; } - /* - * Our endpoint might get disabled by another thread during - * dwc3_gadget_giveback(). If that happens, we're just gonna return 1 - * early on so DWC3_EP_BUSY flag gets cleared - */ - if (!dep->endpoint.desc) - return 1; + req->request.actual = req->request.length - req->remaining; - if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && - list_empty(&dep->started_list)) { - if (list_empty(&dep->pending_list)) { - /* - * If there is no entry in request list then do - * not issue END TRANSFER now. Just set PENDING - * flag, so that END TRANSFER is issued when an - * entry is added into request list. - */ - dep->flags = DWC3_EP_PENDING_REQUEST; - } else { - dwc3_stop_active_transfer(dwc, dep->number, true); - dep->flags = DWC3_EP_ENABLED; - } - return 1; + if (!dwc3_gadget_ep_request_completed(req) && + req->num_pending_sgs) { + __dwc3_gadget_kick_transfer(dep); + goto out; } - if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && ioc) - return 0; + dwc3_gadget_giveback(dep, req, status); + +out: + return ret; +} + +static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, + const struct dwc3_event_depevt *event, int status) +{ + struct dwc3_request *req; + struct dwc3_request *tmp; + + list_for_each_entry_safe(req, tmp, &dep->started_list, list) { + int ret; + + ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, + req, status); + if (ret) + break; + } +} - return 1; +static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, + const struct dwc3_event_depevt *event) +{ + dep->frame_number = event->parameters; } -static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc, - struct dwc3_ep *dep, const struct dwc3_event_depevt *event) +static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, + const struct dwc3_event_depevt *event) { + struct dwc3 *dwc = dep->dwc; unsigned status = 0; - int clean_busy; - u32 is_xfer_complete; + bool stop = false; - is_xfer_complete = (event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE); + dwc3_gadget_endpoint_frame_from_event(dep, event); if (event->status & DEPEVT_STATUS_BUSERR) status = -ECONNRESET; - clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status); - if (clean_busy && (!dep->endpoint.desc || is_xfer_complete || - usb_endpoint_xfer_isoc(dep->endpoint.desc))) - dep->flags &= ~DWC3_EP_BUSY; + if (event->status & DEPEVT_STATUS_MISSED_ISOC) { + status = -EXDEV; + + if (list_empty(&dep->started_list)) + stop = true; + } + + dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); + + if (stop) { + dwc3_stop_active_transfer(dep, true); + dep->flags = DWC3_EP_ENABLED; + } /* * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. @@ -2446,17 +2421,13 @@ static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc, dwc->u1u2 = 0; } +} - /* - * Our endpoint might get disabled by another thread during - * dwc3_gadget_giveback(). If that happens, we're just gonna return 1 - * early on so DWC3_EP_BUSY flag gets cleared - */ - if (!dep->endpoint.desc) - return; - - if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) - __dwc3_gadget_kick_transfer(dep); +static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, + const struct dwc3_event_depevt *event) +{ + dwc3_gadget_endpoint_frame_from_event(dep, event); + __dwc3_gadget_start_isoc(dep); } static void dwc3_endpoint_interrupt(struct dwc3 *dwc, @@ -2483,32 +2454,11 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc, } switch (event->endpoint_event) { - case DWC3_DEPEVT_XFERCOMPLETE: - dep->resource_index = 0; - - if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { - dev_err(dwc->dev, "XferComplete for Isochronous endpoint\n"); - return; - } - - dwc3_endpoint_transfer_complete(dwc, dep, event); - break; case DWC3_DEPEVT_XFERINPROGRESS: - dwc3_endpoint_transfer_complete(dwc, dep, event); + dwc3_gadget_endpoint_transfer_in_progress(dep, event); break; case DWC3_DEPEVT_XFERNOTREADY: - if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) - dwc3_gadget_start_isoc(dwc, dep, event); - else - __dwc3_gadget_kick_transfer(dep); - - break; - case DWC3_DEPEVT_STREAMEVT: - if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) { - dev_err(dwc->dev, "Stream event for non-Bulk %s\n", - dep->name); - return; - } + dwc3_gadget_endpoint_transfer_not_ready(dep, event); break; case DWC3_DEPEVT_EPCMDCMPLT: cmd = DEPEVT_PARAMETER_CMD(event->parameters); @@ -2518,6 +2468,8 @@ static void dwc3_endpoint_interrupt(struct dwc3 *dwc, wake_up(&dep->wait_end_transfer); } break; + case DWC3_DEPEVT_STREAMEVT: + case DWC3_DEPEVT_XFERCOMPLETE: case DWC3_DEPEVT_RXTXFIFOEVT: break; } @@ -2562,15 +2514,13 @@ static void dwc3_reset_gadget(struct dwc3 *dwc) } } -static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force) +static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force) { - struct dwc3_ep *dep; + struct dwc3 *dwc = dep->dwc; struct dwc3_gadget_ep_cmd_params params; u32 cmd; int ret; - dep = dwc->eps[epnum]; - if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || !dep->resource_index) return; @@ -2614,7 +2564,6 @@ static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force) ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); WARN_ON_ONCE(ret); dep->resource_index = 0; - dep->flags &= ~DWC3_EP_BUSY; if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) { dep->flags |= DWC3_EP_END_TRANSFER_PENDING; @@ -2816,14 +2765,14 @@ static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) } dep = dwc->eps[0]; - ret = __dwc3_gadget_ep_enable(dep, true, false); + ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); if (ret) { dev_err(dwc->dev, "failed to enable %s\n", dep->name); return; } dep = dwc->eps[1]; - ret = __dwc3_gadget_ep_enable(dep, true, false); + ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); if (ret) { dev_err(dwc->dev, "failed to enable %s\n", dep->name); return; |