diff options
Diffstat (limited to '')
-rw-r--r-- | drivers/s390/cio/qdio_main.c | 1263 |
1 files changed, 398 insertions, 865 deletions
diff --git a/drivers/s390/cio/qdio_main.c b/drivers/s390/cio/qdio_main.c index 3475317c42e5..9cde55730b65 100644 --- a/drivers/s390/cio/qdio_main.c +++ b/drivers/s390/cio/qdio_main.c @@ -10,7 +10,7 @@ #include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h> -#include <linux/timer.h> +#include <linux/kmemleak.h> #include <linux/delay.h> #include <linux/gfp.h> #include <linux/io.h> @@ -31,38 +31,41 @@ MODULE_DESCRIPTION("QDIO base support"); MODULE_LICENSE("GPL"); static inline int do_siga_sync(unsigned long schid, - unsigned int out_mask, unsigned int in_mask, + unsigned long out_mask, unsigned long in_mask, unsigned int fc) { - register unsigned long __fc asm ("0") = fc; - register unsigned long __schid asm ("1") = schid; - register unsigned long out asm ("2") = out_mask; - register unsigned long in asm ("3") = in_mask; int cc; asm volatile( + " lgr 0,%[fc]\n" + " lgr 1,%[schid]\n" + " lgr 2,%[out]\n" + " lgr 3,%[in]\n" " siga 0\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (cc) - : "d" (__fc), "d" (__schid), "d" (out), "d" (in) : "cc"); + " ipm %[cc]\n" + " srl %[cc],28\n" + : [cc] "=&d" (cc) + : [fc] "d" (fc), [schid] "d" (schid), + [out] "d" (out_mask), [in] "d" (in_mask) + : "cc", "0", "1", "2", "3"); return cc; } -static inline int do_siga_input(unsigned long schid, unsigned int mask, - unsigned int fc) +static inline int do_siga_input(unsigned long schid, unsigned long mask, + unsigned long fc) { - register unsigned long __fc asm ("0") = fc; - register unsigned long __schid asm ("1") = schid; - register unsigned long __mask asm ("2") = mask; int cc; asm volatile( + " lgr 0,%[fc]\n" + " lgr 1,%[schid]\n" + " lgr 2,%[mask]\n" " siga 0\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (cc) - : "d" (__fc), "d" (__schid), "d" (__mask) : "cc"); + " ipm %[cc]\n" + " srl %[cc],28\n" + : [cc] "=&d" (cc) + : [fc] "d" (fc), [schid] "d" (schid), [mask] "d" (mask) + : "cc", "0", "1", "2"); return cc; } @@ -78,23 +81,24 @@ static inline int do_siga_input(unsigned long schid, unsigned int mask, * Note: For IQDC unicast queues only the highest priority queue is processed. */ static inline int do_siga_output(unsigned long schid, unsigned long mask, - unsigned int *bb, unsigned int fc, + unsigned int *bb, unsigned long fc, unsigned long aob) { - register unsigned long __fc asm("0") = fc; - register unsigned long __schid asm("1") = schid; - register unsigned long __mask asm("2") = mask; - register unsigned long __aob asm("3") = aob; int cc; asm volatile( + " lgr 0,%[fc]\n" + " lgr 1,%[schid]\n" + " lgr 2,%[mask]\n" + " lgr 3,%[aob]\n" " siga 0\n" - " ipm %0\n" - " srl %0,28\n" - : "=d" (cc), "+d" (__fc), "+d" (__aob) - : "d" (__schid), "d" (__mask) - : "cc"); - *bb = __fc >> 31; + " lgr %[fc],0\n" + " ipm %[cc]\n" + " srl %[cc],28\n" + : [cc] "=&d" (cc), [fc] "+&d" (fc) + : [schid] "d" (schid), [mask] "d" (mask), [aob] "d" (aob) + : "cc", "0", "1", "2", "3"); + *bb = fc >> 31; return cc; } @@ -143,7 +147,7 @@ again: DBF_ERROR("%4x EQBS ERROR", SCH_NO(q)); DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr, - q->first_to_kick, count, q->irq_ptr->int_parm); + q->first_to_check, count, q->irq_ptr->int_parm); return 0; } } @@ -166,8 +170,6 @@ static int qdio_do_sqbs(struct qdio_q *q, unsigned char state, int start, int tmp_count = count, tmp_start = start; int nr = q->nr; - if (!count) - return 0; qperf_inc(q, sqbs); if (!q->is_input_q) @@ -191,7 +193,7 @@ again: DBF_ERROR("%4x SQBS ERROR", SCH_NO(q)); DBF_ERROR("%3d%3d%2d", count, tmp_count, nr); q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr, - q->first_to_kick, count, q->irq_ptr->int_parm); + q->first_to_check, count, q->irq_ptr->int_parm); return 0; } } @@ -202,7 +204,7 @@ again: */ static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr, unsigned char *state, unsigned int count, - int auto_ack, int merge_pending) + int auto_ack) { unsigned char __state = 0; int i = 1; @@ -217,18 +219,9 @@ static inline int get_buf_states(struct qdio_q *q, unsigned int bufnr, if (__state & SLSB_OWNER_CU) goto out; - if (merge_pending && __state == SLSB_P_OUTPUT_PENDING) - __state = SLSB_P_OUTPUT_EMPTY; - for (; i < count; i++) { bufnr = next_buf(bufnr); - /* merge PENDING into EMPTY: */ - if (merge_pending && - q->slsb.val[bufnr] == SLSB_P_OUTPUT_PENDING && - __state == SLSB_P_OUTPUT_EMPTY) - continue; - /* stop if next state differs from initial state: */ if (q->slsb.val[bufnr] != __state) break; @@ -242,7 +235,7 @@ out: static inline int get_buf_state(struct qdio_q *q, unsigned int bufnr, unsigned char *state, int auto_ack) { - return get_buf_states(q, bufnr, state, 1, auto_ack, 0); + return get_buf_states(q, bufnr, state, 1, auto_ack); } /* wrap-around safe setting of slsb states, returns number of changed buffers */ @@ -254,10 +247,17 @@ static inline int set_buf_states(struct qdio_q *q, int bufnr, if (is_qebsm(q)) return qdio_do_sqbs(q, state, bufnr, count); + /* Ensure that all preceding changes to the SBALs are visible: */ + mb(); + for (i = 0; i < count; i++) { - xchg(&q->slsb.val[bufnr], state); + WRITE_ONCE(q->slsb.val[bufnr], state); bufnr = next_buf(bufnr); } + + /* Make our SLSB changes visible: */ + mb(); + return count; } @@ -302,12 +302,22 @@ static inline int qdio_siga_sync(struct qdio_q *q, unsigned int output, return (cc) ? -EIO : 0; } +static inline int qdio_sync_input_queue(struct qdio_q *q) +{ + return qdio_siga_sync(q, 0, q->mask); +} + +static inline int qdio_sync_output_queue(struct qdio_q *q) +{ + return qdio_siga_sync(q, q->mask, 0); +} + static inline int qdio_siga_sync_q(struct qdio_q *q) { if (q->is_input_q) - return qdio_siga_sync(q, 0, q->mask); + return qdio_sync_input_queue(q); else - return qdio_siga_sync(q, q->mask, 0); + return qdio_sync_output_queue(q); } static int qdio_siga_output(struct qdio_q *q, unsigned int count, @@ -371,57 +381,36 @@ static inline int qdio_siga_input(struct qdio_q *q) return (cc) ? -EIO : 0; } -#define qdio_siga_sync_out(q) qdio_siga_sync(q, ~0U, 0) -#define qdio_siga_sync_all(q) qdio_siga_sync(q, ~0U, ~0U) - -static inline void qdio_sync_queues(struct qdio_q *q) -{ - /* PCI capable outbound queues will also be scanned so sync them too */ - if (pci_out_supported(q->irq_ptr)) - qdio_siga_sync_all(q); - else - qdio_siga_sync_q(q); -} - int debug_get_buf_state(struct qdio_q *q, unsigned int bufnr, unsigned char *state) { - if (need_siga_sync(q)) + if (qdio_need_siga_sync(q->irq_ptr)) qdio_siga_sync_q(q); return get_buf_state(q, bufnr, state, 0); } static inline void qdio_stop_polling(struct qdio_q *q) { - if (!q->u.in.ack_count) + if (!q->u.in.batch_count) return; qperf_inc(q, stop_polling); /* show the card that we are not polling anymore */ - set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT, - q->u.in.ack_count); - q->u.in.ack_count = 0; + set_buf_states(q, q->u.in.batch_start, SLSB_P_INPUT_NOT_INIT, + q->u.in.batch_count); + q->u.in.batch_count = 0; } static inline void account_sbals(struct qdio_q *q, unsigned int count) { - int pos; - q->q_stats.nr_sbal_total += count; - if (count == QDIO_MAX_BUFFERS_MASK) { - q->q_stats.nr_sbals[7]++; - return; - } - pos = ilog2(count); - q->q_stats.nr_sbals[pos]++; + q->q_stats.nr_sbals[ilog2(count)]++; } static void process_buffer_error(struct qdio_q *q, unsigned int start, int count) { - q->qdio_error = QDIO_ERROR_SLSB_STATE; - /* special handling for no target buffer empty */ if (queue_type(q) == QDIO_IQDIO_QFMT && !q->is_input_q && q->sbal[start]->element[15].sflags == 0x10) { @@ -438,119 +427,101 @@ static void process_buffer_error(struct qdio_q *q, unsigned int start, q->sbal[start]->element[15].sflags); } -static inline void inbound_primed(struct qdio_q *q, unsigned int start, - int count) +static inline void inbound_handle_work(struct qdio_q *q, unsigned int start, + int count, bool auto_ack) { - int new; - - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, count); + /* ACK the newest SBAL: */ + if (!auto_ack) + set_buf_state(q, add_buf(start, count - 1), SLSB_P_INPUT_ACK); - /* for QEBSM the ACK was already set by EQBS */ - if (is_qebsm(q)) { - if (!q->u.in.ack_count) { - q->u.in.ack_count = count; - q->u.in.ack_start = start; - return; - } - - /* delete the previous ACK's */ - set_buf_states(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT, - q->u.in.ack_count); - q->u.in.ack_count = count; - q->u.in.ack_start = start; - return; - } - - /* - * ACK the newest buffer. The ACK will be removed in qdio_stop_polling - * or by the next inbound run. - */ - new = add_buf(start, count - 1); - if (q->u.in.ack_count) { - /* reset the previous ACK but first set the new one */ - set_buf_state(q, new, SLSB_P_INPUT_ACK); - set_buf_state(q, q->u.in.ack_start, SLSB_P_INPUT_NOT_INIT); - } else { - q->u.in.ack_count = 1; - set_buf_state(q, new, SLSB_P_INPUT_ACK); - } - - q->u.in.ack_start = new; - count--; - if (!count) - return; - /* need to change ALL buffers to get more interrupts */ - set_buf_states(q, start, SLSB_P_INPUT_NOT_INIT, count); + if (!q->u.in.batch_count) + q->u.in.batch_start = start; + q->u.in.batch_count += count; } -static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start) +static int get_inbound_buffer_frontier(struct qdio_q *q, unsigned int start, + unsigned int *error) { unsigned char state = 0; int count; q->timestamp = get_tod_clock_fast(); - /* - * Don't check 128 buffers, as otherwise qdio_inbound_q_moved - * would return 0. - */ - count = min(atomic_read(&q->nr_buf_used), QDIO_MAX_BUFFERS_MASK); + count = atomic_read(&q->nr_buf_used); if (!count) return 0; - /* - * No siga sync here, as a PCI or we after a thin interrupt - * already sync'ed the queues. - */ - count = get_buf_states(q, start, &state, count, 1, 0); + if (qdio_need_siga_sync(q->irq_ptr)) + qdio_sync_input_queue(q); + + count = get_buf_states(q, start, &state, count, 1); if (!count) return 0; switch (state) { case SLSB_P_INPUT_PRIMED: - inbound_primed(q, start, count); + DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr, + count); + + inbound_handle_work(q, start, count, is_qebsm(q)); if (atomic_sub_return(count, &q->nr_buf_used) == 0) qperf_inc(q, inbound_queue_full); if (q->irq_ptr->perf_stat_enabled) account_sbals(q, count); return count; case SLSB_P_INPUT_ERROR: + DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr, + count); + + *error = QDIO_ERROR_SLSB_STATE; process_buffer_error(q, start, count); - /* - * Interrupts may be avoided as long as the error is present - * so change the buffer state immediately to avoid starvation. - */ - set_buf_states(q, start, SLSB_P_INPUT_NOT_INIT, count); + inbound_handle_work(q, start, count, false); if (atomic_sub_return(count, &q->nr_buf_used) == 0) qperf_inc(q, inbound_queue_full); if (q->irq_ptr->perf_stat_enabled) account_sbals_error(q, count); return count; case SLSB_CU_INPUT_EMPTY: - case SLSB_P_INPUT_NOT_INIT: - case SLSB_P_INPUT_ACK: if (q->irq_ptr->perf_stat_enabled) q->q_stats.nr_sbal_nop++; DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x", q->nr, start); return 0; + case SLSB_P_INPUT_NOT_INIT: + case SLSB_P_INPUT_ACK: + /* We should never see this state, throw a WARN: */ default: - WARN_ON_ONCE(1); + dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, + "found state %#x at index %u on queue %u\n", + state, start, q->nr); return 0; } } -static int qdio_inbound_q_moved(struct qdio_q *q, unsigned int start) +int qdio_inspect_input_queue(struct ccw_device *cdev, unsigned int nr, + unsigned int *bufnr, unsigned int *error) { + struct qdio_irq *irq = cdev->private->qdio_data; + unsigned int start; + struct qdio_q *q; int count; - count = get_inbound_buffer_frontier(q, start); + if (!irq) + return -ENODEV; - if (count && !is_thinint_irq(q->irq_ptr) && MACHINE_IS_LPAR) - q->u.in.timestamp = get_tod_clock(); + q = irq->input_qs[nr]; + start = q->first_to_check; + *error = 0; + count = get_inbound_buffer_frontier(q, start, error); + if (count == 0) + return 0; + + *bufnr = start; + q->first_to_check = add_buf(start, count); return count; } +EXPORT_SYMBOL_GPL(qdio_inspect_input_queue); static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) { @@ -559,168 +530,41 @@ static inline int qdio_inbound_q_done(struct qdio_q *q, unsigned int start) if (!atomic_read(&q->nr_buf_used)) return 1; - if (need_siga_sync(q)) - qdio_siga_sync_q(q); + if (qdio_need_siga_sync(q->irq_ptr)) + qdio_sync_input_queue(q); get_buf_state(q, start, &state, 0); if (state == SLSB_P_INPUT_PRIMED || state == SLSB_P_INPUT_ERROR) /* more work coming */ return 0; - if (is_thinint_irq(q->irq_ptr)) - return 1; - - /* don't poll under z/VM */ - if (MACHINE_IS_VM) - return 1; - - /* - * At this point we know, that inbound first_to_check - * has (probably) not moved (see qdio_inbound_processing). - */ - if (get_tod_clock_fast() > q->u.in.timestamp + QDIO_INPUT_THRESHOLD) { - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in done:%02x", start); - return 1; - } else - return 0; + return 1; } -static inline void qdio_handle_aobs(struct qdio_q *q, int start, int count) -{ - unsigned char state = 0; - int j, b = start; - - for (j = 0; j < count; ++j) { - get_buf_state(q, b, &state, 0); - if (state == SLSB_P_OUTPUT_PENDING) { - struct qaob *aob = q->u.out.aobs[b]; - if (aob == NULL) - continue; - - q->u.out.sbal_state[b].flags |= - QDIO_OUTBUF_STATE_FLAG_PENDING; - q->u.out.aobs[b] = NULL; - } - b = next_buf(b); - } -} - -static inline unsigned long qdio_aob_for_buffer(struct qdio_output_q *q, - int bufnr) -{ - unsigned long phys_aob = 0; - - if (!q->aobs[bufnr]) { - struct qaob *aob = qdio_allocate_aob(); - q->aobs[bufnr] = aob; - } - if (q->aobs[bufnr]) { - q->aobs[bufnr]->user1 = (u64) q->sbal_state[bufnr].user; - phys_aob = virt_to_phys(q->aobs[bufnr]); - WARN_ON_ONCE(phys_aob & 0xFF); - } - - q->sbal_state[bufnr].flags = 0; - return phys_aob; -} - -static void qdio_kick_handler(struct qdio_q *q, unsigned int count) -{ - int start = q->first_to_kick; - - if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) - return; - - if (q->is_input_q) { - qperf_inc(q, inbound_handler); - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "kih s:%02x c:%02x", start, count); - } else { - qperf_inc(q, outbound_handler); - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "koh: s:%02x c:%02x", - start, count); - } - - q->handler(q->irq_ptr->cdev, q->qdio_error, q->nr, start, count, - q->irq_ptr->int_parm); - - /* for the next time */ - q->first_to_kick = add_buf(start, count); - q->qdio_error = 0; -} - -static inline int qdio_tasklet_schedule(struct qdio_q *q) -{ - if (likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) { - tasklet_schedule(&q->tasklet); - return 0; - } - return -EPERM; -} - -static void __qdio_inbound_processing(struct qdio_q *q) -{ - unsigned int start = q->first_to_check; - int count; - - qperf_inc(q, tasklet_inbound); - - count = qdio_inbound_q_moved(q, start); - if (count == 0) - return; - - start = add_buf(start, count); - q->first_to_check = start; - qdio_kick_handler(q, count); - - if (!qdio_inbound_q_done(q, start)) { - /* means poll time is not yet over */ - qperf_inc(q, tasklet_inbound_resched); - if (!qdio_tasklet_schedule(q)) - return; - } - - qdio_stop_polling(q); - /* - * We need to check again to not lose initiative after - * resetting the ACK state. - */ - if (!qdio_inbound_q_done(q, start)) { - qperf_inc(q, tasklet_inbound_resched2); - qdio_tasklet_schedule(q); - } -} - -void qdio_inbound_processing(unsigned long data) -{ - struct qdio_q *q = (struct qdio_q *)data; - __qdio_inbound_processing(q); -} - -static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start) +static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start, + unsigned int *error) { unsigned char state = 0; int count; q->timestamp = get_tod_clock_fast(); - if (need_siga_sync(q)) - if (((queue_type(q) != QDIO_IQDIO_QFMT) && - !pci_out_supported(q->irq_ptr)) || - (queue_type(q) == QDIO_IQDIO_QFMT && - multicast_outbound(q))) - qdio_siga_sync_q(q); - count = atomic_read(&q->nr_buf_used); if (!count) return 0; - count = get_buf_states(q, start, &state, count, 0, q->u.out.use_cq); + if (qdio_need_siga_sync(q->irq_ptr)) + qdio_sync_output_queue(q); + + count = get_buf_states(q, start, &state, count, 0); if (!count) return 0; switch (state) { - case SLSB_P_OUTPUT_EMPTY: case SLSB_P_OUTPUT_PENDING: + *error = QDIO_ERROR_SLSB_PENDING; + fallthrough; + case SLSB_P_OUTPUT_EMPTY: /* the adapter got it */ DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out empty:%1d %02x", q->nr, count); @@ -730,6 +574,10 @@ static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start) account_sbals(q, count); return count; case SLSB_P_OUTPUT_ERROR: + DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x", + q->nr, count); + + *error = QDIO_ERROR_SLSB_STATE; process_buffer_error(q, start, count); atomic_sub(count, &q->nr_buf_used); if (q->irq_ptr->perf_stat_enabled) @@ -742,35 +590,42 @@ static int get_outbound_buffer_frontier(struct qdio_q *q, unsigned int start) DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d", q->nr); return 0; - case SLSB_P_OUTPUT_NOT_INIT: case SLSB_P_OUTPUT_HALTED: return 0; + case SLSB_P_OUTPUT_NOT_INIT: + /* We should never see this state, throw a WARN: */ default: - WARN_ON_ONCE(1); + dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1, + "found state %#x at index %u on queue %u\n", + state, start, q->nr); return 0; } } -/* all buffers processed? */ -static inline int qdio_outbound_q_done(struct qdio_q *q) -{ - return atomic_read(&q->nr_buf_used) == 0; -} - -static inline int qdio_outbound_q_moved(struct qdio_q *q, unsigned int start) +int qdio_inspect_output_queue(struct ccw_device *cdev, unsigned int nr, + unsigned int *bufnr, unsigned int *error) { + struct qdio_irq *irq = cdev->private->qdio_data; + unsigned int start; + struct qdio_q *q; int count; - count = get_outbound_buffer_frontier(q, start); + if (!irq) + return -ENODEV; - if (count) { - DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out moved:%1d", q->nr); - if (q->u.out.use_cq) - qdio_handle_aobs(q, start, count); - } + q = irq->output_qs[nr]; + start = q->first_to_check; + *error = 0; + count = get_outbound_buffer_frontier(q, start, error); + if (count == 0) + return 0; + + *bufnr = start; + q->first_to_check = add_buf(start, count); return count; } +EXPORT_SYMBOL_GPL(qdio_inspect_output_queue); static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count, unsigned long aob) @@ -778,7 +633,7 @@ static int qdio_kick_outbound_q(struct qdio_q *q, unsigned int count, int retries = 0, cc; unsigned int busy_bit; - if (!need_siga_out(q)) + if (!qdio_need_siga_out(q->irq_ptr)) return 0; DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr); @@ -815,114 +670,6 @@ retry: return cc; } -static void __qdio_outbound_processing(struct qdio_q *q) -{ - unsigned int start = q->first_to_check; - int count; - - qperf_inc(q, tasklet_outbound); - WARN_ON_ONCE(atomic_read(&q->nr_buf_used) < 0); - - count = qdio_outbound_q_moved(q, start); - if (count) { - q->first_to_check = add_buf(start, count); - qdio_kick_handler(q, count); - } - - if (queue_type(q) == QDIO_ZFCP_QFMT && !pci_out_supported(q->irq_ptr) && - !qdio_outbound_q_done(q)) - goto sched; - - if (q->u.out.pci_out_enabled) - return; - - /* - * Now we know that queue type is either qeth without pci enabled - * or HiperSockets. Make sure buffer switch from PRIMED to EMPTY - * is noticed and outbound_handler is called after some time. - */ - if (qdio_outbound_q_done(q)) - del_timer_sync(&q->u.out.timer); - else - if (!timer_pending(&q->u.out.timer) && - likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) - mod_timer(&q->u.out.timer, jiffies + 10 * HZ); - return; - -sched: - qdio_tasklet_schedule(q); -} - -/* outbound tasklet */ -void qdio_outbound_processing(unsigned long data) -{ - struct qdio_q *q = (struct qdio_q *)data; - __qdio_outbound_processing(q); -} - -void qdio_outbound_timer(struct timer_list *t) -{ - struct qdio_q *q = from_timer(q, t, u.out.timer); - - qdio_tasklet_schedule(q); -} - -static inline void qdio_check_outbound_pci_queues(struct qdio_irq *irq) -{ - struct qdio_q *out; - int i; - - if (!pci_out_supported(irq) || !irq->scan_threshold) - return; - - for_each_output_queue(irq, out, i) - if (!qdio_outbound_q_done(out)) - qdio_tasklet_schedule(out); -} - -static void __tiqdio_inbound_processing(struct qdio_q *q) -{ - unsigned int start = q->first_to_check; - int count; - - qperf_inc(q, tasklet_inbound); - if (need_siga_sync(q) && need_siga_sync_after_ai(q)) - qdio_sync_queues(q); - - /* The interrupt could be caused by a PCI request: */ - qdio_check_outbound_pci_queues(q->irq_ptr); - - count = qdio_inbound_q_moved(q, start); - if (count == 0) - return; - - start = add_buf(start, count); - q->first_to_check = start; - qdio_kick_handler(q, count); - - if (!qdio_inbound_q_done(q, start)) { - qperf_inc(q, tasklet_inbound_resched); - if (!qdio_tasklet_schedule(q)) - return; - } - - qdio_stop_polling(q); - /* - * We need to check again to not lose initiative after - * resetting the ACK state. - */ - if (!qdio_inbound_q_done(q, start)) { - qperf_inc(q, tasklet_inbound_resched2); - qdio_tasklet_schedule(q); - } -} - -void tiqdio_inbound_processing(unsigned long data) -{ - struct qdio_q *q = (struct qdio_q *)data; - __tiqdio_inbound_processing(q); -} - static inline void qdio_set_state(struct qdio_irq *irq_ptr, enum qdio_irq_states state) { @@ -944,63 +691,29 @@ static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb) /* PCI interrupt handler */ static void qdio_int_handler_pci(struct qdio_irq *irq_ptr) { - int i; - struct qdio_q *q; - if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) return; - for_each_input_queue(irq_ptr, q, i) { - if (q->u.in.queue_start_poll) { - /* skip if polling is enabled or already in work */ - if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, - &q->u.in.queue_irq_state)) { - QDIO_PERF_STAT_INC(irq_ptr, int_discarded); - continue; - } - q->u.in.queue_start_poll(q->irq_ptr->cdev, q->nr, - q->irq_ptr->int_parm); - } else { - tasklet_schedule(&q->tasklet); - } - } - - if (!pci_out_supported(irq_ptr) || !irq_ptr->scan_threshold) - return; - - for_each_output_queue(irq_ptr, q, i) { - if (qdio_outbound_q_done(q)) - continue; - if (need_siga_sync(q) && need_siga_sync_out_after_pci(q)) - qdio_siga_sync_q(q); - qdio_tasklet_schedule(q); - } + qdio_deliver_irq(irq_ptr); + irq_ptr->last_data_irq_time = S390_lowcore.int_clock; } -static void qdio_handle_activate_check(struct ccw_device *cdev, - unsigned long intparm, int cstat, int dstat) +static void qdio_handle_activate_check(struct qdio_irq *irq_ptr, + unsigned long intparm, int cstat, + int dstat) { - struct qdio_irq *irq_ptr = cdev->private->qdio_data; - struct qdio_q *q; - int count; + unsigned int first_to_check = 0; DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no); DBF_ERROR("intp :%lx", intparm); DBF_ERROR("ds: %2x cs:%2x", dstat, cstat); - if (irq_ptr->nr_input_qs) { - q = irq_ptr->input_qs[0]; - } else if (irq_ptr->nr_output_qs) { - q = irq_ptr->output_qs[0]; - } else { - dump_stack(); - goto no_handler; - } + /* zfcp wants this: */ + if (irq_ptr->nr_input_qs) + first_to_check = irq_ptr->input_qs[0]->first_to_check; - count = sub_buf(q->first_to_check, q->first_to_kick); - q->handler(q->irq_ptr->cdev, QDIO_ERROR_ACTIVATE, - q->nr, q->first_to_kick, count, irq_ptr->int_parm); -no_handler: + irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0, + first_to_check, 0, irq_ptr->int_parm); qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); /* * In case of z/VM LGR (Live Guest Migration) QDIO recovery will happen. @@ -1009,11 +722,9 @@ no_handler: lgr_info_log(); } -static void qdio_establish_handle_irq(struct ccw_device *cdev, int cstat, +static void qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat, int dstat) { - struct qdio_irq *irq_ptr = cdev->private->qdio_data; - DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq"); if (cstat) @@ -1060,7 +771,7 @@ void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm, switch (irq_ptr->state) { case QDIO_IRQ_STATE_INACTIVE: - qdio_establish_handle_irq(cdev, cstat, dstat); + qdio_establish_handle_irq(irq_ptr, cstat, dstat); break; case QDIO_IRQ_STATE_CLEANUP: qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); @@ -1072,7 +783,7 @@ void qdio_int_handler(struct ccw_device *cdev, unsigned long intparm, return; } if (cstat || dstat) - qdio_handle_activate_check(cdev, intparm, cstat, + qdio_handle_activate_check(irq_ptr, intparm, cstat, dstat); break; case QDIO_IRQ_STATE_STOPPED: @@ -1105,19 +816,34 @@ int qdio_get_ssqd_desc(struct ccw_device *cdev, } EXPORT_SYMBOL_GPL(qdio_get_ssqd_desc); -static void qdio_shutdown_queues(struct ccw_device *cdev) +static int qdio_cancel_ccw(struct qdio_irq *irq, int how) { - struct qdio_irq *irq_ptr = cdev->private->qdio_data; - struct qdio_q *q; - int i; - - for_each_input_queue(irq_ptr, q, i) - tasklet_kill(&q->tasklet); + struct ccw_device *cdev = irq->cdev; + long timeout; + int rc; - for_each_output_queue(irq_ptr, q, i) { - del_timer_sync(&q->u.out.timer); - tasklet_kill(&q->tasklet); + spin_lock_irq(get_ccwdev_lock(cdev)); + qdio_set_state(irq, QDIO_IRQ_STATE_CLEANUP); + if (how & QDIO_FLAG_CLEANUP_USING_CLEAR) + rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP); + else + /* default behaviour is halt */ + rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP); + spin_unlock_irq(get_ccwdev_lock(cdev)); + if (rc) { + DBF_ERROR("%4x SHUTD ERR", irq->schid.sch_no); + DBF_ERROR("rc:%4d", rc); + return rc; } + + timeout = wait_event_interruptible_timeout(cdev->private->wait_q, + irq->state == QDIO_IRQ_STATE_INACTIVE || + irq->state == QDIO_IRQ_STATE_ERR, + 10 * HZ); + if (timeout <= 0) + rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME; + + return rc; } /** @@ -1149,46 +875,15 @@ int qdio_shutdown(struct ccw_device *cdev, int how) } /* - * Indicate that the device is going down. Scheduling the queue - * tasklets is forbidden from here on. + * Indicate that the device is going down. */ qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED); - tiqdio_remove_device(irq_ptr); - qdio_shutdown_queues(cdev); qdio_shutdown_debug_entries(irq_ptr); - /* cleanup subchannel */ - spin_lock_irq(get_ccwdev_lock(cdev)); - - if (how & QDIO_FLAG_CLEANUP_USING_CLEAR) - rc = ccw_device_clear(cdev, QDIO_DOING_CLEANUP); - else - /* default behaviour is halt */ - rc = ccw_device_halt(cdev, QDIO_DOING_CLEANUP); - if (rc) { - DBF_ERROR("%4x SHUTD ERR", irq_ptr->schid.sch_no); - DBF_ERROR("rc:%4d", rc); - goto no_cleanup; - } - - qdio_set_state(irq_ptr, QDIO_IRQ_STATE_CLEANUP); - spin_unlock_irq(get_ccwdev_lock(cdev)); - wait_event_interruptible_timeout(cdev->private->wait_q, - irq_ptr->state == QDIO_IRQ_STATE_INACTIVE || - irq_ptr->state == QDIO_IRQ_STATE_ERR, - 10 * HZ); - spin_lock_irq(get_ccwdev_lock(cdev)); - -no_cleanup: + rc = qdio_cancel_ccw(irq_ptr, how); qdio_shutdown_thinint(irq_ptr); - - /* restore interrupt handler */ - if ((void *)cdev->handler == (void *)qdio_int_handler) { - cdev->handler = irq_ptr->orig_handler; - cdev->private->intparm = 0; - } - spin_unlock_irq(get_ccwdev_lock(cdev)); + qdio_shutdown_irq(irq_ptr); qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); mutex_unlock(&irq_ptr->setup_mutex); @@ -1219,43 +914,53 @@ int qdio_free(struct ccw_device *cdev) cdev->private->qdio_data = NULL; mutex_unlock(&irq_ptr->setup_mutex); - qdio_release_memory(irq_ptr); + qdio_free_queues(irq_ptr); + free_page((unsigned long) irq_ptr->qdr); + free_page(irq_ptr->chsc_page); + kfree(irq_ptr->ccw); + free_page((unsigned long) irq_ptr); return 0; } EXPORT_SYMBOL_GPL(qdio_free); /** * qdio_allocate - allocate qdio queues and associated data - * @init_data: initialization data + * @cdev: associated ccw device + * @no_input_qs: allocate this number of Input Queues + * @no_output_qs: allocate this number of Output Queues */ -int qdio_allocate(struct qdio_initialize *init_data) +int qdio_allocate(struct ccw_device *cdev, unsigned int no_input_qs, + unsigned int no_output_qs) { struct subchannel_id schid; struct qdio_irq *irq_ptr; + int rc = -ENOMEM; - ccw_device_get_schid(init_data->cdev, &schid); + ccw_device_get_schid(cdev, &schid); DBF_EVENT("qallocate:%4x", schid.sch_no); - if ((init_data->no_input_qs && !init_data->input_handler) || - (init_data->no_output_qs && !init_data->output_handler)) + if (no_input_qs > QDIO_MAX_QUEUES_PER_IRQ || + no_output_qs > QDIO_MAX_QUEUES_PER_IRQ) return -EINVAL; - if ((init_data->no_input_qs > QDIO_MAX_QUEUES_PER_IRQ) || - (init_data->no_output_qs > QDIO_MAX_QUEUES_PER_IRQ)) - return -EINVAL; + irq_ptr = (void *) get_zeroed_page(GFP_KERNEL); + if (!irq_ptr) + return -ENOMEM; - if ((!init_data->input_sbal_addr_array) || - (!init_data->output_sbal_addr_array)) - return -EINVAL; + irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA); + if (!irq_ptr->ccw) + goto err_ccw; - /* irq_ptr must be in GFP_DMA since it contains ccw1.cda */ - irq_ptr = (void *) get_zeroed_page(GFP_KERNEL | GFP_DMA); - if (!irq_ptr) - goto out_err; + /* kmemleak doesn't scan the page-allocated irq_ptr: */ + kmemleak_not_leak(irq_ptr->ccw); + irq_ptr->cdev = cdev; mutex_init(&irq_ptr->setup_mutex); - if (qdio_allocate_dbf(init_data, irq_ptr)) - goto out_rel; + if (qdio_allocate_dbf(irq_ptr)) + goto err_dbf; + + DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs, + no_output_qs); /* * Allocate a page for the chsc calls in qdio_establish. @@ -1265,118 +970,152 @@ int qdio_allocate(struct qdio_initialize *init_data) */ irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL); if (!irq_ptr->chsc_page) - goto out_rel; + goto err_chsc; /* qdr is used in ccw1.cda which is u32 */ irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!irq_ptr->qdr) - goto out_rel; + goto err_qdr; - if (qdio_allocate_qs(irq_ptr, init_data->no_input_qs, - init_data->no_output_qs)) - goto out_rel; + rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs); + if (rc) + goto err_queues; - INIT_LIST_HEAD(&irq_ptr->entry); - init_data->cdev->private->qdio_data = irq_ptr; + cdev->private->qdio_data = irq_ptr; qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); return 0; -out_rel: - qdio_release_memory(irq_ptr); -out_err: - return -ENOMEM; + +err_queues: + free_page((unsigned long) irq_ptr->qdr); +err_qdr: + free_page(irq_ptr->chsc_page); +err_chsc: +err_dbf: + kfree(irq_ptr->ccw); +err_ccw: + free_page((unsigned long) irq_ptr); + return rc; } EXPORT_SYMBOL_GPL(qdio_allocate); -static void qdio_detect_hsicq(struct qdio_irq *irq_ptr) +static void qdio_trace_init_data(struct qdio_irq *irq, + struct qdio_initialize *data) { - struct qdio_q *q = irq_ptr->input_qs[0]; - int i, use_cq = 0; - - if (irq_ptr->nr_input_qs > 1 && queue_type(q) == QDIO_IQDIO_QFMT) - use_cq = 1; - - for_each_output_queue(irq_ptr, q, i) { - if (use_cq) { - if (multicast_outbound(q)) - continue; - if (qdio_enable_async_operation(&q->u.out) < 0) { - use_cq = 0; - continue; - } - } else - qdio_disable_async_operation(&q->u.out); - } - DBF_EVENT("use_cq:%d", use_cq); + DBF_DEV_EVENT(DBF_ERR, irq, "qfmt:%1u", data->q_format); + DBF_DEV_EVENT(DBF_ERR, irq, "qpff%4x", data->qib_param_field_format); + DBF_DEV_HEX(irq, &data->qib_param_field, sizeof(void *), DBF_ERR); + DBF_DEV_EVENT(DBF_ERR, irq, "niq:%1u noq:%1u", data->no_input_qs, + data->no_output_qs); + DBF_DEV_HEX(irq, &data->input_handler, sizeof(void *), DBF_ERR); + DBF_DEV_HEX(irq, &data->output_handler, sizeof(void *), DBF_ERR); + DBF_DEV_HEX(irq, &data->int_parm, sizeof(long), DBF_ERR); + DBF_DEV_HEX(irq, &data->input_sbal_addr_array, sizeof(void *), DBF_ERR); + DBF_DEV_HEX(irq, &data->output_sbal_addr_array, sizeof(void *), + DBF_ERR); } /** * qdio_establish - establish queues on a qdio subchannel + * @cdev: associated ccw device * @init_data: initialization data */ -int qdio_establish(struct qdio_initialize *init_data) +int qdio_establish(struct ccw_device *cdev, + struct qdio_initialize *init_data) { - struct ccw_device *cdev = init_data->cdev; + struct qdio_irq *irq_ptr = cdev->private->qdio_data; struct subchannel_id schid; - struct qdio_irq *irq_ptr; + struct ciw *ciw; + long timeout; int rc; ccw_device_get_schid(cdev, &schid); DBF_EVENT("qestablish:%4x", schid.sch_no); - irq_ptr = cdev->private->qdio_data; if (!irq_ptr) return -ENODEV; + if (init_data->no_input_qs > irq_ptr->max_input_qs || + init_data->no_output_qs > irq_ptr->max_output_qs) + return -EINVAL; + + /* Needed as error_handler: */ + if (!init_data->input_handler) + return -EINVAL; + + if (init_data->no_output_qs && !init_data->output_handler) + return -EINVAL; + + if (!init_data->input_sbal_addr_array || + !init_data->output_sbal_addr_array) + return -EINVAL; + + if (!init_data->irq_poll) + return -EINVAL; + + ciw = ccw_device_get_ciw(cdev, CIW_TYPE_EQUEUE); + if (!ciw) { + DBF_ERROR("%4x NO EQ", schid.sch_no); + return -EIO; + } + mutex_lock(&irq_ptr->setup_mutex); - qdio_setup_irq(init_data); + qdio_trace_init_data(irq_ptr, init_data); + qdio_setup_irq(irq_ptr, init_data); rc = qdio_establish_thinint(irq_ptr); - if (rc) { - mutex_unlock(&irq_ptr->setup_mutex); - qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); - return rc; - } + if (rc) + goto err_thinint; /* establish q */ - irq_ptr->ccw.cmd_code = irq_ptr->equeue.cmd; - irq_ptr->ccw.flags = CCW_FLAG_SLI; - irq_ptr->ccw.count = irq_ptr->equeue.count; - irq_ptr->ccw.cda = (u32)((addr_t)irq_ptr->qdr); + irq_ptr->ccw->cmd_code = ciw->cmd; + irq_ptr->ccw->flags = CCW_FLAG_SLI; + irq_ptr->ccw->count = ciw->count; + irq_ptr->ccw->cda = (u32) virt_to_phys(irq_ptr->qdr); spin_lock_irq(get_ccwdev_lock(cdev)); ccw_device_set_options_mask(cdev, 0); - rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0); + rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0); spin_unlock_irq(get_ccwdev_lock(cdev)); if (rc) { DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no); DBF_ERROR("rc:%4x", rc); - mutex_unlock(&irq_ptr->setup_mutex); - qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); - return rc; + goto err_ccw_start; } - wait_event_interruptible_timeout(cdev->private->wait_q, - irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED || - irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ); + timeout = wait_event_interruptible_timeout(cdev->private->wait_q, + irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED || + irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ); + if (timeout <= 0) { + rc = (timeout == -ERESTARTSYS) ? -EINTR : -ETIME; + goto err_ccw_timeout; + } if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) { - mutex_unlock(&irq_ptr->setup_mutex); - qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR); - return -EIO; + rc = -EIO; + goto err_ccw_error; } qdio_setup_ssqd_info(irq_ptr); - qdio_detect_hsicq(irq_ptr); - /* qebsm is now setup if available, initialize buffer states */ qdio_init_buf_states(irq_ptr); mutex_unlock(&irq_ptr->setup_mutex); - qdio_print_subchannel_info(irq_ptr, cdev); - qdio_setup_debug_entries(irq_ptr, cdev); + qdio_print_subchannel_info(irq_ptr); + qdio_setup_debug_entries(irq_ptr); return 0; + +err_ccw_timeout: + qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR); +err_ccw_error: +err_ccw_start: + qdio_shutdown_thinint(irq_ptr); +err_thinint: + qdio_shutdown_irq(irq_ptr); + qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE); + mutex_unlock(&irq_ptr->setup_mutex); + return rc; } EXPORT_SYMBOL_GPL(qdio_establish); @@ -1386,32 +1125,38 @@ EXPORT_SYMBOL_GPL(qdio_establish); */ int qdio_activate(struct ccw_device *cdev) { + struct qdio_irq *irq_ptr = cdev->private->qdio_data; struct subchannel_id schid; - struct qdio_irq *irq_ptr; + struct ciw *ciw; int rc; ccw_device_get_schid(cdev, &schid); DBF_EVENT("qactivate:%4x", schid.sch_no); - irq_ptr = cdev->private->qdio_data; if (!irq_ptr) return -ENODEV; + ciw = ccw_device_get_ciw(cdev, CIW_TYPE_AQUEUE); + if (!ciw) { + DBF_ERROR("%4x NO AQ", schid.sch_no); + return -EIO; + } + mutex_lock(&irq_ptr->setup_mutex); if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) { rc = -EBUSY; goto out; } - irq_ptr->ccw.cmd_code = irq_ptr->aqueue.cmd; - irq_ptr->ccw.flags = CCW_FLAG_SLI; - irq_ptr->ccw.count = irq_ptr->aqueue.count; - irq_ptr->ccw.cda = 0; + irq_ptr->ccw->cmd_code = ciw->cmd; + irq_ptr->ccw->flags = CCW_FLAG_SLI; + irq_ptr->ccw->count = ciw->count; + irq_ptr->ccw->cda = 0; spin_lock_irq(get_ccwdev_lock(cdev)); ccw_device_set_options(cdev, CCWDEV_REPORT_ALL); - rc = ccw_device_start(cdev, &irq_ptr->ccw, QDIO_DOING_ACTIVATE, + rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE, 0, DOIO_DENY_PREFETCH); spin_unlock_irq(get_ccwdev_lock(cdev)); if (rc) { @@ -1420,9 +1165,6 @@ int qdio_activate(struct ccw_device *cdev) goto out; } - if (is_thinint_irq(irq_ptr)) - tiqdio_add_device(irq_ptr); - /* wait for subchannel to become active */ msleep(5); @@ -1441,85 +1183,74 @@ out: } EXPORT_SYMBOL_GPL(qdio_activate); -static inline int buf_in_between(int bufnr, int start, int count) -{ - int end = add_buf(start, count); - - if (end > start) { - if (bufnr >= start && bufnr < end) - return 1; - else - return 0; - } - - /* wrap-around case */ - if ((bufnr >= start && bufnr <= QDIO_MAX_BUFFERS_PER_Q) || - (bufnr < end)) - return 1; - else - return 0; -} - /** * handle_inbound - reset processed input buffers * @q: queue containing the buffers - * @callflags: flags * @bufnr: first buffer to process * @count: how many buffers are emptied */ -static int handle_inbound(struct qdio_q *q, unsigned int callflags, - int bufnr, int count) +static int handle_inbound(struct qdio_q *q, int bufnr, int count) { - int diff; + int overlap; qperf_inc(q, inbound_call); - if (!q->u.in.ack_count) - goto set; - - /* protect against stop polling setting an ACK for an emptied slsb */ - if (count == QDIO_MAX_BUFFERS_PER_Q) { - /* overwriting everything, just delete polling status */ - q->u.in.ack_count = 0; - goto set; - } else if (buf_in_between(q->u.in.ack_start, bufnr, count)) { - if (is_qebsm(q)) { - /* partial overwrite, just update ack_start */ - diff = add_buf(bufnr, count); - diff = sub_buf(diff, q->u.in.ack_start); - q->u.in.ack_count -= diff; - if (q->u.in.ack_count <= 0) { - q->u.in.ack_count = 0; - goto set; - } - q->u.in.ack_start = add_buf(q->u.in.ack_start, diff); - } else { - /* the only ACK will be deleted */ - q->u.in.ack_count = 0; - } + /* If any processed SBALs are returned to HW, adjust our tracking: */ + overlap = min_t(int, count - sub_buf(q->u.in.batch_start, bufnr), + q->u.in.batch_count); + if (overlap > 0) { + q->u.in.batch_start = add_buf(q->u.in.batch_start, overlap); + q->u.in.batch_count -= overlap; } -set: count = set_buf_states(q, bufnr, SLSB_CU_INPUT_EMPTY, count); atomic_add(count, &q->nr_buf_used); - if (need_siga_in(q)) + if (qdio_need_siga_in(q->irq_ptr)) return qdio_siga_input(q); return 0; } /** + * qdio_add_bufs_to_input_queue - process buffers on an Input Queue + * @cdev: associated ccw_device for the qdio subchannel + * @q_nr: queue number + * @bufnr: buffer number + * @count: how many buffers to process + */ +int qdio_add_bufs_to_input_queue(struct ccw_device *cdev, unsigned int q_nr, + unsigned int bufnr, unsigned int count) +{ + struct qdio_irq *irq_ptr = cdev->private->qdio_data; + + if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) + return -EINVAL; + + if (!irq_ptr) + return -ENODEV; + + DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count); + + if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) + return -EIO; + if (!count) + return 0; + + return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count); +} +EXPORT_SYMBOL_GPL(qdio_add_bufs_to_input_queue); + +/** * handle_outbound - process filled outbound buffers * @q: queue containing the buffers - * @callflags: flags * @bufnr: first buffer to process * @count: how many buffers are filled + * @aob: asynchronous operation block */ -static int handle_outbound(struct qdio_q *q, unsigned int callflags, - unsigned int bufnr, unsigned int count) +static int handle_outbound(struct qdio_q *q, unsigned int bufnr, unsigned int count, + struct qaob *aob) { - const unsigned int scan_threshold = q->irq_ptr->scan_threshold; unsigned char state = 0; int used, rc = 0; @@ -1531,21 +1262,13 @@ static int handle_outbound(struct qdio_q *q, unsigned int callflags, if (used == QDIO_MAX_BUFFERS_PER_Q) qperf_inc(q, outbound_queue_full); - if (callflags & QDIO_FLAG_PCI_OUT) { - q->u.out.pci_out_enabled = 1; - qperf_inc(q, pci_request_int); - } else - q->u.out.pci_out_enabled = 0; - if (queue_type(q) == QDIO_IQDIO_QFMT) { - unsigned long phys_aob = 0; - - if (q->u.out.use_cq && count == 1) - phys_aob = qdio_aob_for_buffer(&q->u.out, bufnr); + unsigned long phys_aob = aob ? virt_to_phys(aob) : 0; + WARN_ON_ONCE(!IS_ALIGNED(phys_aob, 256)); rc = qdio_kick_outbound_q(q, count, phys_aob); - } else if (need_siga_sync(q)) { - rc = qdio_siga_sync_q(q); + } else if (qdio_need_siga_sync(q->irq_ptr)) { + rc = qdio_sync_output_queue(q); } else if (count < QDIO_MAX_BUFFERS_PER_Q && get_buf_state(q, prev_buf(bufnr), &state, 0) > 0 && state == SLSB_CU_OUTPUT_PRIMED) { @@ -1555,79 +1278,61 @@ static int handle_outbound(struct qdio_q *q, unsigned int callflags, rc = qdio_kick_outbound_q(q, count, 0); } - /* Let drivers implement their own completion scanning: */ - if (!scan_threshold) - return rc; - - /* in case of SIGA errors we must process the error immediately */ - if (used >= scan_threshold || rc) - qdio_tasklet_schedule(q); - else - /* free the SBALs in case of no further traffic */ - if (!timer_pending(&q->u.out.timer) && - likely(q->irq_ptr->state == QDIO_IRQ_STATE_ACTIVE)) - mod_timer(&q->u.out.timer, jiffies + HZ); return rc; } /** - * do_QDIO - process input or output buffers + * qdio_add_bufs_to_output_queue - process buffers on an Output Queue * @cdev: associated ccw_device for the qdio subchannel - * @callflags: input or output and special flags from the program * @q_nr: queue number * @bufnr: buffer number * @count: how many buffers to process + * @aob: asynchronous operation block */ -int do_QDIO(struct ccw_device *cdev, unsigned int callflags, - int q_nr, unsigned int bufnr, unsigned int count) +int qdio_add_bufs_to_output_queue(struct ccw_device *cdev, unsigned int q_nr, + unsigned int bufnr, unsigned int count, + struct qaob *aob) { - struct qdio_irq *irq_ptr; + struct qdio_irq *irq_ptr = cdev->private->qdio_data; if (bufnr >= QDIO_MAX_BUFFERS_PER_Q || count > QDIO_MAX_BUFFERS_PER_Q) return -EINVAL; - irq_ptr = cdev->private->qdio_data; if (!irq_ptr) return -ENODEV; - DBF_DEV_EVENT(DBF_INFO, irq_ptr, - "do%02x b:%02x c:%02x", callflags, bufnr, count); + DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count); if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE) return -EIO; if (!count) return 0; - if (callflags & QDIO_FLAG_SYNC_INPUT) - return handle_inbound(irq_ptr->input_qs[q_nr], - callflags, bufnr, count); - else if (callflags & QDIO_FLAG_SYNC_OUTPUT) - return handle_outbound(irq_ptr->output_qs[q_nr], - callflags, bufnr, count); - return -EINVAL; + + return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob); } -EXPORT_SYMBOL_GPL(do_QDIO); +EXPORT_SYMBOL_GPL(qdio_add_bufs_to_output_queue); /** - * qdio_start_irq - process input buffers + * qdio_start_irq - enable interrupt processing for the device * @cdev: associated ccw_device for the qdio subchannel - * @nr: input queue number * * Return codes * 0 - success * 1 - irqs not started since new data is available */ -int qdio_start_irq(struct ccw_device *cdev, int nr) +int qdio_start_irq(struct ccw_device *cdev) { struct qdio_q *q; struct qdio_irq *irq_ptr = cdev->private->qdio_data; + unsigned int i; if (!irq_ptr) return -ENODEV; - q = irq_ptr->input_qs[nr]; - clear_nonshared_ind(irq_ptr); - qdio_stop_polling(q); - clear_bit(QDIO_QUEUE_IRQS_DISABLED, &q->u.in.queue_irq_state); + for_each_input_queue(irq_ptr, q, i) + qdio_stop_polling(q); + + clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state); /* * We need to check again to not lose initiative after @@ -1635,13 +1340,16 @@ int qdio_start_irq(struct ccw_device *cdev, int nr) */ if (test_nonshared_ind(irq_ptr)) goto rescan; - if (!qdio_inbound_q_done(q, q->first_to_check)) - goto rescan; + + for_each_input_queue(irq_ptr, q, i) { + if (!qdio_inbound_q_done(q, q->first_to_check)) + goto rescan; + } + return 0; rescan: - if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, - &q->u.in.queue_irq_state)) + if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) return 0; else return 1; @@ -1649,197 +1357,28 @@ rescan: } EXPORT_SYMBOL(qdio_start_irq); -static int __qdio_inspect_queue(struct qdio_q *q, unsigned int *bufnr, - unsigned int *error) -{ - unsigned int start = q->first_to_check; - int count; - - count = q->is_input_q ? qdio_inbound_q_moved(q, start) : - qdio_outbound_q_moved(q, start); - if (count == 0) - return 0; - - *bufnr = start; - *error = q->qdio_error; - - /* for the next time */ - q->first_to_check = add_buf(start, count); - q->qdio_error = 0; - - return count; -} - -int qdio_inspect_queue(struct ccw_device *cdev, unsigned int nr, bool is_input, - unsigned int *bufnr, unsigned int *error) -{ - struct qdio_irq *irq_ptr = cdev->private->qdio_data; - struct qdio_q *q; - - if (!irq_ptr) - return -ENODEV; - q = is_input ? irq_ptr->input_qs[nr] : irq_ptr->output_qs[nr]; - - if (need_siga_sync(q)) - qdio_siga_sync_q(q); - - return __qdio_inspect_queue(q, bufnr, error); -} -EXPORT_SYMBOL_GPL(qdio_inspect_queue); - -/** - * qdio_get_next_buffers - process input buffers - * @cdev: associated ccw_device for the qdio subchannel - * @nr: input queue number - * @bufnr: first filled buffer number - * @error: buffers are in error state - * - * Return codes - * < 0 - error - * = 0 - no new buffers found - * > 0 - number of processed buffers - */ -int qdio_get_next_buffers(struct ccw_device *cdev, int nr, int *bufnr, - int *error) -{ - struct qdio_q *q; - struct qdio_irq *irq_ptr = cdev->private->qdio_data; - - if (!irq_ptr) - return -ENODEV; - q = irq_ptr->input_qs[nr]; - - /* - * Cannot rely on automatic sync after interrupt since queues may - * also be examined without interrupt. - */ - if (need_siga_sync(q)) - qdio_sync_queues(q); - - qdio_check_outbound_pci_queues(irq_ptr); - - /* Note: upper-layer MUST stop processing immediately here ... */ - if (unlikely(q->irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)) - return -EIO; - - return __qdio_inspect_queue(q, bufnr, error); -} -EXPORT_SYMBOL(qdio_get_next_buffers); - /** * qdio_stop_irq - disable interrupt processing for the device * @cdev: associated ccw_device for the qdio subchannel - * @nr: input queue number * * Return codes * 0 - interrupts were already disabled * 1 - interrupts successfully disabled */ -int qdio_stop_irq(struct ccw_device *cdev, int nr) +int qdio_stop_irq(struct ccw_device *cdev) { - struct qdio_q *q; struct qdio_irq *irq_ptr = cdev->private->qdio_data; if (!irq_ptr) return -ENODEV; - q = irq_ptr->input_qs[nr]; - if (test_and_set_bit(QDIO_QUEUE_IRQS_DISABLED, - &q->u.in.queue_irq_state)) + if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state)) return 0; else return 1; } EXPORT_SYMBOL(qdio_stop_irq); -/** - * qdio_pnso_brinfo() - perform network subchannel op #0 - bridge info. - * @schid: Subchannel ID. - * @cnc: Boolean Change-Notification Control - * @response: Response code will be stored at this address - * @cb: Callback function will be executed for each element - * of the address list - * @priv: Pointer to pass to the callback function. - * - * Performs "Store-network-bridging-information list" operation and calls - * the callback function for every entry in the list. If "change- - * notification-control" is set, further changes in the address list - * will be reported via the IPA command. - */ -int qdio_pnso_brinfo(struct subchannel_id schid, - int cnc, u16 *response, - void (*cb)(void *priv, enum qdio_brinfo_entry_type type, - void *entry), - void *priv) -{ - struct chsc_pnso_area *rr; - int rc; - u32 prev_instance = 0; - int isfirstblock = 1; - int i, size, elems; - - rr = (struct chsc_pnso_area *)get_zeroed_page(GFP_KERNEL); - if (rr == NULL) - return -ENOMEM; - do { - /* on the first iteration, naihdr.resume_token will be zero */ - rc = chsc_pnso_brinfo(schid, rr, rr->naihdr.resume_token, cnc); - if (rc != 0 && rc != -EBUSY) - goto out; - if (rr->response.code != 1) { - rc = -EIO; - continue; - } else - rc = 0; - - if (cb == NULL) - continue; - - size = rr->naihdr.naids; - elems = (rr->response.length - - sizeof(struct chsc_header) - - sizeof(struct chsc_brinfo_naihdr)) / - size; - - if (!isfirstblock && (rr->naihdr.instance != prev_instance)) { - /* Inform the caller that they need to scrap */ - /* the data that was already reported via cb */ - rc = -EAGAIN; - break; - } - isfirstblock = 0; - prev_instance = rr->naihdr.instance; - for (i = 0; i < elems; i++) - switch (size) { - case sizeof(struct qdio_brinfo_entry_l3_ipv6): - (*cb)(priv, l3_ipv6_addr, - &rr->entries.l3_ipv6[i]); - break; - case sizeof(struct qdio_brinfo_entry_l3_ipv4): - (*cb)(priv, l3_ipv4_addr, - &rr->entries.l3_ipv4[i]); - break; - case sizeof(struct qdio_brinfo_entry_l2): - (*cb)(priv, l2_addr_lnid, - &rr->entries.l2[i]); - break; - default: - WARN_ON_ONCE(1); - rc = -EIO; - goto out; - } - } while (rr->response.code == 0x0107 || /* channel busy */ - (rr->response.code == 1 && /* list stored */ - /* resume token is non-zero => list incomplete */ - (rr->naihdr.resume_token.t1 || rr->naihdr.resume_token.t2))); - (*response) = rr->response.code; - -out: - free_page((unsigned long)rr); - return rc; -} -EXPORT_SYMBOL_GPL(qdio_pnso_brinfo); - static int __init init_QDIO(void) { int rc; @@ -1850,16 +1389,11 @@ static int __init init_QDIO(void) rc = qdio_setup_init(); if (rc) goto out_debug; - rc = tiqdio_allocate_memory(); + rc = qdio_thinint_init(); if (rc) goto out_cache; - rc = tiqdio_register_thinints(); - if (rc) - goto out_ti; return 0; -out_ti: - tiqdio_free_memory(); out_cache: qdio_setup_exit(); out_debug: @@ -1869,8 +1403,7 @@ out_debug: static void __exit exit_QDIO(void) { - tiqdio_unregister_thinints(); - tiqdio_free_memory(); + qdio_thinint_exit(); qdio_setup_exit(); qdio_debug_exit(); } |