Iopoolification. Testing by marco@.

ok dlg@ marco@

8 files changed, 64 insertions, 100 deletions

diff --git a/sys/dev/softraid.c b/sys/dev/softraid.c
index 74867a1a3f6..0a756f68e9f 100644
--- a/sys/dev/softraid.c
+++ b/sys/dev/softraid.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid.c,v 1.222 2011/03/15 13:29:41 jsing Exp $ */
+/* $OpenBSD: softraid.c,v 1.223 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2007, 2008, 2009 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2008 Chris Kuethe <ckuethe@openbsd.org>
@@ -1805,7 +1805,7 @@ sr_wu_alloc(struct sr_discipline *sd)
 	for (i = 0; i < no_wu; i++) {
 		wu = &sd->sd_wu[i];
 		wu->swu_dis = sd;
-		sr_wu_put(wu);
+		sr_wu_put(sd, wu);
 	}
 
 	return (0);
@@ -1833,9 +1833,10 @@ sr_wu_free(struct sr_discipline *sd)
 }
 
 void
-sr_wu_put(struct sr_workunit *wu)
+sr_wu_put(void *xsd, void *xwu)
 {
-	struct sr_discipline	*sd = wu->swu_dis;
+	struct sr_discipline	*sd = (struct sr_discipline *)xsd;
+	struct sr_workunit	*wu = (struct sr_workunit *)xwu;
 	struct sr_ccb		*ccb;
 
 	int			s;
@@ -1843,65 +1844,37 @@ sr_wu_put(struct sr_workunit *wu)
 	DNPRINTF(SR_D_WU, "%s: sr_wu_put: %p\n", DEVNAME(sd->sd_sc), wu);
 
 	s = splbio();
-
-	wu->swu_xs = NULL;
-	wu->swu_state = SR_WU_FREE;
-	wu->swu_ios_complete = 0;
-	wu->swu_ios_failed = 0;
-	wu->swu_ios_succeeded = 0;
-	wu->swu_io_count = 0;
-	wu->swu_blk_start = 0;
-	wu->swu_blk_end = 0;
-	wu->swu_collider = NULL;
-	wu->swu_fake = 0;
-	wu->swu_flags = 0;
-
 	if (wu->swu_cb_active == 1)
-		panic("%s: sr_wu_put", DEVNAME(sd->sd_sc));
+		panic("%s: sr_wu_put got active wu", DEVNAME(sd->sd_sc));
 	while ((ccb = TAILQ_FIRST(&wu->swu_ccb)) != NULL) {
 		TAILQ_REMOVE(&wu->swu_ccb, ccb, ccb_link);
 		sr_ccb_put(ccb);
 	}
+	splx(s);
+
+	bzero(wu, sizeof(*wu));
 	TAILQ_INIT(&wu->swu_ccb);
+	wu->swu_dis = sd;
 
+	mtx_enter(&sd->sd_wu_mtx);
 	TAILQ_INSERT_TAIL(&sd->sd_wu_freeq, wu, swu_link);
 	sd->sd_wu_pending--;
-
-	/* wake up sleepers */
-#ifdef DIAGNOSTIC
-	if (sd->sd_wu_sleep < 0)
-		panic("negative wu sleepers");
-#endif /* DIAGNOSTIC */
-	if (sd->sd_wu_sleep)
-		wakeup(&sd->sd_wu_sleep);
-
-	splx(s);
+	mtx_leave(&sd->sd_wu_mtx);
 }
 
-struct sr_workunit *
-sr_wu_get(struct sr_discipline *sd, int canwait)
+void *
+sr_wu_get(void *xsd)
 {
+	struct sr_discipline	*sd = (struct sr_discipline *)xsd;
 	struct sr_workunit	*wu;
-	int			s;
-
-	s = splbio();
 
-	for (;;) {
-		wu = TAILQ_FIRST(&sd->sd_wu_freeq);
-		if (wu) {
-			TAILQ_REMOVE(&sd->sd_wu_freeq, wu, swu_link);
-			wu->swu_state = SR_WU_INPROGRESS;
-			sd->sd_wu_pending++;
-			break;
-		} else if (wu == NULL && canwait) {
-			sd->sd_wu_sleep++;
-			tsleep(&sd->sd_wu_sleep, PRIBIO, "sr_wu_get", 0);
-			sd->sd_wu_sleep--;
-		} else
-			break;
+	mtx_enter(&sd->sd_wu_mtx);
+	wu = TAILQ_FIRST(&sd->sd_wu_freeq);
+	if (wu) {
+		TAILQ_REMOVE(&sd->sd_wu_freeq, wu, swu_link);
+		sd->sd_wu_pending++;
 	}
-
-	splx(s);
+	mtx_leave(&sd->sd_wu_mtx);
 
 	DNPRINTF(SR_D_WU, "%s: sr_wu_get: %p\n", DEVNAME(sd->sd_sc), wu);
 
@@ -1924,6 +1897,7 @@ sr_scsi_cmd(struct scsi_xfer *xs)
 	struct sr_softc		*sc = link->adapter_softc;
 	struct sr_workunit	*wu = NULL;
 	struct sr_discipline	*sd;
+	struct sr_ccb		*ccb;
 
 	DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd: scsibus%d xs: %p "
 	    "flags: %#x\n", DEVNAME(sc), link->scsibus, xs, xs->flags);
@@ -1949,18 +1923,21 @@ sr_scsi_cmd(struct scsi_xfer *xs)
 		goto stuffup;
 	}
 
-	/*
-	 * we'll let the midlayer deal with stalls instead of being clever
-	 * and sending sr_wu_get !(xs->flags & SCSI_NOSLEEP) in cansleep
-	 */
-	if ((wu = sr_wu_get(sd, 0)) == NULL) {
-		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd no wu\n", DEVNAME(sc));
-		xs->error = XS_NO_CCB;
-		sr_scsi_done(sd, xs);
-		return;
+	wu = xs->io;
+	/* scsi layer *can* re-send wu without calling sr_wu_put(). */
+	s = splbio();
+	if (wu->swu_cb_active == 1)
+		panic("%s: sr_scsi_cmd got active wu", DEVNAME(sd->sd_sc));
+	while ((ccb = TAILQ_FIRST(&wu->swu_ccb)) != NULL) {
+		TAILQ_REMOVE(&wu->swu_ccb, ccb, ccb_link);
+		sr_ccb_put(ccb);
 	}
+	splx(s);
 
-	xs->error = XS_NOERROR;
+	bzero(wu, sizeof(*wu));
+	TAILQ_INIT(&wu->swu_ccb);
+	wu->swu_state = SR_WU_INPROGRESS;
+	wu->swu_dis = sd;
 	wu->swu_xs = xs;
 
 	/* the midlayer will query LUNs so report sense to stop scanning */
@@ -2049,8 +2026,6 @@ stuffup:
 		xs->error = XS_DRIVER_STUFFUP;
 	}
 complete:
-	if (wu)
-		sr_wu_put(wu);
 	sr_scsi_done(sd, xs);
 }
 int
@@ -3042,6 +3017,8 @@ sr_ioctl_createraid(struct sr_softc *sc, struct bioc_createraid *bc, int user)
 		}
 
 		/* setup scsi midlayer */
+		mtx_init(&sd->sd_wu_mtx, IPL_BIO);
+		scsi_iopool_init(&sd->sd_iopool, sd, sr_wu_get, sr_wu_put);
 		if (sd->sd_openings)
 			sd->sd_link.openings = sd->sd_openings(sd);
 		else
@@ -3051,6 +3028,7 @@ sr_ioctl_createraid(struct sr_softc *sc, struct bioc_createraid *bc, int user)
 		sd->sd_link.adapter = &sr_switch;
 		sd->sd_link.adapter_target = SR_MAX_LD;
 		sd->sd_link.adapter_buswidth = 1;
+		sd->sd_link.pool = &sd->sd_iopool;
 		bzero(&saa, sizeof(saa));
 		saa.saa_sc_link = &sd->sd_link;
 
@@ -3954,9 +3932,9 @@ sr_rebuild_thread(void *arg)
 		lba = blk * sz;
 
 		/* get some wu */
-		if ((wu_r = sr_wu_get(sd, 1)) == NULL)
+		if ((wu_r = scsi_io_get(&sd->sd_iopool, 0)) == NULL)
 			panic("%s: rebuild exhausted wu_r", DEVNAME(sc));
-		if ((wu_w = sr_wu_get(sd, 1)) == NULL)
+		if ((wu_w = scsi_io_get(&sd->sd_iopool, 0)) == NULL)
 			panic("%s: rebuild exhausted wu_w", DEVNAME(sc));
 
 		/* setup read io */
@@ -4026,8 +4004,8 @@ queued:
 		if (slept == 0)
 			tsleep(sc, PWAIT, "sr_yield", 1);
 
-		sr_wu_put(wu_r);
-		sr_wu_put(wu_w);
+		scsi_io_put(&sd->sd_iopool, wu_r);
+		scsi_io_put(&sd->sd_iopool, wu_w);
 
 		sd->sd_meta->ssd_rebuild = lba;
 
diff --git a/sys/dev/softraid_aoe.c b/sys/dev/softraid_aoe.c
index 5103b10996d..fd05b902be8 100644
--- a/sys/dev/softraid_aoe.c
+++ b/sys/dev/softraid_aoe.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_aoe.c,v 1.17 2010/07/02 15:49:25 krw Exp $ */
+/* $OpenBSD: softraid_aoe.c,v 1.18 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2008 Ted Unangst <tedu@openbsd.org>
  * Copyright (c) 2008 Marco Peereboom <marco@openbsd.org>
@@ -576,9 +576,6 @@ sr_aoe_input(struct aoe_handler *ah, struct mbuf *m)
 				break;
 			}
 		}
-
-		/* do not change the order of these 2 functions */
-		sr_wu_put(wu);
 		sr_scsi_done(sd, xs);
 	}
 
diff --git a/sys/dev/softraid_crypto.c b/sys/dev/softraid_crypto.c
index 8d6f11585ae..93326d435f0 100644
--- a/sys/dev/softraid_crypto.c
+++ b/sys/dev/softraid_crypto.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_crypto.c,v 1.63 2011/03/06 21:41:57 stsp Exp $ */
+/* $OpenBSD: softraid_crypto.c,v 1.64 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2008 Hans-Joerg Hoexer <hshoexer@openbsd.org>
@@ -1345,8 +1345,6 @@ sr_crypto_finish_io(struct sr_workunit *wu)
 		sr_crypto_putcryptop(ccb->ccb_opaque);
 	}
 
-	/* do not change the order of these 2 functions */
-	sr_wu_put(wu);
 	sr_scsi_done(sd, xs);
 
 	if (sd->sd_sync && sd->sd_wu_pending == 0)
diff --git a/sys/dev/softraid_raid0.c b/sys/dev/softraid_raid0.c
index f8473a876e6..5f2bcc0c35e 100644
--- a/sys/dev/softraid_raid0.c
+++ b/sys/dev/softraid_raid0.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_raid0.c,v 1.22 2010/07/02 09:20:26 jsing Exp $ */
+/* $OpenBSD: softraid_raid0.c,v 1.23 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2008 Marco Peereboom <marco@peereboom.us>
  *
@@ -456,8 +456,6 @@ sr_raid0_intr(struct buf *bp)
 			printf("%s: wu: %p not on pending queue\n",
 			    DEVNAME(sc), wu);
 
-		/* do not change the order of these 2 functions */
-		sr_wu_put(wu);
 		sr_scsi_done(sd, xs);
 
 		if (sd->sd_sync && sd->sd_wu_pending == 0)
@@ -468,7 +466,6 @@ sr_raid0_intr(struct buf *bp)
 	return;
 bad:
 	xs->error = XS_DRIVER_STUFFUP;
-	sr_wu_put(wu);
 	sr_scsi_done(sd, xs);
 	splx(s);
 }
diff --git a/sys/dev/softraid_raid1.c b/sys/dev/softraid_raid1.c
index 2a44e996cc2..0979492e3eb 100644
--- a/sys/dev/softraid_raid1.c
+++ b/sys/dev/softraid_raid1.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_raid1.c,v 1.26 2010/11/06 23:01:56 marco Exp $ */
+/* $OpenBSD: softraid_raid1.c,v 1.27 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2007 Marco Peereboom <marco@peereboom.us>
  *
@@ -615,8 +615,6 @@ sr_raid1_intr(struct buf *bp)
 				wakeup(wu);
 			}
 		} else {
-			/* do not change the order of these 2 functions */
-			sr_wu_put(wu);
 			scsi_done(xs);
 		}
 
@@ -633,8 +631,6 @@ bad:
 		wu->swu_flags |= SR_WUF_REBUILDIOCOMP;
 		wakeup(wu);
 	} else {
-		/* do not change the order of these 2 functions */
-		sr_wu_put(wu);
 		scsi_done(xs);
 	}
 
diff --git a/sys/dev/softraid_raid6.c b/sys/dev/softraid_raid6.c
index cc296556da7..85d3106a89a 100644
--- a/sys/dev/softraid_raid6.c
+++ b/sys/dev/softraid_raid6.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_raid6.c,v 1.19 2010/08/07 03:50:01 krw Exp $ */
+/* $OpenBSD: softraid_raid6.c,v 1.20 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2009 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2009 Jordan Hargrave <jordan@openbsd.org>
@@ -480,7 +480,7 @@ sr_raid6_rw(struct sr_workunit *wu)
 	rwmode = (xs->flags & SCSI_DATA_IN) ? M_RFLG : M_WFLG;
 	if (xs->flags & SCSI_DATA_OUT)
 		/* create write workunit */
-		if ((wu_w = sr_wu_get(sd, 0)) == NULL) {
+		if ((wu_w = scsi_io_get(&sd->sd_iopool, SCSI_NOSLEEP)) == NULL){
 			printf("%s: can't get wu_w", DEVNAME(sd->sd_sc));
 			goto bad;
 		}
@@ -744,7 +744,7 @@ queued:
 bad:
 	/* wu is unwound by sr_wu_put */
 	if (wu_w)
-		sr_wu_put(wu_w);
+		scsi_io_put(&sd->sd_iopool, wu_w);
 	return (1);
 }
 
@@ -889,10 +889,10 @@ sr_raid6_intr(struct buf *bp)
 				wakeup(wu);
 			}
 		} else {
-			/* do not change the order of these 2 functions */
-			sr_wu_put(wu);
 			if (xs != NULL)
 				scsi_done(xs);
+			else
+				scsi_io_put(&sd->sd_iopool, wu);
 		}
 
 		if (sd->sd_sync && sd->sd_wu_pending == 0)
@@ -908,8 +908,6 @@ bad:
 		wu->swu_flags |= SR_WUF_REBUILDIOCOMP;
 		wakeup(wu);
 	} else {
-		/* do not change the order of these 2 functions */
-		sr_wu_put(wu);
 		scsi_done(xs);
 	}
 
diff --git a/sys/dev/softraid_raidp.c b/sys/dev/softraid_raidp.c
index d46f1ac2398..57ba38e2f0f 100644
--- a/sys/dev/softraid_raidp.c
+++ b/sys/dev/softraid_raidp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraid_raidp.c,v 1.19 2010/08/07 03:50:01 krw Exp $ */
+/* $OpenBSD: softraid_raidp.c,v 1.20 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2009 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2009 Jordan Hargrave <jordan@openbsd.org>
@@ -396,7 +396,7 @@ sr_raidp_rw(struct sr_workunit *wu)
 
 	if (xs->flags & SCSI_DATA_OUT)
 		/* create write workunit */
-		if ((wu_w = sr_wu_get(sd, 0)) == NULL) {
+		if ((wu_w = scsi_io_get(&sd->sd_iopool, SCSI_NOSLEEP)) == NULL){
 			printf("%s: can't get wu_w", DEVNAME(sd->sd_sc));
 			goto bad;
 		}
@@ -553,7 +553,7 @@ queued:
 bad:
 	/* wu is unwound by sr_wu_put */
 	if (wu_w)
-		sr_wu_put(wu_w);
+		scsi_io_put(&sd->sd_iopool, wu_w);
 	return (1);
 }
 
@@ -669,10 +669,10 @@ sr_raidp_intr(struct buf *bp)
 				wakeup(wu);
 			}
 		} else {
-			/* do not change the order of these 2 functions */
-			sr_wu_put(wu);
 			if (xs != NULL)
 				scsi_done(xs);
+			else
+				scsi_io_put(&sd->sd_iopool, wu);
 		}
 
 		if (sd->sd_sync && sd->sd_wu_pending == 0)
@@ -688,8 +688,6 @@ bad:
 		wu->swu_flags |= SR_WUF_REBUILDIOCOMP;
 		wakeup(wu);
 	} else {
-		/* do not change the order of these 2 functions */
-		sr_wu_put(wu);
 		scsi_done(xs);
 	}
 
@@ -831,9 +829,9 @@ sr_raidp_scrub(struct sr_discipline *sd)
 	int s, slept;
 	void *xorbuf;
 
-	if ((wu_r = sr_wu_get(sd, 1)) == NULL)
+	if ((wu_w = scsi_io_get(&sd->sd_iopool, 0)) == NULL)
 		goto done;
-	if ((wu_w = sr_wu_get(sd, 1)) == NULL)
+	if ((wu_r = scsi_io_get(&sd->sd_iopool, 0)) == NULL)
 		goto done;
 
 	no_chunk = sd->sd_meta->ssdi.ssd_chunk_no - 1;
diff --git a/sys/dev/softraidvar.h b/sys/dev/softraidvar.h
index a310ab3b79d..821173f5fbd 100644
--- a/sys/dev/softraidvar.h
+++ b/sys/dev/softraidvar.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: softraidvar.h,v 1.98 2011/03/15 13:29:41 jsing Exp $ */
+/* $OpenBSD: softraidvar.h,v 1.99 2011/04/05 19:52:02 krw Exp $ */
 /*
  * Copyright (c) 2006 Marco Peereboom <marco@peereboom.us>
  * Copyright (c) 2008 Chris Kuethe <ckuethe@openbsd.org>
@@ -524,7 +524,9 @@ struct sr_discipline {
 	struct sr_wu_list	sd_wu_freeq;	/* free wu queue */
 	struct sr_wu_list	sd_wu_pendq;	/* pending wu queue */
 	struct sr_wu_list	sd_wu_defq;	/* deferred wu queue */
-	int			sd_wu_sleep;	/* wu sleepers counter */
+
+	struct mutex		sd_wu_mtx;
+	struct scsi_iopool	sd_iopool;
 
 	/* discipline stats */
 	int			sd_wu_pending;
@@ -604,8 +606,8 @@ struct sr_ccb		*sr_ccb_get(struct sr_discipline *);
 void			sr_ccb_put(struct sr_ccb *);
 int			sr_wu_alloc(struct sr_discipline *);
 void			sr_wu_free(struct sr_discipline *);
-struct sr_workunit	*sr_wu_get(struct sr_discipline *, int);
-void			sr_wu_put(struct sr_workunit *);
+void			*sr_wu_get(void *);
+void			sr_wu_put(void *, void *);
 
 /* misc functions */
 int32_t			sr_validate_stripsize(u_int32_t);