Proper support for the so-called `fast mode' of the Indigo2 ECC memory

controller. In this mode, access to physical memory are not allowed to
bypass the cache, and this allows the memory subsystem to run faster.

Of course, some device drivers will require uncached memory access (e.g.
for proper HPC DMA descriptor operation).

New ip22-specific functions to switch between `fast mode' and `slow mode'
are introduced.

hpc(4) now provides read and write routines to fetch a dma descriptor from
uncached memory into a local copy, and update it from said modified copy.
On systems without the ECC MC, these will do nothing and operation will
continue to access the uncached memory directly. On systems with the ECC MC,
they will perform a copy, and the writeback will be done in slow mode.

bus_dmamem_map() requests for DMA memory with BUS_DMA_COHERENT set in flags,
which would return uncached memory, will now always fail on systems with
the ECC memory controller. Drivers which really need uncached memory, and
are aware of this particular setup, will now pass
BUS_DMA_COHERENT | BUS_DMA_BUS1, which will let the request succeed.

sq(4) will use all of the above to work mostly unmodified on ECC MC systems
in fast mode.

Finally, fast mode is enabled after autoconf.

Tested on IP22 and IP28.

4 files changed, 141 insertions, 35 deletions

diff --git a/sys/arch/sgi/hpc/hpc.c b/sys/arch/sgi/hpc/hpc.c
index 99c1cea1a64..6c21704dd51 100644
--- a/sys/arch/sgi/hpc/hpc.c
+++ b/sys/arch/sgi/hpc/hpc.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hpc.c,v 1.12 2012/05/17 19:38:59 miod Exp $	*/
+/*	$OpenBSD: hpc.c,v 1.13 2012/05/27 14:27:08 miod Exp $	*/
 /*	$NetBSD: hpc.c,v 1.66 2011/07/01 18:53:46 dyoung Exp $	*/
 /*	$NetBSD: ioc.c,v 1.9 2011/07/01 18:53:47 dyoung Exp $	 */
 
@@ -363,6 +363,18 @@ void	hpc_blink_ioc(void *);
 int	hpc_read_eeprom(int, bus_space_tag_t, bus_space_handle_t, uint8_t *,
 	    size_t);
 
+struct hpc_dma_desc *hpc_read_dma_desc_par(struct hpc_dma_desc *,
+	    struct hpc_dma_desc *);
+struct hpc_dma_desc *hpc_read_dma_desc_ecc(struct hpc_dma_desc *,
+	    struct hpc_dma_desc *);
+void	hpc_write_dma_desc_par(struct hpc_dma_desc *, struct hpc_dma_desc *);
+void	hpc_write_dma_desc_ecc(struct hpc_dma_desc *, struct hpc_dma_desc *);
+
+/* globals since they depend upon the system type, not the hpc version */
+struct hpc_dma_desc *(*hpc_read_dma_desc_fn)(struct hpc_dma_desc *,
+	    struct hpc_dma_desc *);
+void	(*hpc_write_dma_desc_fn)(struct hpc_dma_desc *, struct hpc_dma_desc *);
+
 const struct cfattach hpc_ca = {
 	sizeof(struct hpc_softc), hpc_match, hpc_attach
 };
@@ -423,6 +435,17 @@ hpc_attach(struct device *parent, struct device *self, void *aux)
 	sc->sc_ch = PHYS_TO_XKPHYS(sc->sc_base, CCA_NC);
 	sc->sc_dmat = ga->ga_dmat;
 
+	/* setup HPC DMA helpers if not done already */
+	if (hpc_read_dma_desc_fn == NULL) {
+		if (ip22_ecc) {
+			hpc_read_dma_desc_fn = hpc_read_dma_desc_ecc;
+			hpc_write_dma_desc_fn = hpc_write_dma_desc_ecc;
+		} else {
+			hpc_read_dma_desc_fn = hpc_read_dma_desc_par;
+			hpc_write_dma_desc_fn = hpc_write_dma_desc_par;
+		}
+	}
+
 	switch (sys_config.system_type) {
 	case SGI_IP20:
 		sysmask = HPCDEV_IP20;
@@ -749,7 +772,7 @@ hpc_attach(struct device *parent, struct device *self, void *aux)
  *	o Indigo2, Challenge M
  *		One on-board HPC3.
  *
- * All we really have to worry about is the IP22 case.
+ * All we really have to worry about is the IP24 case.
  */
 int
 hpc_revision(struct hpc_softc *sc, struct gio_attach_args *ga)
@@ -775,12 +798,16 @@ hpc_revision(struct hpc_softc *sc, struct gio_attach_args *ga)
 	case SGI_IP22:
 	case SGI_IP26:
 	case SGI_IP28:
+		if (ga->ga_addr == HPC_BASE_ADDRESS_0)
+			return 3;
+
+		if (sys_config.system_subtype == IP22_INDIGO2)
+			return 0;
+
 		/*
-		 * If IP22, probe slot 0 to determine if HPC1.5 or HPC3. Slot 1
+		 * If IP24, probe slot 0 to determine if HPC1.5 or HPC3. Slot 1
 		 * must be HPC1.5.
 		 */
-		if (ga->ga_addr == HPC_BASE_ADDRESS_0)
-			return 3;
 
 		if (ga->ga_addr == HPC_BASE_ADDRESS_2)
 			return 15;
@@ -943,3 +970,52 @@ hpc_read_eeprom(int hpctype, bus_space_tag_t t, bus_space_handle_t h,
 
 	return 0;
 }
+
+/*
+ * Routines to copy and update HPC DMA descriptors in uncached memory.
+ */
+
+struct hpc_dma_desc *
+hpc_read_dma_desc(struct hpc_dma_desc *src, struct hpc_dma_desc *store)
+{
+	return (*hpc_read_dma_desc_fn)(src, store);
+}
+
+void
+hpc_write_dma_desc(struct hpc_dma_desc *dst, struct hpc_dma_desc *src)
+{
+	(*hpc_write_dma_desc_fn)(dst, src);
+}
+
+/* parity MC flavour: no copy */
+struct hpc_dma_desc *
+hpc_read_dma_desc_par(struct hpc_dma_desc *src, struct hpc_dma_desc *store)
+{
+	return src;
+}
+
+void
+hpc_write_dma_desc_par(struct hpc_dma_desc *dst, struct hpc_dma_desc *src)
+{
+}
+
+/* ECC MC flavour: copy, and update in slow mode */
+struct hpc_dma_desc *
+hpc_read_dma_desc_ecc(struct hpc_dma_desc *src, struct hpc_dma_desc *store)
+{
+	bcopy(src, store, sizeof(struct hpc_dma_desc));
+	return store;
+}
+
+void
+hpc_write_dma_desc_ecc(struct hpc_dma_desc *dst, struct hpc_dma_desc *src)
+{
+	uint32_t sr;
+	int mode;
+
+	sr = disableintr();
+	mode = ip22_slow_mode();
+	bcopy(src, dst, sizeof(struct hpc_dma_desc));
+	ip22_restore_mode(mode);
+	setsr(sr);
+}
diff --git a/sys/arch/sgi/hpc/hpcvar.h b/sys/arch/sgi/hpc/hpcvar.h
index 9a1e656c6b7..6c0f7a86aed 100644
--- a/sys/arch/sgi/hpc/hpcvar.h
+++ b/sys/arch/sgi/hpc/hpcvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hpcvar.h,v 1.7 2012/04/30 21:30:33 miod Exp $	*/
+/*	$OpenBSD: hpcvar.h,v 1.8 2012/05/27 14:27:08 miod Exp $	*/
 /*	$NetBSD: hpcvar.h,v 1.12 2011/01/25 12:21:04 tsutsui Exp $	*/
 
 /*
@@ -110,4 +110,14 @@ int	 hpc_is_intr_pending(int);
 void	 hpc_intr_disable(void *);
 void	 hpc_intr_enable(void *);
 
+/*
+ * Routines to copy and update HPC DMA descriptors in uncached memory;
+ * needed for proper operation on ECC MC systems.
+ */
+struct hpc_dma_desc;
+
+struct hpc_dma_desc *hpc_read_dma_desc(struct hpc_dma_desc *src,
+	    struct hpc_dma_desc *store);
+void	hpc_write_dma_desc(struct hpc_dma_desc *dst, struct hpc_dma_desc *src);
+
 extern bus_space_t hpc3bus_tag;
diff --git a/sys/arch/sgi/hpc/if_sq.c b/sys/arch/sgi/hpc/if_sq.c
index e9fdd98218e..df697555416 100644
--- a/sys/arch/sgi/hpc/if_sq.c
+++ b/sys/arch/sgi/hpc/if_sq.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_sq.c,v 1.6 2012/05/22 19:53:40 miod Exp $	*/
+/*	$OpenBSD: if_sq.c,v 1.7 2012/05/27 14:27:08 miod Exp $	*/
 /*	$NetBSD: if_sq.c,v 1.42 2011/07/01 18:53:47 dyoung Exp $	*/
 
 /*
@@ -229,9 +229,15 @@ sq_attach(struct device *parent, struct device *self, void *aux)
 		goto fail_0;
 	}
 
+	/*
+	 * Note that we need to pass BUS_DMA_BUS1 in order to get this
+	 * allocation to succeed on ECC MC systems. This code is
+	 * uncached-write safe, as all updates of the DMA descriptors are
+	 * handled in RCU style with hpc_{read,write}_dma_desc().
+	 */
 	if ((rc = bus_dmamem_map(sc->sc_dmat, &sc->sc_cdseg, sc->sc_ncdseg,
 	    sizeof(struct sq_control), (caddr_t *)&sc->sc_control,
-	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT)) != 0) {
+	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT | BUS_DMA_BUS1)) != 0) {
 		printf(": unable to map control data, error = %d\n", rc);
 		goto fail_1;
 	}
@@ -656,9 +662,10 @@ void
 sq_start(struct ifnet *ifp)
 {
 	struct sq_softc *sc = ifp->if_softc;
-	uint32_t status;
 	struct mbuf *m0, *m;
+	struct hpc_dma_desc *txd, txd_store;
 	bus_dmamap_t dmamap;
+	uint32_t status;
 	int err, len, totlen, nexttx, firsttx, lasttx = -1, ofree, seg;
 
 	if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
@@ -789,31 +796,31 @@ sq_start(struct ifnet *ifp)
 		for (nexttx = sc->sc_nexttx, seg = 0, totlen = 0;
 		     seg < dmamap->dm_nsegs;
 		     seg++, nexttx = SQ_NEXTTX(nexttx)) {
+			txd = hpc_read_dma_desc(sc->sc_txdesc + nexttx,
+			    &txd_store);
 			if (sc->hpc_regs->revision == 3) {
-				sc->sc_txdesc[nexttx].hpc3_hdd_bufptr =
+				txd->hpc3_hdd_bufptr =
 				    dmamap->dm_segs[seg].ds_addr;
-				sc->sc_txdesc[nexttx].hpc3_hdd_ctl =
-				    dmamap->dm_segs[seg].ds_len;
+				txd->hpc3_hdd_ctl = dmamap->dm_segs[seg].ds_len;
 			} else {
-				sc->sc_txdesc[nexttx].hpc1_hdd_bufptr =
+				txd->hpc1_hdd_bufptr =
 				    dmamap->dm_segs[seg].ds_addr;
-				sc->sc_txdesc[nexttx].hpc1_hdd_ctl =
-				    dmamap->dm_segs[seg].ds_len;
+				txd->hpc1_hdd_ctl = dmamap->dm_segs[seg].ds_len;
 			}
-			sc->sc_txdesc[nexttx].hdd_descptr =
-			    SQ_CDTXADDR(sc, SQ_NEXTTX(nexttx));
+			txd->hdd_descptr = SQ_CDTXADDR(sc, SQ_NEXTTX(nexttx));
+			hpc_write_dma_desc(sc->sc_txdesc + nexttx, txd);
 			lasttx = nexttx;
 			totlen += dmamap->dm_segs[seg].ds_len;
 		}
 
 		/* Last descriptor gets end-of-packet */
 		KASSERT(lasttx != -1);
+		/* txd = hpc_read_dma_desc(sc->sc_txdesc + lasttx, &txd_store); */
 		if (sc->hpc_regs->revision == 3)
-			sc->sc_txdesc[lasttx].hpc3_hdd_ctl |=
-			    HPC3_HDD_CTL_EOPACKET;
+			txd->hpc3_hdd_ctl |= HPC3_HDD_CTL_EOPACKET;
 		else
-			sc->sc_txdesc[lasttx].hpc1_hdd_ctl |=
-			    HPC1_HDD_CTL_EOPACKET;
+			txd->hpc1_hdd_ctl |= HPC1_HDD_CTL_EOPACKET;
+		hpc_write_dma_desc(sc->sc_txdesc + lasttx, txd);
 
 		SQ_DPRINTF(("%s: transmit %d-%d, len %d\n",
 		    sc->sc_dev.dv_xname, sc->sc_nexttx, lasttx, totlen));
@@ -869,15 +876,15 @@ sq_start(struct ifnet *ifp)
 		 * addition to HPC3_HDD_CTL_INTR to interrupt.
 		 */
 		KASSERT(lasttx != -1);
+		txd = hpc_read_dma_desc(sc->sc_txdesc + lasttx, &txd_store);
 		if (sc->hpc_regs->revision == 3) {
-			sc->sc_txdesc[lasttx].hpc3_hdd_ctl |=
+			txd->hpc3_hdd_ctl |=
 			    HPC3_HDD_CTL_INTR | HPC3_HDD_CTL_EOCHAIN;
 		} else {
-			sc->sc_txdesc[lasttx].hpc1_hdd_ctl |= HPC1_HDD_CTL_INTR;
-			sc->sc_txdesc[lasttx].hpc1_hdd_bufptr |=
-			    HPC1_HDD_CTL_EOCHAIN;
+			txd->hpc1_hdd_ctl |= HPC1_HDD_CTL_INTR;
+			txd->hpc1_hdd_bufptr |= HPC1_HDD_CTL_EOCHAIN;
 		}
-
+		hpc_write_dma_desc(sc->sc_txdesc + lasttx, txd);
 		SQ_CDTXSYNC(sc, lasttx, 1,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
@@ -898,17 +905,18 @@ sq_start(struct ifnet *ifp)
 		if ((status & sc->hpc_regs->enetx_ctl_active) != 0) {
 			SQ_TRACE(SQ_ADD_TO_DMA, sc, firsttx, status);
 
+			txd = hpc_read_dma_desc(sc->sc_txdesc +
+			    SQ_PREVTX(firsttx), &txd_store);
 			/*
 			 * NB: hpc3_hdd_ctl == hpc1_hdd_bufptr, and
 			 * HPC1_HDD_CTL_EOCHAIN == HPC3_HDD_CTL_EOCHAIN
 			 */
-			sc->sc_txdesc[SQ_PREVTX(firsttx)].hpc3_hdd_ctl &=
-			    ~HPC3_HDD_CTL_EOCHAIN;
-
+			txd->hpc3_hdd_ctl &= ~HPC3_HDD_CTL_EOCHAIN;
 			if (sc->hpc_regs->revision != 3)
-				sc->sc_txdesc[SQ_PREVTX(firsttx)].hpc1_hdd_ctl
-				    &= ~HPC1_HDD_CTL_INTR;
+				txd->hpc1_hdd_ctl &= ~HPC1_HDD_CTL_INTR;
 
+			hpc_write_dma_desc(sc->sc_txdesc + SQ_PREVTX(firsttx),
+			    txd);
 			SQ_CDTXSYNC(sc, SQ_PREVTX(firsttx),  1,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		} else if (sc->hpc_regs->revision == 3) {
@@ -1105,6 +1113,7 @@ sq_rxintr(struct sq_softc *sc)
 {
 	struct ifnet *ifp = &sc->sc_ac.ac_if;
 	struct mbuf* m;
+	struct hpc_dma_desc *rxd, rxd_store;
 	int i, framelen;
 	uint8_t pktstat;
 	uint32_t status;
@@ -1206,12 +1215,16 @@ sq_rxintr(struct sq_softc *sc)
 		 */
 
 		new_end = SQ_PREVRX(i);
-		sc->sc_rxdesc[new_end].hpc3_hdd_ctl |= HPC3_HDD_CTL_EOCHAIN;
+		rxd = hpc_read_dma_desc(sc->sc_rxdesc + new_end, &rxd_store);
+		rxd->hpc3_hdd_ctl |= HPC3_HDD_CTL_EOCHAIN;
+		hpc_write_dma_desc(sc->sc_rxdesc + new_end, rxd);
 		SQ_CDRXSYNC(sc, new_end,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 		orig_end = SQ_PREVRX(sc->sc_nextrx);
-		sc->sc_rxdesc[orig_end].hpc3_hdd_ctl &= ~HPC3_HDD_CTL_EOCHAIN;
+		rxd = hpc_read_dma_desc(sc->sc_rxdesc + orig_end, &rxd_store);
+		rxd->hpc3_hdd_ctl &= ~HPC3_HDD_CTL_EOCHAIN;
+		hpc_write_dma_desc(sc->sc_rxdesc + orig_end, rxd);
 		SQ_CDRXSYNC(sc, orig_end,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
diff --git a/sys/arch/sgi/hpc/if_sqvar.h b/sys/arch/sgi/hpc/if_sqvar.h
index 4f55053acc5..9bf5df1323b 100644
--- a/sys/arch/sgi/hpc/if_sqvar.h
+++ b/sys/arch/sgi/hpc/if_sqvar.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: if_sqvar.h,v 1.2 2012/04/30 21:31:03 miod Exp $	*/
+/*	$OpenBSD: if_sqvar.h,v 1.3 2012/05/27 14:27:08 miod Exp $	*/
 /*	$NetBSD: sqvar.h,v 1.12 2011/01/25 13:12:39 tsutsui Exp $	*/
 
 /*
@@ -159,6 +159,7 @@ struct sq_softc {
 #define	SQ_CDTXADDR(sc, x)	((sc)->sc_cddma + SQ_CDTXOFF((x)))
 #define	SQ_CDRXADDR(sc, x)	((sc)->sc_cddma + SQ_CDRXOFF((x)))
 
+#if 0 /* not necessary as this memory is mapped uncached */
 static inline void
 SQ_CDTXSYNC(struct sq_softc *sc, int __x, int __n, int ops)
 {
@@ -179,13 +180,18 @@ SQ_CDTXSYNC(struct sq_softc *sc, int __x, int __n, int ops)
 #define	SQ_CDRXSYNC(sc, x, ops)						\
 	bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_cdmap,			\
 	    SQ_CDRXOFF((x)), sizeof(struct hpc_dma_desc), (ops))
+#else
+#define	SQ_CDTXSYNC(sc, x, n, ops)	do { } while (0)
+#define	SQ_CDRXSYNC(sc, x, ops)		do { } while (0)
+#endif
 
 static inline void
 SQ_INIT_RXDESC(struct sq_softc *sc, unsigned int x)
 {
-	struct hpc_dma_desc* __rxd = &(sc)->sc_rxdesc[(x)];
+	struct hpc_dma_desc *__rxd, rxd_store;
 	struct mbuf *__m = (sc)->sc_rxmbuf[(x)];
 
+	__rxd = hpc_read_dma_desc(&(sc)->sc_rxdesc[(x)], &rxd_store);
 	__m->m_data = __m->m_ext.ext_buf;
 	if (sc->hpc_regs->revision == 3) {
 		__rxd->hpc3_hdd_bufptr =
@@ -201,5 +207,6 @@ SQ_INIT_RXDESC(struct sq_softc *sc, unsigned int x)
 		    HPC1_HDD_CTL_INTR | HPC1_HDD_CTL_EOPACKET;
 	}
 	__rxd->hdd_descptr = SQ_CDRXADDR((sc), SQ_NEXTRX((x)));
+	hpc_write_dma_desc(&(sc)->sc_rxdesc[(x)], __rxd);
 	SQ_CDRXSYNC((sc), (x), BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 }