avoid extensive mbuf allocation for IPsec by replacing m_inject(4)

with m_makespace(4) from freebsd; ok mpi@, bluhm@, mikeb@, dlg@

7 files changed, 157 insertions, 103 deletions

diff --git a/share/man/man9/mbuf.9 b/share/man/man9/mbuf.9
index d3a44aa30af..3ebee83a933 100644
--- a/share/man/man9/mbuf.9
+++ b/share/man/man9/mbuf.9
@@ -1,4 +1,4 @@
-.\"     $OpenBSD: mbuf.9,v 1.102 2016/09/04 02:26:44 lteo Exp $
+.\"     $OpenBSD: mbuf.9,v 1.103 2016/09/13 19:56:55 markus Exp $
 .\"
 .\" Copyright (c) 2001 Jean-Jacques Bernard-Gundol <jjbg@openbsd.org>
 .\" All rights reserved.
@@ -25,7 +25,7 @@
 .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 .\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
-.Dd $Mdocdate: September 4 2016 $
+.Dd $Mdocdate: September 13 2016 $
 .Dt MGET 9
 .Os
 .Sh NAME
@@ -43,7 +43,7 @@
 .Nm m_pulldown ,
 .Nm m_pullup ,
 .Nm m_split ,
-.Nm m_inject ,
+.Nm m_makespace ,
 .Nm m_getptr ,
 .Nm m_adj ,
 .Nm m_copyback ,
@@ -95,7 +95,7 @@
 .Ft struct mbuf *
 .Fn m_split "struct mbuf *m0" "int len0" "int wait"
 .Ft struct mbuf *
-.Fn m_inject "struct mbuf *m0" "int len0" "int siz" "int wait"
+.Fn m_makespace "struct mbuf *m0" "int skip" "int hlen" "int *off"
 .Ft struct mbuf *
 .Fn m_getptr "struct mbuf *m" "int loc" "int *off"
 .Ft void
@@ -568,18 +568,18 @@ Split an mbuf chain in two pieces, returning a pointer to
 the tail (which is made of the previous mbuf chain except the first
 .Fa len0
 bytes).
-.It Fn m_inject "struct mbuf *m0" "int len0" "int siz" "int wait"
-Inject a new mbuf chain of length
-.Fa siz
-into the mbuf chain pointed to by
-.Fa m0
-at position
-.Fa len0 .
-If there is enough space for an object of size
-.Fa siz
-in the appropriate location, no memory will be allocated.
-On failure, the function returns NULL (the mbuf is left untouched) and
-on success, a pointer to the first injected mbuf is returned.
+.It Fn m_makespace "struct mbuf *m0" "int skip" "int hlen" "int *off"
+Make space for a continuous memory region of length
+.Fa hlen
+at
+.Fa skip
+bytes into the mbuf chain.
+On success, the mbuf of the continuous memory is returned
+together with an offset
+.Fa off
+into the mbuf.
+On failure, NULL is returned and the mbuf chain may have been modified.
+The caller is assumed to always free the chain.
 .It Fn m_getptr "struct mbuf *m" "int loc" "int *off"
 Returns a pointer to the mbuf containing the data located at
 .Fa loc
diff --git a/sys/kern/uipc_mbuf.c b/sys/kern/uipc_mbuf.c
index bc8490f23e3..4eaad77104d 100644
--- a/sys/kern/uipc_mbuf.c
+++ b/sys/kern/uipc_mbuf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_mbuf.c,v 1.227 2016/09/03 14:17:37 bluhm Exp $	*/
+/*	$OpenBSD: uipc_mbuf.c,v 1.228 2016/09/13 19:56:55 markus Exp $	*/
 /*	$NetBSD: uipc_mbuf.c,v 1.15.4.1 1996/06/13 17:11:44 cgd Exp $	*/
 
 /*
@@ -967,67 +967,6 @@ m_getptr(struct mbuf *m, int loc, int *off)
 }
 
 /*
- * Inject a new mbuf chain of length siz in mbuf chain m0 at
- * position len0. Returns a pointer to the first injected mbuf, or
- * NULL on failure (m0 is left undisturbed). Note that if there is
- * enough space for an object of size siz in the appropriate position,
- * no memory will be allocated. Also, there will be no data movement in
- * the first len0 bytes (pointers to that will remain valid).
- *
- * XXX It is assumed that siz is less than the size of an mbuf at the moment.
- */
-struct mbuf *
-m_inject(struct mbuf *m0, int len0, int siz, int wait)
-{
-	struct mbuf *m, *n, *n2 = NULL, *n3;
-	unsigned len = len0, remain;
-
-	if ((siz >= MHLEN) || (len0 <= 0))
-		return (NULL);
-	for (m = m0; m && len > m->m_len; m = m->m_next)
-		len -= m->m_len;
-	if (m == NULL)
-		return (NULL);
-	remain = m->m_len - len;
-	if (remain == 0) {
-		if ((m->m_next) && (M_LEADINGSPACE(m->m_next) >= siz)) {
-			m->m_next->m_len += siz;
-			if (m0->m_flags & M_PKTHDR)
-				m0->m_pkthdr.len += siz;
-			m->m_next->m_data -= siz;
-			return m->m_next;
-		}
-	} else {
-		n2 = m_copym2(m, len, remain, wait);
-		if (n2 == NULL)
-			return (NULL);
-	}
-
-	MGET(n, wait, MT_DATA);
-	if (n == NULL) {
-		if (n2)
-			m_freem(n2);
-		return (NULL);
-	}
-
-	n->m_len = siz;
-	if (m0->m_flags & M_PKTHDR)
-		m0->m_pkthdr.len += siz;
-	m->m_len -= remain; /* Trim */
-	if (n2)	{
-		for (n3 = n; n3->m_next != NULL; n3 = n3->m_next)
-			;
-		n3->m_next = n2;
-	} else
-		n3 = n;
-	for (; n3->m_next != NULL; n3 = n3->m_next)
-		;
-	n3->m_next = m->m_next;
-	m->m_next = n;
-	return n;
-}
-
-/*
  * Partition an mbuf chain in two pieces, returning the tail --
  * all but the first len0 bytes.  In case of failure, it returns NULL and
  * attempts to restore the chain to its original state.
@@ -1094,6 +1033,121 @@ extpacket:
 }
 
 /*
+ * Make space for a new header of length hlen at skip bytes
+ * into the packet.  When doing this we allocate new mbufs only
+ * when absolutely necessary.  The mbuf where the new header
+ * is to go is returned together with an offset into the mbuf.
+ * If NULL is returned then the mbuf chain may have been modified;
+ * the caller is assumed to always free the chain.
+ */
+struct mbuf *
+m_makespace(struct mbuf *m0, int skip, int hlen, int *off)
+{
+	struct mbuf *m;
+	unsigned remain;
+
+	KASSERT(m0 != NULL);
+	KASSERT(hlen < MHLEN);
+
+	for (m = m0; m && skip > m->m_len; m = m->m_next)
+		skip -= m->m_len;
+	if (m == NULL)
+		return (NULL);
+	/*
+	 * At this point skip is the offset into the mbuf m
+	 * where the new header should be placed.  Figure out
+	 * if there's space to insert the new header.  If so,
+	 * and copying the remainder makese sense then do so.
+	 * Otherwise insert a new mbuf in the chain, splitting
+	 * the contents of m as needed.
+	 */
+	remain = m->m_len - skip;		/* data to move */
+	if (skip < remain && hlen <= M_LEADINGSPACE(m)) {
+		if (skip)
+			memmove(m->m_data-hlen, m->m_data, skip);
+		m->m_data -= hlen;
+		m->m_len += hlen;
+		(*off) = skip;
+	} else if (hlen > M_TRAILINGSPACE(m)) {
+		struct mbuf *n0, *n, **np;
+		int todo, len, done, alloc;
+
+		n0 = NULL;
+		np = &n0;
+		alloc = 0;
+		done = 0;
+		todo = remain;
+		while (todo > 0) {
+			MGET(n, M_DONTWAIT, m->m_type);
+			len = MHLEN;
+			if (n && todo > MHLEN) {
+				MCLGET(n, M_DONTWAIT);
+				len = MCLBYTES;
+				if ((n->m_flags & M_EXT) == 0) {
+					m_free(n);
+					n = NULL;
+				}
+			}
+			if (n == NULL) {
+				m_freem(n0);
+				return NULL;
+			}
+			*np = n;
+			np = &n->m_next;
+			alloc++;
+			len = min(todo, len);
+			memcpy(n->m_data, mtod(m, char *) + skip + done, len);
+			n->m_len = len;
+			done += len;
+			todo -= len;
+		}
+
+		if (hlen <= M_TRAILINGSPACE(m) + remain) {
+			m->m_len = skip + hlen;
+			*off = skip;
+			if (n0 != NULL) {
+				*np = m->m_next;
+				m->m_next = n0;
+			}
+		}
+		else {
+			n = m_get(M_DONTWAIT, m->m_type);
+			if (n == NULL) {
+				m_freem(n0);
+				return NULL;
+			}
+			alloc++;
+
+			if ((n->m_next = n0) == NULL)
+				np = &n->m_next;
+			n0 = n;
+
+			*np = m->m_next;
+			m->m_next = n0;
+
+			n->m_len = hlen;
+			m->m_len = skip;
+
+			m = n;			/* header is at front ... */
+			*off = 0;		/* ... of new mbuf */
+		}
+	} else {
+		/*
+		 * Copy the remainder to the back of the mbuf
+		 * so there's space to write the new header.
+		 */
+		if (remain > 0)
+			memmove(mtod(m, caddr_t) + skip + hlen,
+			      mtod(m, caddr_t) + skip, remain);
+		m->m_len += hlen;
+		*off = skip;
+	}
+	m0->m_pkthdr.len += hlen;		/* adjust packet length */
+	return m;
+}
+
+
+/*
  * Routine to copy from device local memory into mbufs.
  */
 struct mbuf *
diff --git a/sys/netinet/ip_ah.c b/sys/netinet/ip_ah.c
index 222a6ee7ffc..0d5adfc526e 100644
--- a/sys/netinet/ip_ah.c
+++ b/sys/netinet/ip_ah.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_ah.c,v 1.121 2016/08/18 06:01:10 dlg Exp $ */
+/*	$OpenBSD: ip_ah.c,v 1.122 2016/09/13 19:56:55 markus Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -932,7 +932,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 	struct mbuf *mi;
 	struct cryptop *crp;
 	u_int16_t iplen;
-	int len, rplen;
+	int len, rplen, roff;
 	u_int8_t prot;
 	struct ah *ah;
 #if NBPFILTER > 0
@@ -1057,7 +1057,7 @@ ah_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 	}
 
 	/* Inject AH header. */
-	mi = m_inject(m, skip, rplen + ahx->authsize, M_DONTWAIT);
+	mi = m_makespace(m, skip, rplen + ahx->authsize, &roff);
 	if (mi == NULL) {
 		DPRINTF(("ah_output(): failed to inject AH header for SA "
 		    "%s/%08x\n", ipsp_address(&tdb->tdb_dst, buf, sizeof(buf)),
@@ -1069,10 +1069,10 @@ ah_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 	}
 
 	/*
-	 * The AH header is guaranteed by m_inject() to be in
-	 * contiguous memory, at the beginning of the returned mbuf.
+	 * The AH header is guaranteed by m_makespace() to be in
+	 * contiguous memory, at 'roff' of the returned mbuf.
 	 */
-	ah = mtod(mi, struct ah *);
+	ah = (struct ah *)(mtod(mi, caddr_t) + roff);
 
 	/* Initialize the AH header. */
 	m_copydata(m, protoff, sizeof(u_int8_t), (caddr_t) &ah->ah_nh);
diff --git a/sys/netinet/ip_esp.c b/sys/netinet/ip_esp.c
index 2c7b988e73f..48408e316e1 100644
--- a/sys/netinet/ip_esp.c
+++ b/sys/netinet/ip_esp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ip_esp.c,v 1.139 2016/08/18 06:01:10 dlg Exp $ */
+/*	$OpenBSD: ip_esp.c,v 1.140 2016/09/13 19:56:55 markus Exp $ */
 /*
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
@@ -775,7 +775,7 @@ esp_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 {
 	struct enc_xform *espx = (struct enc_xform *) tdb->tdb_encalgxform;
 	struct auth_hash *esph = (struct auth_hash *) tdb->tdb_authalgxform;
-	int ilen, hlen, rlen, padding, blks, alen;
+	int ilen, hlen, rlen, padding, blks, alen, roff;
 	u_int32_t replay;
 	struct mbuf *mi, *mo = (struct mbuf *) NULL;
 	struct tdb_crypto *tc;
@@ -907,7 +907,7 @@ esp_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 	}
 
 	/* Inject ESP header. */
-	mo = m_inject(m, skip, hlen, M_DONTWAIT);
+	mo = m_makespace(m, skip, hlen, &roff);
 	if (mo == NULL) {
 		DPRINTF(("esp_output(): failed to inject ESP header for "
 		    "SA %s/%08x\n", ipsp_address(&tdb->tdb_dst, buf,
@@ -918,10 +918,11 @@ esp_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 	}
 
 	/* Initialize ESP header. */
-	bcopy((caddr_t) &tdb->tdb_spi, mtod(mo, caddr_t), sizeof(u_int32_t));
+	bcopy((caddr_t) &tdb->tdb_spi, mtod(mo, caddr_t) + roff,
+	    sizeof(u_int32_t));
 	tdb->tdb_rpl++;
 	replay = htonl((u_int32_t)tdb->tdb_rpl);
-	bcopy((caddr_t) &replay, mtod(mo, caddr_t) + sizeof(u_int32_t),
+	bcopy((caddr_t) &replay, mtod(mo, caddr_t) + roff + sizeof(u_int32_t),
 	    sizeof(u_int32_t));
 
 #if NPFSYNC > 0
@@ -932,15 +933,15 @@ esp_output(struct mbuf *m, struct tdb *tdb, struct mbuf **mp, int skip,
 	 * Add padding -- better to do it ourselves than use the crypto engine,
 	 * although if/when we support compression, we'd have to do that.
 	 */
-	mo = m_inject(m, m->m_pkthdr.len, padding + alen, M_DONTWAIT);
+	mo = m_makespace(m, m->m_pkthdr.len, padding + alen, &roff);
 	if (mo == NULL) {
-		DPRINTF(("esp_output(): m_inject failed for SA %s/%08x\n",
+		DPRINTF(("esp_output(): m_makespace() failed for SA %s/%08x\n",
 		    ipsp_address(&tdb->tdb_dst, buf, sizeof(buf)),
 		    ntohl(tdb->tdb_spi)));
 		m_freem(m);
 		return ENOBUFS;
 	}
-	pad = mtod(mo, u_char *);
+	pad = mtod(mo, caddr_t) + roff;
 
 	/* Apply self-describing padding */
 	for (ilen = 0; ilen < padding - 2; ilen++)
diff --git a/sys/netinet/ip_ipcomp.c b/sys/netinet/ip_ipcomp.c
index 120511bf37e..721fe0f40e0 100644
--- a/sys/netinet/ip_ipcomp.c
+++ b/sys/netinet/ip_ipcomp.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: ip_ipcomp.c,v 1.46 2016/08/18 06:01:10 dlg Exp $ */
+/* $OpenBSD: ip_ipcomp.c,v 1.47 2016/09/13 19:56:55 markus Exp $ */
 
 /*
  * Copyright (c) 2001 Jean-Jacques Bernard-Gundol (jj@wabbitt.org)
@@ -540,7 +540,7 @@ ipcomp_output_cb(struct cryptop *crp)
 	struct tdb_crypto *tc;
 	struct tdb *tdb;
 	struct mbuf *m, *mo;
-	int error, s, skip, rlen;
+	int error, s, skip, rlen, roff;
 	u_int16_t cpi;
 	struct ip *ip;
 #ifdef INET6
@@ -605,7 +605,7 @@ ipcomp_output_cb(struct cryptop *crp)
 	}
 
 	/* Inject IPCOMP header */
-	mo = m_inject(m, skip, IPCOMP_HLENGTH, M_DONTWAIT);
+	mo = m_makespace(m, skip, IPCOMP_HLENGTH, &roff);
 	if (mo == NULL) {
 		DPRINTF(("ipcomp_output_cb(): failed to inject IPCOMP header "
 		    "for IPCA %s/%08x\n", ipsp_address(&tdb->tdb_dst, buf,
@@ -616,7 +616,7 @@ ipcomp_output_cb(struct cryptop *crp)
 	}
 
 	/* Initialize the IPCOMP header */
-	ipcomp = mtod(mo, struct ipcomp *);
+	ipcomp = (struct ipcomp *)(mtod(mo, caddr_t) + roff);
 	memset(ipcomp, 0, sizeof(struct ipcomp));
 	cpi = (u_int16_t) ntohl(tdb->tdb_spi);
 	ipcomp->ipcomp_cpi = htons(cpi);
diff --git a/sys/netinet/ipsec_output.c b/sys/netinet/ipsec_output.c
index 10119faccc9..532d7daf607 100644
--- a/sys/netinet/ipsec_output.c
+++ b/sys/netinet/ipsec_output.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: ipsec_output.c,v 1.62 2016/02/28 16:16:10 mikeb Exp $ */
+/*	$OpenBSD: ipsec_output.c,v 1.63 2016/09/13 19:56:55 markus Exp $ */
 /*
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
@@ -362,13 +362,12 @@ int
 ipsp_process_done(struct mbuf *m, struct tdb *tdb)
 {
 	struct ip *ip;
-
 #ifdef INET6
 	struct ip6_hdr *ip6;
 #endif /* INET6 */
-
 	struct tdb_ident *tdbi;
 	struct m_tag *mtag;
+	int roff;
 
 	tdb->tdb_last_used = time_second;
 
@@ -398,12 +397,12 @@ ipsp_process_done(struct mbuf *m, struct tdb *tdb)
 			return ENXIO;
 		}
 
-		mi = m_inject(m, iphlen, sizeof(struct udphdr), M_DONTWAIT);
+		mi = m_makespace(m, iphlen, sizeof(struct udphdr), &roff);
 		if (mi == NULL) {
 			m_freem(m);
 			return ENOMEM;
 		}
-		uh = mtod(mi, struct udphdr *);
+		uh = (struct udphdr *)(mtod(mi, caddr_t) + roff);
 		uh->uh_sport = uh->uh_dport = htons(udpencap_port);
 		if (tdb->tdb_udpencap_port)
 			uh->uh_dport = tdb->tdb_udpencap_port;
diff --git a/sys/sys/mbuf.h b/sys/sys/mbuf.h
index 0cf95922ba0..5184713868c 100644
--- a/sys/sys/mbuf.h
+++ b/sys/sys/mbuf.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: mbuf.h,v 1.217 2016/09/03 13:42:28 reyk Exp $	*/
+/*	$OpenBSD: mbuf.h,v 1.218 2016/09/13 19:56:55 markus Exp $	*/
 /*	$NetBSD: mbuf.h,v 1.19 1996/02/09 18:25:14 christos Exp $	*/
 
 /*
@@ -436,7 +436,7 @@ struct	mbuf *m_prepend(struct mbuf *, int, int);
 struct	mbuf *m_pulldown(struct mbuf *, int, int, int *);
 struct	mbuf *m_pullup(struct mbuf *, int);
 struct	mbuf *m_split(struct mbuf *, int, int);
-struct  mbuf *m_inject(struct mbuf *, int, int, int);
+struct	mbuf *m_makespace(struct mbuf *, int, int, int *);
 struct  mbuf *m_getptr(struct mbuf *, int, int *);
 int	m_leadingspace(struct mbuf *);
 int	m_trailingspace(struct mbuf *);