summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/Makefile10
-rwxr-xr-xsrc/clean_patch.sh3
-rw-r--r--src/if_wg.c127
-rw-r--r--src/mpq.h171
4 files changed, 195 insertions, 116 deletions
diff --git a/src/Makefile b/src/Makefile
index deda3be6250..5d37d9c35c2 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -23,9 +23,8 @@ patch_ifconfig: /usr/src/sbin/ifconfig/ifconfig.c /usr/src/sbin/ifconfig/Makefil
ifconfig: patch_ifconfig
make -C /usr/src/sbin/ifconfig
-/usr/src/sys/net/if_wg.c: if_wg.c mpq.c
- #cp if_wg.c /usr/src/sys/net/if_wg.c
- cat if_wg.c mpq.c > /usr/src/sys/net/if_wg.c
+/usr/src/sys/net/if_wg.c: if_wg.c
+ cp if_wg.c /usr/src/sys/net/if_wg.c
/usr/src/sys/net/if_wg.h: if_wg.h
cp if_wg.h /usr/src/sys/net/if_wg.h
@@ -51,9 +50,6 @@ ifconfig: patch_ifconfig
/usr/src/sys/sys/mpq.h: mpq.h
cp mpq.h /usr/src/sys/sys/mpq.h
-/usr/src/sys/sys/mpq2.h: mpq2.h
- cp mpq2.h /usr/src/sys/sys/mpq2.h
-
/usr/src/sys/sys/hashmap.h: hashmap.h
cp hashmap.h /usr/src/sys/sys/hashmap.h
@@ -96,7 +92,7 @@ ifconfig: patch_ifconfig
patch -uN /usr/src/distrib/sets/lists/comp/mi < patches/mi.patch || touch /usr/src/distrib/sets/lists/comp/mi
.PHONY:
-patch_kernel: /usr/src/sys/net/wireguard.c /usr/src/sys/net/wireguard.h /usr/src/sys/crypto/blake2s.c /usr/src/sys/crypto/blake2s.h /usr/src/sys/crypto/curve25519.c /usr/src/sys/crypto/curve25519.h /usr/src/sys/crypto/chacha_private.h /usr/src/sys/crypto/chachapoly.h /usr/src/sys/crypto/chachapoly.c /usr/src/sys/conf/files /usr/src/sys/conf/GENERIC /usr/src/sys/netinet/in_pcb.h /usr/src/sys/netinet/udp_usrreq.c /usr/src/sys/net/if_wg.c /usr/src/sys/net/if_wg.h /usr/src/sys/sys/mpq.h /usr/src/sys/sys/hashmap.h /usr/src/sys/sys/mpq2.h
+patch_kernel: /usr/src/sys/net/wireguard.c /usr/src/sys/net/wireguard.h /usr/src/sys/crypto/blake2s.c /usr/src/sys/crypto/blake2s.h /usr/src/sys/crypto/curve25519.c /usr/src/sys/crypto/curve25519.h /usr/src/sys/crypto/chacha_private.h /usr/src/sys/crypto/chachapoly.h /usr/src/sys/crypto/chachapoly.c /usr/src/sys/conf/files /usr/src/sys/conf/GENERIC /usr/src/sys/netinet/in_pcb.h /usr/src/sys/netinet/udp_usrreq.c /usr/src/sys/net/if_wg.c /usr/src/sys/net/if_wg.h /usr/src/sys/sys/mpq.h /usr/src/sys/sys/hashmap.h
.PHONY:
patch_userspace: /usr/src/usr.bin/kdump/mkioctls /usr/src/usr.bin/kdump/Makefile /usr/src/distrib/sets/lists/comp/mi
diff --git a/src/clean_patch.sh b/src/clean_patch.sh
index f2fa0d5f93b..7160676ef32 100755
--- a/src/clean_patch.sh
+++ b/src/clean_patch.sh
@@ -25,6 +25,3 @@ cp $DIR/mpq.h /usr/src/sys/sys/
cp $DIR/hashmap.h /usr/src/sys/sys/
cp $DIR/if_wg.* $DIR/wireguard.* /usr/src/sys/net/
cp $DIR/blake2s.* $DIR/curve25519.* /usr/src/sys/crypto/
-
-# Crappy temp hack so don't need to add new file
-cat $DIR/mpq.c >> /usr/src/sys/net/if_wg.c
diff --git a/src/if_wg.c b/src/if_wg.c
index 73e7e018051..cdf91796f87 100644
--- a/src/if_wg.c
+++ b/src/if_wg.c
@@ -30,8 +30,6 @@
#include <sys/rwlock.h>
#include <sys/protosw.h>
#include <sys/hashmap.h>
-#include <sys/mpq.h>
-#include <sys/mpq2.h>
#include <net/if.h>
#include <net/if_var.h>
@@ -90,6 +88,8 @@ struct wg_pkt {
} p_state;
};
+#include <sys/mpq.h>
+
struct wg_peer_item {
u_int8_t key[WG_KEY_SIZE];
struct wg_peer *peer;
@@ -312,7 +312,7 @@ wg_peer_queue_flush(void *_sc, struct wg_conn *c)
{
struct wg_softc *sc = _sc;
struct wg_peer *p = wg_peer_lookup(sc, c->c_pubkey);
- struct wg_pkt *pkt;
+ struct mpq_item i;
struct mbuf *m;
wg_timer_reinit_unflag(&p->p_timers);
@@ -322,22 +322,15 @@ wg_peer_queue_flush(void *_sc, struct wg_conn *c)
wg_want_keepalive(p);
while ((m = mq_dequeue(&p->p_outgoing)) != NULL) {
- pkt = malloc(sizeof(*pkt), M_DEVBUF, M_NOWAIT);
- if (pkt == NULL) {
- m_free(m);
- continue;
- }
-
- pkt->p_p = p;
- pkt->p_m = m;
- pkt->p_sc = sc;
- pkt->p_state = WG_PKT_CLEAR;
- pkt->p_type = WG_PKT_TRANSPORT;
+ i.pkt.p_p = p;
+ i.pkt.p_m = m;
+ i.pkt.p_sc = sc;
+ i.pkt.p_state = WG_PKT_CLEAR;
+ i.pkt.p_type = WG_PKT_TRANSPORT;
- if (mpq_add(&sc->sc_txq, pkt) != 0) {
- m_free(m);
+ if (mpq_add(&sc->sc_txq, &i) != 0) {
counters_inc(sc->sc_if.if_counters, ifc_oqdrops);
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
+ m_free(m);
}
}
}
@@ -384,26 +377,20 @@ void
wg_send_pkt(struct wg_peer *p, enum wg_pkt_type t)
{
struct wg_softc *sc = p->p_sc;
- struct wg_pkt *pkt = malloc(sizeof(*pkt), M_DEVBUF, M_NOWAIT);
- if (pkt == NULL)
+ struct mpq_item i;
+ i.pkt.p_state = WG_PKT_CLEAR;
+ i.pkt.p_sc = p->p_sc;
+ i.pkt.p_type = t;
+ i.pkt.p_p = p;
+ i.pkt.p_m = m_clget(NULL, M_NOWAIT, 1000);
+ if (i.pkt.p_m == NULL)
return;
- pkt->p_state = WG_PKT_CLEAR;
- pkt->p_sc = p->p_sc;
- pkt->p_type = t;
- pkt->p_p = p;
- pkt->p_m = m_clget(NULL, M_NOWAIT, 1000);
- if (pkt->p_m == NULL) {
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
- return;
- }
-
- pkt->p_m->m_len = 0;
- m_calchdrlen(pkt->p_m);
+ i.pkt.p_m->m_len = 0;
+ m_calchdrlen(i.pkt.p_m);
- if (mpq_add(&sc->sc_txq, pkt) != 0) {
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
- }
+ if (mpq_add(&sc->sc_txq, &i) != 0)
+ m_freem(i.pkt.p_m);
}
void wg_timer_reinit_flag(struct wg_timers *t);
@@ -734,7 +721,7 @@ struct mbuf *
wg_input(void *_sc, struct mbuf *m, struct sockaddr *sa, int hlen)
{
struct wg_softc *sc = _sc;
- struct wg_pkt *pkt;
+ struct mpq_item i;
rw_enter_read(&sc->sc_lock);
@@ -746,38 +733,29 @@ wg_input(void *_sc, struct mbuf *m, struct sockaddr *sa, int hlen)
if (m_defrag(m, M_NOWAIT))
goto exit;
- pkt = malloc(sizeof(*pkt), M_DEVBUF, M_NOWAIT);
+ i.pkt.p_m = m;
+ i.pkt.p_sc = sc;
+ i.pkt.p_state = WG_PKT_CRYPTED;
+ i.pkt.p_type = wg_util_pkt_type(mtod(m, uint8_t *), m->m_pkthdr.len);
- if (pkt == NULL) {
- counters_inc(sc->sc_if.if_counters, ifc_ierrors);
+ if (i.pkt.p_type == WG_PKT_UNKNOWN)
goto free;
- }
-
- pkt->p_m = m;
- pkt->p_sc = sc;
- pkt->p_state = WG_PKT_CRYPTED;
- pkt->p_type = wg_util_pkt_type(mtod(m, uint8_t *), m->m_pkthdr.len);
- if (pkt->p_type == WG_PKT_UNKNOWN)
- goto drop;
-
- if (mpq_add(&sc->sc_rxq, pkt) == 0)
+ if (mpq_add(&sc->sc_rxq, &i) == 0)
goto exit;
-drop:
- counters_inc(sc->sc_if.if_counters, ifc_iqdrops);
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
free:
m_free(m);
+ counters_inc(sc->sc_if.if_counters, ifc_ierrors);
exit:
rw_exit_read(&sc->sc_lock);
return NULL;
}
void
-wg_input_worker(void *_pkt)
+wg_input_worker(struct mpq_item *i)
{
- struct wg_pkt *pkt = _pkt;
+ struct wg_pkt *pkt = &i->pkt;
rw_enter_read(&pkt->p_sc->sc_lock);
wg_pkt_handlers[pkt->p_type][0](pkt);
rw_exit_read(&pkt->p_sc->sc_lock);
@@ -788,9 +766,9 @@ wg_input_worker(void *_pkt)
}
void
-wg_input_serial(void *_pkt)
+wg_input_serial(struct mpq_item *i)
{
- struct wg_pkt *pkt = _pkt;
+ struct wg_pkt *pkt = &i->pkt;
NET_RLOCK();
rw_enter_read(&pkt->p_sc->sc_lock);
@@ -811,14 +789,12 @@ wg_input_serial(void *_pkt)
}
rw_exit_read(&pkt->p_sc->sc_lock);
NET_RUNLOCK();
-
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
}
void
-wg_output_worker(void *_pkt)
+wg_output_worker(struct mpq_item *i)
{
- struct wg_pkt *pkt = _pkt;
+ struct wg_pkt *pkt = &i->pkt;
rw_enter_read(&pkt->p_p->p_sc->sc_lock);
wg_pkt_handlers[pkt->p_type][1](pkt);
rw_exit_read(&pkt->p_p->p_sc->sc_lock);
@@ -829,10 +805,10 @@ wg_output_worker(void *_pkt)
}
void
-wg_output_serial(void *_pkt)
+wg_output_serial(struct mpq_item *i)
{
struct mbuf peernam;
- struct wg_pkt *pkt = _pkt;
+ struct wg_pkt *pkt = &i->pkt;
NET_RLOCK();
rw_enter_read(&pkt->p_sc->sc_lock);
@@ -865,8 +841,6 @@ wg_output_serial(void *_pkt)
}
rw_exit_read(&pkt->p_sc->sc_lock);
NET_RUNLOCK();
-
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
}
int
@@ -920,25 +894,18 @@ wg_output(struct ifnet *ifp, struct mbuf *m, struct sockaddr *sa, struct rtentry
goto exit;
}
- struct wg_pkt *pkt = malloc(sizeof(*pkt), M_DEVBUF, M_NOWAIT);
- if (pkt == NULL) {
- counters_inc(sc->sc_if.if_counters, ifc_oerrors);
- error = ENOBUFS;
- goto free;
- }
-
- pkt->p_p = p;
- pkt->p_m = m;
- pkt->p_sc = sc;
- pkt->p_state = WG_PKT_CLEAR;
- pkt->p_type = WG_PKT_TRANSPORT;
+ struct mpq_item i;
+ i.pkt.p_p = p;
+ i.pkt.p_m = m;
+ i.pkt.p_sc = sc;
+ i.pkt.p_state = WG_PKT_CLEAR;
+ i.pkt.p_type = WG_PKT_TRANSPORT;
- if (mpq_add(&sc->sc_txq, pkt) == 0) {
+ if (mpq_add(&sc->sc_txq, &i) == 0) {
goto exit;
} else {
error = ENOBUFS;
counters_inc(sc->sc_if.if_counters, ifc_oqdrops);
- free(pkt, M_DEVBUF, sizeof(struct wg_pkt));
}
free:
m_free(m);
@@ -991,8 +958,10 @@ wg_softc_create()
HASHMAP_INIT(&sc->sc_id_hashmap, 0, wg_id_cmp, wg_id_hash);
HASHMAP_INIT(&sc->sc_peer_hashmap, 0, wg_peer_cmp, wg_peer_hash);
- mpq_init(&sc->sc_txq, wg_taskq, IPL_NET, wg_output_worker, wg_output_serial, 256);
- mpq_init(&sc->sc_rxq, wg_taskq, IPL_NET, wg_input_worker, wg_input_serial, 256);
+ mpq_init(&sc->sc_txq, IPL_NET, 2, wg_output_worker, wg_output_serial);
+ mpq_resize(&sc->sc_txq, 1024);
+ mpq_init(&sc->sc_rxq, IPL_NET, 2, wg_input_worker, wg_input_serial);
+ mpq_resize(&sc->sc_rxq, 1024);
rw_init(&sc->sc_lock, "wg");
diff --git a/src/mpq.h b/src/mpq.h
index ff401b0c266..5a861ca08b5 100644
--- a/src/mpq.h
+++ b/src/mpq.h
@@ -2,42 +2,159 @@
#define _MPQ_H_
#include <sys/mutex.h>
-#include <sys/task.h>
-#include <sys/queue.h>
+#include <sys/kthread.h>
struct mpq_item {
- TAILQ_ENTRY(mpq_item) i_worker;
- TAILQ_ENTRY(mpq_item) i_serial;
- struct cond i_cond;
- enum mpq_istate {
- MPQ_ITEM_WAITING,
- MPQ_ITEM_PROCESSED,
- MPQ_ITEM_BARRIER,
- } i_state;
- void *i_val;
+ struct wg_pkt pkt;
};
struct mpq {
- struct mutex q_mtx;
- struct taskq *q_taskq;
- struct rwlock q_serial_lock;
+ struct mutex mtx;
+ int destroy;
+ void (*parallel)(struct mpq_item *);
+ void (*serial)(struct mpq_item *);
+ size_t parallel_head, serial_head, tail, len;
+ struct mpq_queue_item {
+ enum mpq_item_state {
+ MPQ_ITEM_NONE = 0,
+ MPQ_ITEM_WAITING,
+ MPQ_ITEM_PROCESSING,
+ MPQ_ITEM_DONE,
+ } state;
+ struct mpq_item data;
+ } *items;
+};
- size_t q_max;
- size_t q_len;
+/*
+ * H = Head, T = Tail, N = len
+ * H T T-H N-(H-T)
+ * 5 5 0 8
+ * 5 6 1 9
+ * 5 7 2 10
+ * 5 0 -5 3
+ * 5 1 -4 4
+ * 5 2 -3 5
+ * 5 3 -2 6
+ * 5 4 -1 7
+*/
- struct task q_worker_task;
- struct task q_serial_task;
+#define INC_WRAP(v, max) (v) = ((v) >= (max - 1) ? 0 : (v) + 1)
+#define CIRC_LEN(h, t, n) ((t) >= (h) ? ((t) - (h)) : ((n) - ((h) - (t))))
- void (*q_worker_fn)(void *);
- void (*q_serial_fn)(void *);
+#define MPQ_SERIAL_LEN(q) CIRC_LEN((q)->serial_head, (q)->tail, (q)->len)
+#define MPQ_SERIAL_FULL(q) (MPQ_SERIAL_LEN(q) >= ((q)->len - 1))
+#define MPQ_SERIAL_EMPTY(q) (MPQ_SERIAL_LEN(q) == 0)
- TAILQ_HEAD(, mpq_item) q_worker_items;
- TAILQ_HEAD(, mpq_item) q_serial_items;
-};
+#define MPQ_PARALLEL_LEN(q) CIRC_LEN((q)->parallel_head, (q)->tail, (q)->len)
+#define MPQ_PARALLEL_FULL(q) (MPQ_PARALLEL_LEN(q) >= ((q)->len - 1))
+#define MPQ_PARALLEL_EMPTY(q) (MPQ_PARALLEL_LEN(q) == 0)
+
+
+void mpq_init(struct mpq *, int ipl, uint8_t, void (*)(struct mpq_item *),
+ void (*)(struct mpq_item *));
+int mpq_resize(struct mpq *, size_t);
+int mpq_add(struct mpq *, struct mpq_item *);
+void mpq_destroy(struct mpq *);
+
+void
+mpq_parallel_worker(void *_q)
+{
+ struct mpq *q = _q;
+ struct mpq_queue_item *i;
+ while (1) {
+ while (MPQ_PARALLEL_EMPTY(q)) {
+ tsleep(&q->tail, PZERO, "bored", 0);
+ }
+
+ mtx_enter(&q->mtx);
+ i = &q->items[q->parallel_head];
+ if (i->state != MPQ_ITEM_WAITING) {
+ mtx_leave(&q->mtx);
+ continue;
+ }
+
+ INC_WRAP(q->parallel_head, q->len);
+ i->state = MPQ_ITEM_PROCESSING;
+ mtx_leave(&q->mtx);
+
+ q->parallel(&i->data);
+
+ mtx_enter(&q->mtx);
+ i->state = MPQ_ITEM_DONE;
+ mtx_leave(&q->mtx);
+
+ wakeup_one(&q->parallel_head);
+ }
+}
+
+void
+mpq_serial_worker(void *_q)
+{
+ struct mpq *q = _q;
+ struct mpq_queue_item *i;
+ while (1) {
+ while (q->items[q->serial_head].state != MPQ_ITEM_DONE) {
+ tsleep(&q->parallel_head, PZERO, "bored", 0);
+ }
+
+ i = &q->items[q->serial_head];
+
+ q->serial(&i->data);
+
+ INC_WRAP(q->serial_head, q->len);
+
+ mtx_enter(&q->mtx);
+ i->state = MPQ_ITEM_NONE;
+ mtx_leave(&q->mtx);
+ }
+}
+
+int
+mpq_add(struct mpq *q, struct mpq_item *i)
+{
+ mtx_enter(&q->mtx);
+ if (MPQ_SERIAL_FULL(q)) {
+ mtx_leave(&q->mtx);
+ return 1;
+ }
+
+ q->items[q->tail].data = *i;
+ q->items[q->tail].state = MPQ_ITEM_WAITING;
+ INC_WRAP(q->tail, q->len);
+ mtx_leave(&q->mtx);
+ wakeup_one(&q->tail);
+ return 0;
+}
+
+void
+mpq_init(struct mpq *q, int ipl, uint8_t nthreads, void (*parallel)(struct mpq_item *), void(*serial)(struct mpq_item *))
+{
+ mtx_init(&q->mtx, ipl);
+ q->parallel = parallel;
+ q->serial = serial;
+ q->destroy = 0;
+ q->parallel_head = 0;
+ q->serial_head = 0;
+ q->tail = 0;
+ q->len = 0;
+
+ kthread_create(mpq_serial_worker, q, NULL, "mpq_serial");
+ while(nthreads--)
+ kthread_create(mpq_parallel_worker, q, NULL, "mpq_parallel");
+}
+
+int
+mpq_resize(struct mpq *q, size_t len)
+{
+ q->len = len;
+ q->items = mallocarray(q->len, sizeof(struct mpq_queue_item), M_DEVBUF, M_WAIT | M_ZERO);
+ return 0;
+}
-void mpq_init(struct mpq *, struct taskq *, int ipl, void (*)(void *), void (*)(void *), size_t);
-int mpq_add(struct mpq *, void *);
-int mpq_active(struct mpq *);
-size_t mpq_len(struct mpq *);
+void
+mpq_destroy(struct mpq *q)
+{
+ q->destroy = 1;
+}
#endif /* _MPQ_H_ */