global: use ck for loads/stores, rather than macro maze

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

4 files changed, 70 insertions, 86 deletions

diff --git a/TODO.md b/TODO.md
index cf91982..a69ac25 100644
--- a/TODO.md
+++ b/TODO.md
@@ -13,7 +13,6 @@
 - Run ratelimiter gc in a properly scheduled manner.
 - Make sure noise state machine is correct.
 - Clear mbuf tags and other members properly.
-- Switch `WRITE_ONCE` and `epoch_ptr_*` functions to use the actual helper from `ck_`.
 
 ### Crypto TODO
 
diff --git a/src/if_wg.c b/src/if_wg.c
index 4a91f59..f94a954 100644
--- a/src/if_wg.c
+++ b/src/if_wg.c
@@ -778,10 +778,10 @@ wg_socket_set(struct wg_softc *sc, struct socket *new_so4, struct socket *new_so
 
 	sx_assert(&sc->sc_lock, SX_XLOCKED);
 
-	so4 = atomic_load_ptr(&so->so_so4);
-	so6 = atomic_load_ptr(&so->so_so6);
-	atomic_store_ptr(&so->so_so4, new_so4);
-	atomic_store_ptr(&so->so_so6, new_so6);
+	so4 = ck_pr_load_ptr(&so->so_so4);
+	so6 = ck_pr_load_ptr(&so->so_so6);
+	ck_pr_store_ptr(&so->so_so4, new_so4);
+	ck_pr_store_ptr(&so->so_so6, new_so6);
 
 	if (!so4 && !so6)
 		return;
@@ -884,8 +884,8 @@ wg_send(struct wg_softc *sc, struct wg_endpoint *e, struct mbuf *m)
 	sa = &e->e_remote.r_sa;
 
 	NET_EPOCH_ENTER(et);
-	so4 = atomic_load_ptr(&so->so_so4);
-	so6 = atomic_load_ptr(&so->so_so6);
+	so4 = ck_pr_load_ptr(&so->so_so4);
+	so6 = ck_pr_load_ptr(&so->so_so6);
 	if (e->e_remote.r_sa.sa_family == AF_INET && so4 != NULL)
 		ret = sosend(so4, sa, NULL, m, control, 0, curthread);
 	else if (e->e_remote.r_sa.sa_family == AF_INET6 && so6 != NULL)
@@ -932,16 +932,16 @@ retry:
 static void
 wg_timers_enable(struct wg_peer *peer)
 {
-	WRITE_ONCE(peer->p_enabled, true);
+	ck_pr_store_bool(&peer->p_enabled, true);
 	wg_timers_run_persistent_keepalive(peer);
 }
 
 static void
 wg_timers_disable(struct wg_peer *peer)
 {
-	WRITE_ONCE(peer->p_enabled, false);
+	ck_pr_store_bool(&peer->p_enabled, false);
 	NET_EPOCH_WAIT();
-	WRITE_ONCE(peer->p_need_another_keepalive, false);
+	ck_pr_store_bool(&peer->p_need_another_keepalive, false);
 
 	callout_stop(&peer->p_new_handshake);
 	callout_stop(&peer->p_send_keepalive);
@@ -955,9 +955,9 @@ wg_timers_set_persistent_keepalive(struct wg_peer *peer, uint16_t interval)
 {
 	struct epoch_tracker et;
 	if (interval != peer->p_persistent_keepalive_interval) {
-		WRITE_ONCE(peer->p_persistent_keepalive_interval, interval);
+		ck_pr_store_16(&peer->p_persistent_keepalive_interval, interval);
 		NET_EPOCH_ENTER(et);
-		if (peer->p_enabled)
+		if (ck_pr_load_bool(&peer->p_enabled))
 			wg_timers_run_persistent_keepalive(peer);
 		NET_EPOCH_EXIT(et);
 	}
@@ -977,7 +977,7 @@ wg_timers_event_data_sent(struct wg_peer *peer)
 {
 	struct epoch_tracker et;
 	NET_EPOCH_ENTER(et);
-	if (peer->p_enabled && !callout_pending(&peer->p_new_handshake))
+	if (ck_pr_load_bool(&peer->p_enabled) && !callout_pending(&peer->p_new_handshake))
 		callout_reset(&peer->p_new_handshake, MSEC_2_TICKS(
 		    NEW_HANDSHAKE_TIMEOUT * 1000 +
 		    arc4random_uniform(REKEY_TIMEOUT_JITTER)),
@@ -990,13 +990,13 @@ wg_timers_event_data_received(struct wg_peer *peer)
 {
 	struct epoch_tracker et;
 	NET_EPOCH_ENTER(et);
-	if (peer->p_enabled) {
+	if (ck_pr_load_bool(&peer->p_enabled)) {
 		if (!callout_pending(&peer->p_send_keepalive))
 			callout_reset(&peer->p_send_keepalive,
 			    MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000),
 			    wg_timers_run_send_keepalive, peer);
 		else
-			WRITE_ONCE(peer->p_need_another_keepalive, true);
+			ck_pr_store_bool(&peer->p_need_another_keepalive, true);
 	}
 	NET_EPOCH_EXIT(et);
 }
@@ -1019,8 +1019,8 @@ wg_timers_event_any_authenticated_packet_traversal(struct wg_peer *peer)
 	struct epoch_tracker et;
 	uint16_t interval;
 	NET_EPOCH_ENTER(et);
-	interval = READ_ONCE(peer->p_persistent_keepalive_interval);
-	if (peer->p_enabled && interval > 0)
+	interval = ck_pr_load_16(&peer->p_persistent_keepalive_interval);
+	if (ck_pr_load_bool(&peer->p_enabled) && interval > 0)
 		callout_reset(&peer->p_persistent_keepalive,
 		     MSEC_2_TICKS(interval * 1000),
 		     wg_timers_run_persistent_keepalive, peer);
@@ -1032,7 +1032,7 @@ wg_timers_event_handshake_initiated(struct wg_peer *peer)
 {
 	struct epoch_tracker et;
 	NET_EPOCH_ENTER(et);
-	if (peer->p_enabled)
+	if (ck_pr_load_bool(&peer->p_enabled))
 		callout_reset(&peer->p_retry_handshake, MSEC_2_TICKS(
 		    REKEY_TIMEOUT * 1000 +
 		    arc4random_uniform(REKEY_TIMEOUT_JITTER)),
@@ -1045,7 +1045,7 @@ wg_timers_event_handshake_complete(struct wg_peer *peer)
 {
 	struct epoch_tracker et;
 	NET_EPOCH_ENTER(et);
-	if (peer->p_enabled) {
+	if (ck_pr_load_bool(&peer->p_enabled)) {
 		mtx_lock(&peer->p_handshake_mtx);
 		callout_stop(&peer->p_retry_handshake);
 		peer->p_handshake_retries = 0;
@@ -1061,7 +1061,7 @@ wg_timers_event_session_derived(struct wg_peer *peer)
 {
 	struct epoch_tracker et;
 	NET_EPOCH_ENTER(et);
-	if (peer->p_enabled)
+	if (ck_pr_load_bool(&peer->p_enabled))
 		callout_reset(&peer->p_zero_key_material,
 		    MSEC_2_TICKS(REJECT_AFTER_TIME * 3 * 1000),
 		    wg_timers_run_zero_key_material, peer);
@@ -1073,7 +1073,7 @@ wg_timers_event_want_initiation(struct wg_peer *peer)
 {
 	struct epoch_tracker et;
 	NET_EPOCH_ENTER(et);
-	if (peer->p_enabled)
+	if (ck_pr_load_bool(&peer->p_enabled))
 		wg_timers_run_send_initiation(peer, 0);
 	NET_EPOCH_EXIT(et);
 }
@@ -1124,8 +1124,8 @@ wg_timers_run_send_keepalive(void *_peer)
 	struct wg_peer *peer = _peer;
 
 	wg_send_keepalive(peer);
-	if (READ_ONCE(peer->p_need_another_keepalive)) {
-		WRITE_ONCE(peer->p_need_another_keepalive, false);
+	if (ck_pr_load_bool(&peer->p_need_another_keepalive)) {
+		ck_pr_store_bool(&peer->p_need_another_keepalive, false);
 		callout_reset(&peer->p_send_keepalive,
 		    MSEC_2_TICKS(KEEPALIVE_TIMEOUT * 1000),
 		    wg_timers_run_send_keepalive, peer);
@@ -1161,7 +1161,7 @@ wg_timers_run_persistent_keepalive(void *_peer)
 {
 	struct wg_peer *peer = _peer;
 
-	if (READ_ONCE(peer->p_persistent_keepalive_interval) > 0)
+	if (ck_pr_load_16(&peer->p_persistent_keepalive_interval) > 0)
 		wg_send_keepalive(peer);
 }
 
diff --git a/src/support.h b/src/support.h
index 041ffc3..5eff4fb 100644
--- a/src/support.h
+++ b/src/support.h
@@ -57,6 +57,15 @@ MALLOC_DECLARE(M_WG);
 #define IFT_WIREGUARD IFT_PPP
 #endif
 
+#ifndef ck_pr_store_bool
+#define ck_pr_store_bool(dst, val) ck_pr_store_8((uint8_t *)(dst), (uint8_t)(val))
+#endif
+
+#ifndef ck_pr_load_bool
+#define ck_pr_load_bool(src) ((bool)ck_pr_load_8((uint8_t *)(src)))
+#endif
+
+
 static inline int
 sogetsockaddr(struct socket *so, struct sockaddr **nam)
 {
@@ -68,25 +77,4 @@ sogetsockaddr(struct socket *so, struct sockaddr **nam)
 	return (error);
 }
 
-/* These are defined in sys/compat/linuxkpi/common/include/linux/compiler.h,
- * however I don't really want to include all that. */
-#define barrier()                       __asm__ __volatile__("": : :"memory")
-
-#define ACCESS_ONCE(x)                  (*(volatile __typeof(x) *)&(x))
-
-#define WRITE_ONCE(x,v) do {            \
-        barrier();                      \
-        ACCESS_ONCE(x) = (v);           \
-        barrier();                      \
-} while (0)
-
-#define READ_ONCE(x) ({                 \
-        __typeof(x) __var = ({          \
-                barrier();              \
-                ACCESS_ONCE(x);         \
-        });                             \
-        barrier();                      \
-        __var;                          \
-})
-
 #endif
diff --git a/src/wg_noise.c b/src/wg_noise.c
index fa591bd..4beb9fd 100644
--- a/src/wg_noise.c
+++ b/src/wg_noise.c
@@ -165,11 +165,6 @@ static void	noise_tai64n_now(uint8_t [NOISE_TIMESTAMP_LEN]);
 static int	noise_timer_expired(sbintime_t, uint32_t, uint32_t);
 static uint64_t siphash24(const uint8_t [SIPHASH_KEY_LENGTH], const void *, size_t);
 
-/* I can't find where FreeBSD defines such behaviours, so that is temporarily here. */
-#define epoch_ptr_read(p) ck_pr_load_ptr(p)
-#define epoch_ptr_write(p, v) ck_pr_store_ptr(p, v)
-/* Back to regular programming... */
-
 MALLOC_DEFINE(M_NOISE, "NOISE", "wgnoise");
 
 /* Local configuration */
@@ -565,16 +560,16 @@ noise_remote_keypairs_clear(struct noise_remote *r)
 	struct noise_keypair *kp;
 
 	rw_wlock(&r->r_keypair_lock);
-	kp = epoch_ptr_read(&r->r_next);
-	epoch_ptr_write(&r->r_next, NULL);
+	kp = ck_pr_load_ptr(&r->r_next);
+	ck_pr_store_ptr(&r->r_next, NULL);
 	noise_keypair_drop(kp);
 
-	kp = epoch_ptr_read(&r->r_current);
-	epoch_ptr_write(&r->r_current, NULL);
+	kp = ck_pr_load_ptr(&r->r_current);
+	ck_pr_store_ptr(&r->r_current, NULL);
 	noise_keypair_drop(kp);
 
-	kp = epoch_ptr_read(&r->r_previous);
-	epoch_ptr_write(&r->r_previous, NULL);
+	kp = ck_pr_load_ptr(&r->r_previous);
+	ck_pr_store_ptr(&r->r_previous, NULL);
 	noise_keypair_drop(kp);
 	rw_wunlock(&r->r_keypair_lock);
 }
@@ -588,10 +583,12 @@ noise_remote_expire_current(struct noise_remote *r)
 	noise_remote_handshake_clear(r);
 
 	NET_EPOCH_ENTER(et);
-	kp = epoch_ptr_read(&r->r_next);
-	if (kp != NULL) WRITE_ONCE(kp->kp_can_send, 0);
-	kp = epoch_ptr_read(&r->r_current);
-	if (kp != NULL) WRITE_ONCE(kp->kp_can_send, 0);
+	kp = ck_pr_load_ptr(&r->r_next);
+	if (kp != NULL)
+		ck_pr_store_bool(&kp->kp_can_send, false);
+	kp = ck_pr_load_ptr(&r->r_current);
+	if (kp != NULL)
+		ck_pr_store_bool(&kp->kp_can_send, false);
 	NET_EPOCH_EXIT(et);
 }
 
@@ -605,24 +602,24 @@ noise_add_new_keypair(struct noise_local *l, struct noise_remote *r,
 
 	/* Insert into the keypair table */
 	rw_wlock(&r->r_keypair_lock);
-	next = epoch_ptr_read(&r->r_next);
-	current = epoch_ptr_read(&r->r_current);
-	previous = epoch_ptr_read(&r->r_previous);
+	next = ck_pr_load_ptr(&r->r_next);
+	current = ck_pr_load_ptr(&r->r_current);
+	previous = ck_pr_load_ptr(&r->r_previous);
 
 	if (kp->kp_is_initiator) {
 		if (next != NULL) {
-			epoch_ptr_write(&r->r_next, NULL);
-			epoch_ptr_write(&r->r_previous, next);
+			ck_pr_store_ptr(&r->r_next, NULL);
+			ck_pr_store_ptr(&r->r_previous, next);
 			noise_keypair_drop(current);
 		} else {
-			epoch_ptr_write(&r->r_previous, current);
+			ck_pr_store_ptr(&r->r_previous, current);
 		}
 		noise_keypair_drop(previous);
-		epoch_ptr_write(&r->r_current, kp);
+		ck_pr_store_ptr(&r->r_current, kp);
 	} else {
-		epoch_ptr_write(&r->r_next, kp);
+		ck_pr_store_ptr(&r->r_next, kp);
 		noise_keypair_drop(next);
-		epoch_ptr_write(&r->r_previous, NULL);
+		ck_pr_store_ptr(&r->r_previous, NULL);
 		noise_keypair_drop(previous);
 
 	}
@@ -706,10 +703,10 @@ noise_keypair_current(struct noise_remote *r)
 	struct noise_keypair *kp, *ret = NULL;
 
 	NET_EPOCH_ENTER(et);
-	kp = epoch_ptr_read(&r->r_current);
-	if (kp != NULL && READ_ONCE(kp->kp_can_send)) {
+	kp = ck_pr_load_ptr(&r->r_current);
+	if (kp != NULL && ck_pr_load_bool(&kp->kp_can_send)) {
 		if (noise_timer_expired(kp->kp_birthdate, REJECT_AFTER_TIME, 0))
-			WRITE_ONCE(kp->kp_can_send, 0);
+			ck_pr_store_bool(&kp->kp_can_send, false);
 		else if (refcount_acquire_if_not_zero(&kp->kp_refcnt))
 			ret = kp;
 	}
@@ -730,20 +727,20 @@ noise_keypair_received_with(struct noise_keypair *kp)
 	struct noise_keypair *old;
 	struct noise_remote *r = kp->kp_remote;
 
-	if (kp != epoch_ptr_read(&r->r_next))
+	if (kp != ck_pr_load_ptr(&r->r_next))
 		return (0);
 
 	rw_wlock(&r->r_keypair_lock);
-	if (kp != epoch_ptr_read(&r->r_next)) {
+	if (kp != ck_pr_load_ptr(&r->r_next)) {
 		rw_wunlock(&r->r_keypair_lock);
 		return (0);
 	}
 
-	old = epoch_ptr_read(&r->r_previous);
-	epoch_ptr_write(&r->r_previous, epoch_ptr_read(&r->r_current));
+	old = ck_pr_load_ptr(&r->r_previous);
+	ck_pr_store_ptr(&r->r_previous, ck_pr_load_ptr(&r->r_current));
 	noise_keypair_drop(old);
-	epoch_ptr_write(&r->r_current, kp);
-	epoch_ptr_write(&r->r_next, NULL);
+	ck_pr_store_ptr(&r->r_current, kp);
+	ck_pr_store_ptr(&r->r_next, NULL);
 	rw_wunlock(&r->r_keypair_lock);
 
 	return (ECONNRESET);
@@ -809,7 +806,7 @@ noise_keypair_nonce_next(struct noise_keypair *kp, uint64_t *send)
 #endif
 	if (*send < REJECT_AFTER_MESSAGES)
 		return (0);
-	WRITE_ONCE(kp->kp_can_send, 0);
+	ck_pr_store_bool(&kp->kp_can_send, false);
 	return (EINVAL);
 }
 
@@ -841,7 +838,7 @@ noise_keypair_nonce_check(struct noise_keypair *kp, uint64_t recv)
 		for (i = 1; i <= top; i++)
 			kp->kp_backtrack[
 			    (i + index_ctr) & (COUNTER_NUM - 1)] = 0;
-		WRITE_ONCE(kp->kp_nonce_recv, recv);
+		ck_pr_store_64(&kp->kp_nonce_recv, recv);
 	}
 
 	index_recv %= COUNTER_NUM;
@@ -866,9 +863,9 @@ noise_keep_key_fresh_send(struct noise_remote *r)
 	int keep_key_fresh;
 
 	NET_EPOCH_ENTER(et);
-	current = epoch_ptr_read(&r->r_current);
-	keep_key_fresh = current != NULL && READ_ONCE(current->kp_can_send) && (
-	    READ_ONCE(current->kp_nonce_send) > REKEY_AFTER_MESSAGES ||
+	current = ck_pr_load_ptr(&r->r_current);
+	keep_key_fresh = current != NULL && ck_pr_load_bool(&current->kp_can_send) && (
+	    ck_pr_load_64(&current->kp_nonce_send) > REKEY_AFTER_MESSAGES ||
 	    (current->kp_is_initiator && noise_timer_expired(current->kp_birthdate, REKEY_AFTER_TIME, 0)));
 	NET_EPOCH_EXIT(et);
 
@@ -883,8 +880,8 @@ noise_keep_key_fresh_recv(struct noise_remote *r)
 	int keep_key_fresh;
 
 	NET_EPOCH_ENTER(et);
-	current = epoch_ptr_read(&r->r_current);
-	keep_key_fresh = current != NULL && READ_ONCE(current->kp_can_send) &&
+	current = ck_pr_load_ptr(&r->r_current);
+	keep_key_fresh = current != NULL && ck_pr_load_bool(&current->kp_can_send) &&
 	    current->kp_is_initiator && noise_timer_expired(current->kp_birthdate,
 			REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT, 0);
 	NET_EPOCH_EXIT(et);
@@ -905,7 +902,7 @@ noise_keypair_encrypt(struct noise_keypair *kp, uint32_t *r_idx, uint64_t nonce,
 int
 noise_keypair_decrypt(struct noise_keypair *kp, uint64_t nonce, struct mbuf *m)
 {
-	if (READ_ONCE(kp->kp_nonce_recv) >= REJECT_AFTER_MESSAGES ||
+	if (ck_pr_load_64(&kp->kp_nonce_recv) >= REJECT_AFTER_MESSAGES ||
 	    noise_timer_expired(kp->kp_birthdate, REJECT_AFTER_TIME, 0))
 		return (EINVAL);