wireguard: unify encryption and decryption workersjd/unified-crypt-queue

By unifying encryption and decryption workers into a single worker, this
ensures that encryption and decryption happening at the same time does
not result in workqueues stepping on each other and creating
unnecessary work for the scheduler.

Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>

6 files changed, 37 insertions, 48 deletions

diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c
index 3ac3f8570ca1..9dee3cbdcb54 100644
--- a/drivers/net/wireguard/device.c
+++ b/drivers/net/wireguard/device.c
@@ -232,8 +232,7 @@ static void wg_destruct(struct net_device *dev)
 	destroy_workqueue(wg->handshake_receive_wq);
 	destroy_workqueue(wg->handshake_send_wq);
 	destroy_workqueue(wg->packet_crypt_wq);
-	wg_packet_queue_free(&wg->decrypt_queue, true);
-	wg_packet_queue_free(&wg->encrypt_queue, true);
+	wg_packet_queue_free(&wg->crypt_queue, true);
 	rcu_barrier(); /* Wait for all the peers to be actually freed. */
 	wg_ratelimiter_uninit();
 	memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
@@ -335,19 +334,14 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
 	if (!wg->packet_crypt_wq)
 		goto err_destroy_handshake_send;
 
-	ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
+	ret = wg_packet_queue_init(&wg->crypt_queue, wg_packet_crypt_worker,
 				   true, MAX_QUEUED_PACKETS);
 	if (ret < 0)
 		goto err_destroy_packet_crypt;
 
-	ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
-				   true, MAX_QUEUED_PACKETS);
-	if (ret < 0)
-		goto err_free_encrypt_queue;
-
 	ret = wg_ratelimiter_init();
 	if (ret < 0)
-		goto err_free_decrypt_queue;
+		goto err_free_crypt_queue;
 
 	ret = register_netdevice(dev);
 	if (ret < 0)
@@ -365,10 +359,8 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
 
 err_uninit_ratelimiter:
 	wg_ratelimiter_uninit();
-err_free_decrypt_queue:
-	wg_packet_queue_free(&wg->decrypt_queue, true);
-err_free_encrypt_queue:
-	wg_packet_queue_free(&wg->encrypt_queue, true);
+err_free_crypt_queue:
+	wg_packet_queue_free(&wg->crypt_queue, true);
 err_destroy_packet_crypt:
 	destroy_workqueue(wg->packet_crypt_wq);
 err_destroy_handshake_send:
diff --git a/drivers/net/wireguard/device.h b/drivers/net/wireguard/device.h
index b15a8be9d816..8c63903e6632 100644
--- a/drivers/net/wireguard/device.h
+++ b/drivers/net/wireguard/device.h
@@ -38,7 +38,7 @@ struct crypt_queue {
 
 struct wg_device {
 	struct net_device *dev;
-	struct crypt_queue encrypt_queue, decrypt_queue;
+	struct crypt_queue crypt_queue;
 	struct sock __rcu *sock4, *sock6;
 	struct net *creating_net;
 	struct noise_static_identity static_identity;
diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h
index fb927d76b80a..0f382a4ba017 100644
--- a/drivers/net/wireguard/queueing.h
+++ b/drivers/net/wireguard/queueing.h
@@ -30,8 +30,6 @@ void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb);
 void wg_packet_handshake_receive_worker(struct work_struct *work);
 /* NAPI poll function: */
 int wg_packet_rx_poll(struct napi_struct *napi, int budget);
-/* Workqueue worker: */
-void wg_packet_decrypt_worker(struct work_struct *work);
 
 /* send.c APIs: */
 void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
@@ -46,7 +44,9 @@ void wg_packet_send_staged_packets(struct wg_peer *peer);
 /* Workqueue workers: */
 void wg_packet_handshake_send_worker(struct work_struct *work);
 void wg_packet_tx_worker(struct work_struct *work);
-void wg_packet_encrypt_worker(struct work_struct *work);
+
+/* symmetric.c APIs: */
+void wg_packet_crypt_worker(struct work_struct *work);
 
 enum packet_state {
 	PACKET_STATE_NOT_ENCRYPTED,
diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c
index dd80650abb94..e4a96523c271 100644
--- a/drivers/net/wireguard/receive.c
+++ b/drivers/net/wireguard/receive.c
@@ -472,10 +472,10 @@ static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb)
 	if (unlikely(READ_ONCE(peer->is_dead)))
 		goto err;
 
-	ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue,
+	ret = wg_queue_enqueue_per_device_and_peer(&wg->crypt_queue,
 						   &peer->rx_queue, skb,
 						   wg->packet_crypt_wq,
-						   &wg->decrypt_queue.last_cpu,
+						   &wg->crypt_queue.last_cpu,
 						   PACKET_STATE_NOT_DECRYPTED);
 	if (unlikely(ret == -EPIPE))
 		wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD);
diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c
index c7d5c3643403..e925ef7af45b 100644
--- a/drivers/net/wireguard/send.c
+++ b/drivers/net/wireguard/send.c
@@ -219,10 +219,10 @@ static void wg_packet_create_data(struct sk_buff *first)
 	if (unlikely(READ_ONCE(peer->is_dead)))
 		goto err;
 
-	ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue,
+	ret = wg_queue_enqueue_per_device_and_peer(&wg->crypt_queue,
 						   &peer->tx_queue, first,
 						   wg->packet_crypt_wq,
-						   &wg->encrypt_queue.last_cpu,
+						   &wg->crypt_queue.last_cpu,
 						   PACKET_STATE_NOT_ENCRYPTED);
 	if (unlikely(ret == -EPIPE))
 		wg_queue_enqueue_per_peer(&peer->tx_queue, first,
diff --git a/drivers/net/wireguard/symmetric.c b/drivers/net/wireguard/symmetric.c
index 6e16fec7c25e..a1fd39452821 100644
--- a/drivers/net/wireguard/symmetric.c
+++ b/drivers/net/wireguard/symmetric.c
@@ -134,40 +134,37 @@ static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key)
 	return true;
 }
 
-void wg_packet_encrypt_worker(struct work_struct *work)
+void wg_packet_crypt_worker(struct work_struct *work)
 {
 	struct crypt_queue *queue = container_of(work, struct multicore_worker,
 						 work)->ptr;
 	struct sk_buff *first, *skb, *next;
 
 	while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
-		enum packet_state state = PACKET_STATE_ENCRYPTED;
-
-		skb_list_walk_safe(first, skb, next) {
-			if (likely(encrypt_packet(skb,
-					PACKET_CB(first)->keypair))) {
-				wg_reset_packet(skb);
-			} else {
-				state = PACKET_STATE_DEAD;
-				break;
+		switch (atomic_read_acquire(&PACKET_CB(first)->state)) {
+		case PACKET_STATE_NOT_ENCRYPTED: {
+			enum packet_state state = PACKET_STATE_ENCRYPTED;
+
+			skb_list_walk_safe(first, skb, next) {
+				if (likely(encrypt_packet(skb,
+						PACKET_CB(first)->keypair))) {
+					wg_reset_packet(skb);
+				} else {
+					state = PACKET_STATE_DEAD;
+					break;
+				}
 			}
+			wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue,
+						  first, state);
+			break;
+		}
+		case PACKET_STATE_NOT_DECRYPTED: {
+			enum packet_state state = likely(decrypt_packet(first,
+					&PACKET_CB(first)->keypair->receiving)) ?
+					PACKET_STATE_DECRYPTED : PACKET_STATE_DEAD;
+			wg_queue_enqueue_per_peer_napi(first, state);
+			break;
+		}
 		}
-		wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first,
-					  state);
-	}
-}
-
-void wg_packet_decrypt_worker(struct work_struct *work)
-{
-	struct crypt_queue *queue = container_of(work, struct multicore_worker,
-						 work)->ptr;
-	struct sk_buff *skb;
-
-	while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
-		enum packet_state state = likely(decrypt_packet(skb,
-				&PACKET_CB(skb)->keypair->receiving)) ?
-				PACKET_STATE_DECRYPTED : PACKET_STATE_DEAD;
-		wg_queue_enqueue_per_peer_napi(skb, state);
 	}
 }
-