From 769eafb365ee30352d28d041fbd29c212857cbdb Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Wed, 22 Apr 2020 02:13:42 -0600 Subject: wireguard: unify encryption and decryption workers By unifying encryption and decryption workers into a single worker, this ensures that encryption and decryption happening at the same time does not result in workqueues stepping on each other and creating unnecessary work for the scheduler. Signed-off-by: Jason A. Donenfeld --- drivers/net/wireguard/device.c | 18 ++++---------- drivers/net/wireguard/device.h | 2 +- drivers/net/wireguard/queueing.h | 6 ++--- drivers/net/wireguard/receive.c | 4 +-- drivers/net/wireguard/send.c | 4 +-- drivers/net/wireguard/symmetric.c | 51 ++++++++++++++++++--------------------- 6 files changed, 37 insertions(+), 48 deletions(-) diff --git a/drivers/net/wireguard/device.c b/drivers/net/wireguard/device.c index 3ac3f8570ca1..9dee3cbdcb54 100644 --- a/drivers/net/wireguard/device.c +++ b/drivers/net/wireguard/device.c @@ -232,8 +232,7 @@ static void wg_destruct(struct net_device *dev) destroy_workqueue(wg->handshake_receive_wq); destroy_workqueue(wg->handshake_send_wq); destroy_workqueue(wg->packet_crypt_wq); - wg_packet_queue_free(&wg->decrypt_queue, true); - wg_packet_queue_free(&wg->encrypt_queue, true); + wg_packet_queue_free(&wg->crypt_queue, true); rcu_barrier(); /* Wait for all the peers to be actually freed. */ wg_ratelimiter_uninit(); memzero_explicit(&wg->static_identity, sizeof(wg->static_identity)); @@ -335,19 +334,14 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, if (!wg->packet_crypt_wq) goto err_destroy_handshake_send; - ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker, + ret = wg_packet_queue_init(&wg->crypt_queue, wg_packet_crypt_worker, true, MAX_QUEUED_PACKETS); if (ret < 0) goto err_destroy_packet_crypt; - ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker, - true, MAX_QUEUED_PACKETS); - if (ret < 0) - goto err_free_encrypt_queue; - ret = wg_ratelimiter_init(); if (ret < 0) - goto err_free_decrypt_queue; + goto err_free_crypt_queue; ret = register_netdevice(dev); if (ret < 0) @@ -365,10 +359,8 @@ static int wg_newlink(struct net *src_net, struct net_device *dev, err_uninit_ratelimiter: wg_ratelimiter_uninit(); -err_free_decrypt_queue: - wg_packet_queue_free(&wg->decrypt_queue, true); -err_free_encrypt_queue: - wg_packet_queue_free(&wg->encrypt_queue, true); +err_free_crypt_queue: + wg_packet_queue_free(&wg->crypt_queue, true); err_destroy_packet_crypt: destroy_workqueue(wg->packet_crypt_wq); err_destroy_handshake_send: diff --git a/drivers/net/wireguard/device.h b/drivers/net/wireguard/device.h index b15a8be9d816..8c63903e6632 100644 --- a/drivers/net/wireguard/device.h +++ b/drivers/net/wireguard/device.h @@ -38,7 +38,7 @@ struct crypt_queue { struct wg_device { struct net_device *dev; - struct crypt_queue encrypt_queue, decrypt_queue; + struct crypt_queue crypt_queue; struct sock __rcu *sock4, *sock6; struct net *creating_net; struct noise_static_identity static_identity; diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h index fb927d76b80a..0f382a4ba017 100644 --- a/drivers/net/wireguard/queueing.h +++ b/drivers/net/wireguard/queueing.h @@ -30,8 +30,6 @@ void wg_packet_receive(struct wg_device *wg, struct sk_buff *skb); void wg_packet_handshake_receive_worker(struct work_struct *work); /* NAPI poll function: */ int wg_packet_rx_poll(struct napi_struct *napi, int budget); -/* Workqueue worker: */ -void wg_packet_decrypt_worker(struct work_struct *work); /* send.c APIs: */ void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer, @@ -46,7 +44,9 @@ void wg_packet_send_staged_packets(struct wg_peer *peer); /* Workqueue workers: */ void wg_packet_handshake_send_worker(struct work_struct *work); void wg_packet_tx_worker(struct work_struct *work); -void wg_packet_encrypt_worker(struct work_struct *work); + +/* symmetric.c APIs: */ +void wg_packet_crypt_worker(struct work_struct *work); enum packet_state { PACKET_STATE_NOT_ENCRYPTED, diff --git a/drivers/net/wireguard/receive.c b/drivers/net/wireguard/receive.c index dd80650abb94..e4a96523c271 100644 --- a/drivers/net/wireguard/receive.c +++ b/drivers/net/wireguard/receive.c @@ -472,10 +472,10 @@ static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb) if (unlikely(READ_ONCE(peer->is_dead))) goto err; - ret = wg_queue_enqueue_per_device_and_peer(&wg->decrypt_queue, + ret = wg_queue_enqueue_per_device_and_peer(&wg->crypt_queue, &peer->rx_queue, skb, wg->packet_crypt_wq, - &wg->decrypt_queue.last_cpu, + &wg->crypt_queue.last_cpu, PACKET_STATE_NOT_DECRYPTED); if (unlikely(ret == -EPIPE)) wg_queue_enqueue_per_peer_napi(skb, PACKET_STATE_DEAD); diff --git a/drivers/net/wireguard/send.c b/drivers/net/wireguard/send.c index c7d5c3643403..e925ef7af45b 100644 --- a/drivers/net/wireguard/send.c +++ b/drivers/net/wireguard/send.c @@ -219,10 +219,10 @@ static void wg_packet_create_data(struct sk_buff *first) if (unlikely(READ_ONCE(peer->is_dead))) goto err; - ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue, + ret = wg_queue_enqueue_per_device_and_peer(&wg->crypt_queue, &peer->tx_queue, first, wg->packet_crypt_wq, - &wg->encrypt_queue.last_cpu, + &wg->crypt_queue.last_cpu, PACKET_STATE_NOT_ENCRYPTED); if (unlikely(ret == -EPIPE)) wg_queue_enqueue_per_peer(&peer->tx_queue, first, diff --git a/drivers/net/wireguard/symmetric.c b/drivers/net/wireguard/symmetric.c index 6e16fec7c25e..a1fd39452821 100644 --- a/drivers/net/wireguard/symmetric.c +++ b/drivers/net/wireguard/symmetric.c @@ -134,40 +134,37 @@ static bool decrypt_packet(struct sk_buff *skb, struct noise_symmetric_key *key) return true; } -void wg_packet_encrypt_worker(struct work_struct *work) +void wg_packet_crypt_worker(struct work_struct *work) { struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr; struct sk_buff *first, *skb, *next; while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) { - enum packet_state state = PACKET_STATE_ENCRYPTED; - - skb_list_walk_safe(first, skb, next) { - if (likely(encrypt_packet(skb, - PACKET_CB(first)->keypair))) { - wg_reset_packet(skb); - } else { - state = PACKET_STATE_DEAD; - break; + switch (atomic_read_acquire(&PACKET_CB(first)->state)) { + case PACKET_STATE_NOT_ENCRYPTED: { + enum packet_state state = PACKET_STATE_ENCRYPTED; + + skb_list_walk_safe(first, skb, next) { + if (likely(encrypt_packet(skb, + PACKET_CB(first)->keypair))) { + wg_reset_packet(skb); + } else { + state = PACKET_STATE_DEAD; + break; + } } + wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, + first, state); + break; + } + case PACKET_STATE_NOT_DECRYPTED: { + enum packet_state state = likely(decrypt_packet(first, + &PACKET_CB(first)->keypair->receiving)) ? + PACKET_STATE_DECRYPTED : PACKET_STATE_DEAD; + wg_queue_enqueue_per_peer_napi(first, state); + break; + } } - wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first, - state); - } -} - -void wg_packet_decrypt_worker(struct work_struct *work) -{ - struct crypt_queue *queue = container_of(work, struct multicore_worker, - work)->ptr; - struct sk_buff *skb; - - while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { - enum packet_state state = likely(decrypt_packet(skb, - &PACKET_CB(skb)->keypair->receiving)) ? - PACKET_STATE_DECRYPTED : PACKET_STATE_DEAD; - wg_queue_enqueue_per_peer_napi(skb, state); } } - -- cgit v1.2.3-59-g8ed1b