diff options
| -rw-r--r-- | src/compat.h | 27 | ||||
| -rw-r--r-- | src/data.c | 5 | ||||
| -rw-r--r-- | src/peer.h | 3 |
3 files changed, 14 insertions, 21 deletions
diff --git a/src/compat.h b/src/compat.h index 3c73207..28c1aca 100644 --- a/src/compat.h +++ b/src/compat.h @@ -141,26 +141,6 @@ static inline int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) { retur static inline void dst_cache_destroy(struct dst_cache *dst_cache) { } #endif -/* https://lkml.org/lkml/2015/6/12/415 */ -#include <linux/netdevice.h> -static inline struct net_device *netdev_pub(void *dev) -{ - return (struct net_device *)((char *)dev - ALIGN(sizeof(struct net_device), NETDEV_ALIGN)); -} - -/* https://lkml.org/lkml/2016/10/1/187 */ -#ifdef CONFIG_WIREGUARD_PARALLEL -#include <linux/padata.h> -static inline int padata_queue_len(struct padata_instance *pinst) -{ - int len; - rcu_read_lock_bh(); - len = atomic_read(&rcu_dereference_bh(pinst->pd)->refcnt); - rcu_read_unlock_bh(); - return len; -} -#endif - /* PaX compatibility */ #ifdef CONSTIFY_PLUGIN #include <linux/cache.h> @@ -168,4 +148,11 @@ static inline int padata_queue_len(struct padata_instance *pinst) #define __read_mostly #endif +/* https://lkml.org/lkml/2015/6/12/415 */ +#include <linux/netdevice.h> +static inline struct net_device *netdev_pub(void *dev) +{ + return (struct net_device *)((char *)dev - ALIGN(sizeof(struct net_device), NETDEV_ALIGN)); +} + #endif @@ -215,6 +215,7 @@ static void finish_encryption(struct padata_priv *padata) struct encryption_ctx *ctx = container_of(padata, struct encryption_ctx, padata); ctx->callback(&ctx->queue, ctx->peer); + atomic_dec(&ctx->peer->parallel_encryption_inflight); peer_put(ctx->peer); kmem_cache_free(encryption_ctx_cache, ctx); } @@ -295,7 +296,7 @@ int packet_create_data(struct sk_buff_head *queue, struct wireguard_peer *peer, } #ifdef CONFIG_WIREGUARD_PARALLEL - if ((skb_queue_len(queue) > 1 || queue->next->len > 256 || padata_queue_len(peer->device->parallel_send) > 0) && cpumask_weight(cpu_online_mask) > 1) { + if ((skb_queue_len(queue) > 1 || queue->next->len > 256 || atomic_read(&peer->parallel_encryption_inflight) > 0) && cpumask_weight(cpu_online_mask) > 1) { unsigned int cpu = choose_cpu(keypair->remote_index); struct encryption_ctx *ctx = kmem_cache_alloc(encryption_ctx_cache, GFP_ATOMIC); if (!ctx) @@ -308,8 +309,10 @@ int packet_create_data(struct sk_buff_head *queue, struct wireguard_peer *peer, ret = -EBUSY; if (unlikely(!ctx->peer)) goto err_parallel; + atomic_inc(&peer->parallel_encryption_inflight); ret = start_encryption(peer->device->parallel_send, &ctx->padata, cpu); if (unlikely(ret < 0)) { + atomic_dec(&peer->parallel_encryption_inflight); peer_put(ctx->peer); err_parallel: skb_queue_splice(&ctx->queue, queue); @@ -40,6 +40,9 @@ struct wireguard_peer { struct rcu_head rcu; struct list_head peer_list; uint64_t internal_id; +#ifdef CONFIG_WIREGUARD_PARALLEL + atomic_t parallel_encryption_inflight; +#endif }; struct wireguard_peer *peer_create(struct wireguard_device *wg, const u8 public_key[NOISE_PUBLIC_KEY_LEN]); |