diff options
| -rw-r--r-- | src/compat/compat.h | 4 | ||||
| -rw-r--r-- | src/peer.c | 10 | ||||
| -rw-r--r-- | src/peer.h | 1 | ||||
| -rw-r--r-- | src/queueing.c | 14 | ||||
| -rw-r--r-- | src/queueing.h | 14 | ||||
| -rw-r--r-- | src/receive.c | 19 |
6 files changed, 45 insertions, 17 deletions
diff --git a/src/compat/compat.h b/src/compat/compat.h index 2c1b663..997ae39 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -597,6 +597,10 @@ static inline void *skb_put_data(struct sk_buff *skb, const void *data, unsigned } #endif +#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 19, 0) && !defined(ISRHEL7) +#define napi_complete_done(n, work_done) napi_complete(n) +#endif + /* https://lkml.kernel.org/r/20170624021727.17835-1-Jason@zx2c4.com */ #if IS_ENABLED(CONFIG_NF_CONNTRACK) #include <linux/ip.h> @@ -51,11 +51,13 @@ struct wireguard_peer *peer_create(struct wireguard_device *wg, const u8 public_ rwlock_init(&peer->endpoint_lock); kref_init(&peer->refcount); packet_queue_init(&peer->tx_queue, packet_tx_worker, false, MAX_QUEUED_PACKETS); - packet_queue_init(&peer->rx_queue, packet_rx_worker, false, MAX_QUEUED_PACKETS); + packet_queue_init(&peer->rx_queue, NULL, false, MAX_QUEUED_PACKETS); skb_queue_head_init(&peer->staged_packet_queue); list_add_tail(&peer->peer_list, &wg->peer_list); pubkey_hashtable_add(&wg->peer_hashtable, peer); peer->last_sent_handshake = ktime_get_boot_fast_ns() - (u64)(REKEY_TIMEOUT + 1) * NSEC_PER_SEC; + netif_napi_add(wg->dev, &peer->napi, packet_rx_poll, NAPI_POLL_WEIGHT); + napi_enable(&peer->napi); pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); return peer; } @@ -92,9 +94,11 @@ void peer_remove(struct wireguard_peer *peer) noise_keypairs_clear(&peer->keypairs); list_del_init(&peer->peer_list); timers_stop(peer); - flush_workqueue(peer->device->packet_crypt_wq); /* The first flush is for encrypt/decrypt step. */ - flush_workqueue(peer->device->packet_crypt_wq); /* The second flush is for send/receive step. */ + flush_workqueue(peer->device->packet_crypt_wq); /* The first flush is for encrypt/decrypt. */ + flush_workqueue(peer->device->packet_crypt_wq); /* The second.1 flush is for send (but not receive, since that's napi). */ + napi_disable(&peer->napi); /* The second.2 flush is for receive (but not send, since that's wq). */ flush_workqueue(peer->device->handshake_send_wq); + netif_napi_del(&peer->napi); --peer->device->num_peers; peer_put(peer); } @@ -57,6 +57,7 @@ struct wireguard_peer { struct rcu_head rcu; struct list_head peer_list; u64 internal_id; + struct napi_struct napi; }; struct wireguard_peer *peer_create(struct wireguard_device *wg, const u8 public_key[NOISE_PUBLIC_KEY_LEN], const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]); diff --git a/src/queueing.c b/src/queueing.c index f33395e..c8394fc 100644 --- a/src/queueing.c +++ b/src/queueing.c @@ -28,12 +28,14 @@ int packet_queue_init(struct crypt_queue *queue, work_func_t function, bool mult ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL); if (ret) return ret; - if (multicore) { - queue->worker = packet_alloc_percpu_multicore_worker(function, queue); - if (!queue->worker) - return -ENOMEM; - } else - INIT_WORK(&queue->work, function); + if (function) { + if (multicore) { + queue->worker = packet_alloc_percpu_multicore_worker(function, queue); + if (!queue->worker) + return -ENOMEM; + } else + INIT_WORK(&queue->work, function); + } return 0; } diff --git a/src/queueing.h b/src/queueing.h index 0057cfa..c17b0d8 100644 --- a/src/queueing.h +++ b/src/queueing.h @@ -26,8 +26,9 @@ struct multicore_worker __percpu *packet_alloc_percpu_multicore_worker(work_func /* receive.c APIs: */ void packet_receive(struct wireguard_device *wg, struct sk_buff *skb); void packet_handshake_receive_worker(struct work_struct *work); -/* Workqueue workers: */ -void packet_rx_worker(struct work_struct *work); +/* NAPI poll function: */ +int packet_rx_poll(struct napi_struct *napi, int budget); +/* Workqueue worker: */ void packet_decrypt_worker(struct work_struct *work); /* send.c APIs: */ @@ -138,6 +139,15 @@ static inline void queue_enqueue_per_peer(struct crypt_queue *queue, struct sk_b peer_put(peer); } +static inline void queue_enqueue_per_peer_napi(struct crypt_queue *queue, struct sk_buff *skb, enum packet_state state) +{ + struct wireguard_peer *peer = peer_rcu_get(PACKET_PEER(skb)); + + atomic_set(&PACKET_CB(skb)->state, state); + napi_schedule(&peer->napi); + peer_put(peer); +} + #ifdef DEBUG bool packet_counter_selftest(void); #endif diff --git a/src/receive.c b/src/receive.c index 7dc3a61..1e8a242 100644 --- a/src/receive.c +++ b/src/receive.c @@ -368,17 +368,17 @@ packet_processed: dev_kfree_skb(skb); } -void packet_rx_worker(struct work_struct *work) +int packet_rx_poll(struct napi_struct *napi, int budget) { - struct crypt_queue *queue = container_of(work, struct crypt_queue, work); - struct wireguard_peer *peer; + struct wireguard_peer *peer = container_of(napi, struct wireguard_peer, napi); + struct crypt_queue *queue = &peer->rx_queue; struct noise_keypair *keypair; struct sk_buff *skb; struct endpoint endpoint; enum packet_state state; + int work_done = 0; bool free; - local_bh_disable(); while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && (state = atomic_read(&PACKET_CB(skb)->state)) != PACKET_STATE_UNCRYPTED) { __ptr_ring_discard_one(&queue->ring); peer = PACKET_PEER(skb); @@ -405,8 +405,15 @@ next: peer_put(peer); if (unlikely(free)) dev_kfree_skb(skb); + + if (++work_done >= budget) + break; } - local_bh_enable(); + + if (work_done < budget) + napi_complete_done(napi, work_done); + + return work_done; } void packet_decrypt_worker(struct work_struct *work) @@ -417,7 +424,7 @@ void packet_decrypt_worker(struct work_struct *work) while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { enum packet_state state = likely(skb_decrypt(skb, &PACKET_CB(skb)->keypair->receiving, have_simd)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; - queue_enqueue_per_peer(&PACKET_PEER(skb)->rx_queue, skb, state); + queue_enqueue_per_peer_napi(&PACKET_PEER(skb)->rx_queue, skb, state); have_simd = simd_relax(have_simd); } |