diff options
author | Thomas Gschwantner <tharre3@gmail.com> | 2018-06-01 03:49:50 +0200 |
---|---|---|
committer | Jason A. Donenfeld <Jason@zx2c4.com> | 2018-06-04 20:30:02 +0200 |
commit | 042ff7a40ea7b603366a640ffd070204a0c778a4 (patch) | |
tree | edf6637757ff7f842e0396714b0ebdbed1833371 /src | |
parent | WIP7 (diff) | |
download | wireguard-monolithic-historical-042ff7a40ea7b603366a640ffd070204a0c778a4.tar.xz wireguard-monolithic-historical-042ff7a40ea7b603366a640ffd070204a0c778a4.zip |
WIP8
Diffstat (limited to 'src')
-rw-r--r-- | src/mpmc_ring.h | 58 | ||||
-rw-r--r-- | src/receive.c | 2 | ||||
-rw-r--r-- | src/send.c | 2 |
3 files changed, 21 insertions, 41 deletions
diff --git a/src/mpmc_ring.h b/src/mpmc_ring.h index 6af5939..9ef32ce 100644 --- a/src/mpmc_ring.h +++ b/src/mpmc_ring.h @@ -70,12 +70,10 @@ static inline int ck_ring_init(struct ck_ring *ring, uint size, gfp_t gfp) } __always_inline static int -_ck_ring_enqueue_mp(struct ck_ring *ring, const void *entry, unsigned int ts, - unsigned int *size) +_ck_ring_enqueue_mp(struct ck_ring *ring, const void *entry, uint *size) { const unsigned int mask = ring->mask; unsigned int producer, consumer, delta; - void *buffer; int ret = 0; producer = atomic_read(&ring->p_head); @@ -129,8 +127,7 @@ _ck_ring_enqueue_mp(struct ck_ring *ring, const void *entry, unsigned int ts, } } - buffer = (char *)ring->queue + ts * producer; - memcpy(buffer, entry, ts); + WRITE_ONCE(ring->queue[producer], entry); /* * Wait until all concurrent producers have completed writing @@ -154,54 +151,50 @@ leave: } __always_inline static int -_ck_ring_enqueue_mp_size(struct ck_ring *ring, const void *entry, - unsigned int ts, unsigned int *size) +_ck_ring_enqueue_mp_size(struct ck_ring *ring, const void *entry, uint *size) { unsigned int sz; int ret; - ret = _ck_ring_enqueue_mp(ring, entry, ts, &sz); + ret = _ck_ring_enqueue_mp(ring, entry, &sz); *size = sz; return ret; } -__always_inline static bool -_ck_ring_trydequeue_mc(struct ck_ring *ring, - void *data, unsigned int size) +__always_inline static void *ck_ring_trydequeue_mpmc(struct ck_ring *ring) { const unsigned int mask = ring->mask; unsigned int consumer, producer; - const void *buffer; + void *target; consumer = atomic_read(&ring->c_head); smp_rmb(); producer = atomic_read(&ring->p_tail); if (unlikely(consumer == producer)) - return false; + return NULL; smp_rmb(); - buffer = (const char *)ring->queue + size * consumer; - memcpy(data, buffer, size); + target = READ_ONCE(ring->queue[consumer]); ck_pr_fence_store_atomic(); - return atomic_cmpxchg(&ring->c_head, consumer, (consumer + 1) & mask) == consumer; + if (atomic_cmpxchg(&ring->c_head, consumer, (consumer + 1) & mask) == consumer) + return target; + + return NULL; } -__always_inline static bool -_ck_ring_dequeue_mc(struct ck_ring *ring, - void *data, unsigned int ts) +__always_inline static void *ck_ring_dequeue_mpmc(struct ck_ring *ring) { const unsigned int mask = ring->mask; unsigned int consumer, producer, delta; + void *target; bool cmp; consumer = atomic_read(&ring->c_head); do { - const char *target; - /* * Producer counter must represent state relative to * our latest consumer snapshot. @@ -210,12 +203,11 @@ _ck_ring_dequeue_mc(struct ck_ring *ring, producer = atomic_read(&ring->p_tail); if (unlikely(consumer == producer)) - return false; + return NULL; smp_rmb(); - target = (const char *)ring->queue + ts * consumer; - memcpy(data, target, ts); + target = READ_ONCE(ring->queue[consumer]); /* Serialize load with respect to head update. */ ck_pr_fence_store_atomic(); @@ -225,7 +217,7 @@ _ck_ring_dequeue_mc(struct ck_ring *ring, consumer = delta; } while (!cmp); - return true; + return target; } __always_inline static bool @@ -363,26 +355,14 @@ static __always_inline void mpmc_ptr_ring_discard(struct ck_ring *ring) inline static int ck_ring_enqueue_mpmc(struct ck_ring *ring, const void *entry) { - return _ck_ring_enqueue_mp(ring, &entry, sizeof(entry), NULL); + return _ck_ring_enqueue_mp(ring, entry, NULL); } inline static int ck_ring_enqueue_mpmc_size(struct ck_ring *ring, const void *entry, unsigned int *size) { - return _ck_ring_enqueue_mp_size(ring, &entry, sizeof(entry), size); -} - -inline static bool -ck_ring_trydequeue_mpmc(struct ck_ring *ring, void *data) -{ - return _ck_ring_trydequeue_mc(ring, (void **)data, sizeof(void *)); -} - -inline static bool -ck_ring_dequeue_mpmc(struct ck_ring *ring, void *data) -{ - return _ck_ring_dequeue_mc(ring, (void **)data, sizeof(void *)); + return _ck_ring_enqueue_mp_size(ring, entry, size); } #endif /* _WG_MPMC_RING_H */ diff --git a/src/receive.c b/src/receive.c index 3de698f..b7f14e9 100644 --- a/src/receive.c +++ b/src/receive.c @@ -414,7 +414,7 @@ void packet_decrypt_worker(struct work_struct *work) struct sk_buff *skb; bool have_simd = chacha20poly1305_init_simd(); - while (ck_ring_dequeue_mpmc(&queue->ring, &skb)) { + while ((skb = ck_ring_dequeue_mpmc(&queue->ring)) != NULL) { enum packet_state state = likely(skb_decrypt(skb, &PACKET_CB(skb)->keypair->receiving, have_simd)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; queue_enqueue_per_peer(&PACKET_PEER(skb)->rx_queue, skb, state); @@ -244,7 +244,7 @@ void packet_encrypt_worker(struct work_struct *work) struct sk_buff *first, *skb, *next; bool have_simd = chacha20poly1305_init_simd(); - while (ck_ring_dequeue_mpmc(&queue->ring, &first)) { + while ((first = ck_ring_dequeue_mpmc(&queue->ring)) != NULL) { enum packet_state state = PACKET_STATE_CRYPTED; skb_walk_null_queue_safe(first, skb, next) { |