aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorThomas Gschwantner <tharre3@gmail.com>2018-06-01 03:49:50 +0200
committerJason A. Donenfeld <Jason@zx2c4.com>2018-06-04 20:30:02 +0200
commit042ff7a40ea7b603366a640ffd070204a0c778a4 (patch)
treeedf6637757ff7f842e0396714b0ebdbed1833371
parentWIP7 (diff)
downloadwireguard-monolithic-historical-042ff7a40ea7b603366a640ffd070204a0c778a4.tar.xz
wireguard-monolithic-historical-042ff7a40ea7b603366a640ffd070204a0c778a4.zip
WIP8
-rw-r--r--src/mpmc_ring.h58
-rw-r--r--src/receive.c2
-rw-r--r--src/send.c2
3 files changed, 21 insertions, 41 deletions
diff --git a/src/mpmc_ring.h b/src/mpmc_ring.h
index 6af5939..9ef32ce 100644
--- a/src/mpmc_ring.h
+++ b/src/mpmc_ring.h
@@ -70,12 +70,10 @@ static inline int ck_ring_init(struct ck_ring *ring, uint size, gfp_t gfp)
}
__always_inline static int
-_ck_ring_enqueue_mp(struct ck_ring *ring, const void *entry, unsigned int ts,
- unsigned int *size)
+_ck_ring_enqueue_mp(struct ck_ring *ring, const void *entry, uint *size)
{
const unsigned int mask = ring->mask;
unsigned int producer, consumer, delta;
- void *buffer;
int ret = 0;
producer = atomic_read(&ring->p_head);
@@ -129,8 +127,7 @@ _ck_ring_enqueue_mp(struct ck_ring *ring, const void *entry, unsigned int ts,
}
}
- buffer = (char *)ring->queue + ts * producer;
- memcpy(buffer, entry, ts);
+ WRITE_ONCE(ring->queue[producer], entry);
/*
* Wait until all concurrent producers have completed writing
@@ -154,54 +151,50 @@ leave:
}
__always_inline static int
-_ck_ring_enqueue_mp_size(struct ck_ring *ring, const void *entry,
- unsigned int ts, unsigned int *size)
+_ck_ring_enqueue_mp_size(struct ck_ring *ring, const void *entry, uint *size)
{
unsigned int sz;
int ret;
- ret = _ck_ring_enqueue_mp(ring, entry, ts, &sz);
+ ret = _ck_ring_enqueue_mp(ring, entry, &sz);
*size = sz;
return ret;
}
-__always_inline static bool
-_ck_ring_trydequeue_mc(struct ck_ring *ring,
- void *data, unsigned int size)
+__always_inline static void *ck_ring_trydequeue_mpmc(struct ck_ring *ring)
{
const unsigned int mask = ring->mask;
unsigned int consumer, producer;
- const void *buffer;
+ void *target;
consumer = atomic_read(&ring->c_head);
smp_rmb();
producer = atomic_read(&ring->p_tail);
if (unlikely(consumer == producer))
- return false;
+ return NULL;
smp_rmb();
- buffer = (const char *)ring->queue + size * consumer;
- memcpy(data, buffer, size);
+ target = READ_ONCE(ring->queue[consumer]);
ck_pr_fence_store_atomic();
- return atomic_cmpxchg(&ring->c_head, consumer, (consumer + 1) & mask) == consumer;
+ if (atomic_cmpxchg(&ring->c_head, consumer, (consumer + 1) & mask) == consumer)
+ return target;
+
+ return NULL;
}
-__always_inline static bool
-_ck_ring_dequeue_mc(struct ck_ring *ring,
- void *data, unsigned int ts)
+__always_inline static void *ck_ring_dequeue_mpmc(struct ck_ring *ring)
{
const unsigned int mask = ring->mask;
unsigned int consumer, producer, delta;
+ void *target;
bool cmp;
consumer = atomic_read(&ring->c_head);
do {
- const char *target;
-
/*
* Producer counter must represent state relative to
* our latest consumer snapshot.
@@ -210,12 +203,11 @@ _ck_ring_dequeue_mc(struct ck_ring *ring,
producer = atomic_read(&ring->p_tail);
if (unlikely(consumer == producer))
- return false;
+ return NULL;
smp_rmb();
- target = (const char *)ring->queue + ts * consumer;
- memcpy(data, target, ts);
+ target = READ_ONCE(ring->queue[consumer]);
/* Serialize load with respect to head update. */
ck_pr_fence_store_atomic();
@@ -225,7 +217,7 @@ _ck_ring_dequeue_mc(struct ck_ring *ring,
consumer = delta;
} while (!cmp);
- return true;
+ return target;
}
__always_inline static bool
@@ -363,26 +355,14 @@ static __always_inline void mpmc_ptr_ring_discard(struct ck_ring *ring)
inline static int
ck_ring_enqueue_mpmc(struct ck_ring *ring, const void *entry)
{
- return _ck_ring_enqueue_mp(ring, &entry, sizeof(entry), NULL);
+ return _ck_ring_enqueue_mp(ring, entry, NULL);
}
inline static int
ck_ring_enqueue_mpmc_size(struct ck_ring *ring, const void *entry,
unsigned int *size)
{
- return _ck_ring_enqueue_mp_size(ring, &entry, sizeof(entry), size);
-}
-
-inline static bool
-ck_ring_trydequeue_mpmc(struct ck_ring *ring, void *data)
-{
- return _ck_ring_trydequeue_mc(ring, (void **)data, sizeof(void *));
-}
-
-inline static bool
-ck_ring_dequeue_mpmc(struct ck_ring *ring, void *data)
-{
- return _ck_ring_dequeue_mc(ring, (void **)data, sizeof(void *));
+ return _ck_ring_enqueue_mp_size(ring, entry, size);
}
#endif /* _WG_MPMC_RING_H */
diff --git a/src/receive.c b/src/receive.c
index 3de698f..b7f14e9 100644
--- a/src/receive.c
+++ b/src/receive.c
@@ -414,7 +414,7 @@ void packet_decrypt_worker(struct work_struct *work)
struct sk_buff *skb;
bool have_simd = chacha20poly1305_init_simd();
- while (ck_ring_dequeue_mpmc(&queue->ring, &skb)) {
+ while ((skb = ck_ring_dequeue_mpmc(&queue->ring)) != NULL) {
enum packet_state state = likely(skb_decrypt(skb, &PACKET_CB(skb)->keypair->receiving, have_simd)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
queue_enqueue_per_peer(&PACKET_PEER(skb)->rx_queue, skb, state);
diff --git a/src/send.c b/src/send.c
index b4aff0e..79c5bbe 100644
--- a/src/send.c
+++ b/src/send.c
@@ -244,7 +244,7 @@ void packet_encrypt_worker(struct work_struct *work)
struct sk_buff *first, *skb, *next;
bool have_simd = chacha20poly1305_init_simd();
- while (ck_ring_dequeue_mpmc(&queue->ring, &first)) {
+ while ((first = ck_ring_dequeue_mpmc(&queue->ring)) != NULL) {
enum packet_state state = PACKET_STATE_CRYPTED;
skb_walk_null_queue_safe(first, skb, next) {