diff options
| -rw-r--r-- | src/mpmc_ring.h | 6 | ||||
| -rw-r--r-- | src/queueing.c | 2 | ||||
| -rw-r--r-- | src/queueing.h | 2 | ||||
| -rw-r--r-- | src/receive.c | 20 | ||||
| -rw-r--r-- | src/send.c | 13 |
5 files changed, 28 insertions, 15 deletions
diff --git a/src/mpmc_ring.h b/src/mpmc_ring.h index 4a0f55b..8fd6d8f 100644 --- a/src/mpmc_ring.h +++ b/src/mpmc_ring.h @@ -42,11 +42,6 @@ #include <linux/compiler.h> #include <linux/atomic.h> #include <linux/cache.h> -//#define likely(x) (__builtin_expect(!!(x), 1)) -//#define unlikely(x) (__builtin_expect(!!(x), 0)) -//#define cpu_relax() - -//#define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) /* http://concurrencykit.org/doc/ck_pr_load.html */ #define ck_pr_load_uint(SRC) atomic_read(SRC) @@ -73,6 +68,7 @@ z; __asm__ __volatile__("lock " "cmpxchg" "l" " %3, %0;" "mov %% " "eax" ", %2;" "setz %1;" : "+m" (*(unsigned int *)target), "=a" (z), "=m" (*(unsigned int *)v) : "q" (set), "a" (compare) : "memory", "cc"); return z; } */ +__always_inline static bool ck_pr_cas_uint_value(atomic_t *target, uint old, uint new, uint *v) { uint prev = atomic_cmpxchg(target, old, new); *v = new; diff --git a/src/queueing.c b/src/queueing.c index 4c9ae53..7617622 100644 --- a/src/queueing.c +++ b/src/queueing.c @@ -29,6 +29,7 @@ int packet_queue_init(struct crypt_queue *queue, work_func_t function, bool mult /*if (ret)*/ /*return ret;*/ ck_ring_init(&queue->ring, len); + queue->ring_buffer.value = kcalloc(len, sizeof(void *), GFP_KERNEL); if (multicore) { queue->worker = packet_alloc_percpu_multicore_worker(function, queue); if (!queue->worker) @@ -48,4 +49,5 @@ void packet_queue_free(struct crypt_queue *queue, bool multicore) * http://concurrencykit.org/doc/ck_ring_trydequeue_spmc.html */ /*WARN_ON(!ptr_ring_empty_bh(&queue->ring));*/ /*ptr_ring_cleanup(&queue->ring, NULL);*/ + kfree(queue->ring_buffer.value); } diff --git a/src/queueing.h b/src/queueing.h index b67401b..e90744e 100644 --- a/src/queueing.h +++ b/src/queueing.h @@ -12,8 +12,6 @@ #include <linux/ip.h> #include <linux/ipv6.h> -#include "mpmc_ring.h" - struct wireguard_device; struct wireguard_peer; struct multicore_worker; diff --git a/src/receive.c b/src/receive.c index 27d3d04..5ab4171 100644 --- a/src/receive.c +++ b/src/receive.c @@ -378,9 +378,18 @@ void packet_rx_worker(struct work_struct *work) bool free; local_bh_disable(); - spin_lock_bh(&queue->ring.consumer_lock); - while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && (state = atomic_read(&PACKET_CB(skb)->state)) != PACKET_STATE_UNCRYPTED) { - __ptr_ring_discard_one(&queue->ring); + /*spin_lock_bh(&queue->ring.consumer_lock);*/ + /*while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && (state = atomic_read(&PACKET_CB(skb)->state)) != PACKET_STATE_UNCRYPTED) {*/ + /*__ptr_ring_discard_one(&queue->ring);*/ + + // TODO: this is very wrong + while (ck_ring_dequeue_mpmc(&queue->ring, &queue->ring_buffer, &skb)) { + if ((state = atomic_read(&PACKET_CB(skb)->state)) == PACKET_STATE_UNCRYPTED) { + ck_ring_enqueue_mpmc(&queue->ring, &queue->ring_buffer, &skb); // ignores error + continue; + } + + peer = PACKET_PEER(skb); keypair = PACKET_CB(skb)->keypair; free = true; @@ -406,7 +415,7 @@ next: if (unlikely(free)) dev_kfree_skb(skb); } - spin_unlock_bh(&queue->ring.consumer_lock); + /*spin_unlock_bh(&queue->ring.consumer_lock);*/ local_bh_enable(); } @@ -416,7 +425,8 @@ void packet_decrypt_worker(struct work_struct *work) struct sk_buff *skb; bool have_simd = chacha20poly1305_init_simd(); - while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) { + /*while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {*/ + while (ck_ring_dequeue_mpmc(&queue->ring, &queue->ring_buffer, &skb)) { enum packet_state state = likely(skb_decrypt(skb, &PACKET_CB(skb)->keypair->receiving, have_simd)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD; queue_enqueue_per_peer(&PACKET_PEER(skb)->rx_queue, skb, state); @@ -224,8 +224,15 @@ void packet_tx_worker(struct work_struct *work) enum packet_state state; /*spin_lock_bh(&queue->ring.consumer_lock);*/ - while ((first = __ptr_ring_peek(&queue->ring)) != NULL && (state = atomic_read(&PACKET_CB(first)->state)) != PACKET_STATE_UNCRYPTED) { - __ptr_ring_discard_one(&queue->ring); + /*while ((first = __ptr_ring_peek(&queue->ring)) != NULL && (state = atomic_read(&PACKET_CB(first)->state)) != PACKET_STATE_UNCRYPTED) {*/ + /*__ptr_ring_discard_one(&queue->ring);*/ + + // TODO: this is very wrong + while (ck_ring_dequeue_mpmc(&queue->ring, &queue->ring_buffer, &first)) { + if ((state = atomic_read(&PACKET_CB(first)->state)) == PACKET_STATE_UNCRYPTED) { + ck_ring_enqueue_mpmc(&queue->ring, &queue->ring_buffer, &first); // ignores error + continue; + } peer = PACKET_PEER(first); keypair = PACKET_CB(first)->keypair; @@ -248,7 +255,7 @@ void packet_encrypt_worker(struct work_struct *work) bool have_simd = chacha20poly1305_init_simd(); /*while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {*/ - while ((ck_ring_dequeue_mpmc(&queue->ring, &queue->ring_buffer, &first)) == true) { + while (ck_ring_dequeue_mpmc(&queue->ring, &queue->ring_buffer, &first)) { enum packet_state state = PACKET_STATE_CRYPTED; skb_walk_null_queue_safe(first, skb, next) { |