aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorJason A. Donenfeld <Jason@zx2c4.com>2017-09-15 16:49:42 +0200
committerJason A. Donenfeld <Jason@zx2c4.com>2017-09-15 16:49:42 +0200
commit0ec54810b823e59dc0afa7686aeac048f7eaebbf (patch)
tree6401b471e4bbdc0c544580a4898673318013054d
parentTry out spinlocks for multiconsumer (diff)
downloadwireguard-monolithic-historical-jd/spinlocks-dql.tar.xz
wireguard-monolithic-historical-jd/spinlocks-dql.zip
Try only advancing to next CPU if dql says sojd/spinlocks-dql
-rw-r--r--src/data.c42
-rw-r--r--src/device.c2
-rw-r--r--src/device.h1
3 files changed, 36 insertions, 9 deletions
diff --git a/src/data.c b/src/data.c
index fc48d88..c673ce6 100644
--- a/src/data.c
+++ b/src/data.c
@@ -63,10 +63,18 @@ static inline struct crypt_ctx *queue_dequeue_per_peer(struct crypt_queue *queue
return head ? list_entry(head, struct crypt_ctx, per_peer_head) : NULL;
}
-static inline struct crypt_ctx *queue_dequeue_per_device(struct crypt_queue *queue)
+static inline struct crypt_ctx *queue_dequeue_per_device(struct crypt_queue *queue, bool sending)
{
struct list_head *head = queue_dequeue(queue);
- return head ? list_entry(head, struct crypt_ctx, per_device_head) : NULL;
+ struct crypt_ctx *ctx;
+ if (!head)
+ return NULL;
+ ctx = list_entry(head, struct crypt_ctx, per_device_head);
+ if (sending)
+ dql_completed(&queue->dql, skb_queue_len(&ctx->packets));
+ else
+ dql_completed(&queue->dql, 1);
+ return ctx;
}
static inline bool queue_enqueue_per_peer(struct crypt_queue *queue, struct crypt_ctx *ctx)
@@ -77,9 +85,25 @@ static inline bool queue_enqueue_per_peer(struct crypt_queue *queue, struct cryp
return queue_enqueue(queue, &(ctx)->per_peer_head, MAX_QUEUED_PACKETS);
}
-static inline void queue_enqueue_per_device(struct crypt_queue *queue, struct crypt_ctx *ctx, struct workqueue_struct *wq, int *next_cpu)
+static inline int cpumask_next_online_dql(int *next, struct dql *dql)
+{
+ int cpu = *next;
+ while (unlikely(!cpumask_test_cpu(cpu, cpu_online_mask)))
+ cpu = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits;
+ if (dql_avail(dql) < 0)
+ *next = cpumask_next(cpu, cpu_online_mask) % nr_cpumask_bits;
+ else
+ *next = cpu;
+ return cpu;
+}
+
+static inline void queue_enqueue_per_device(struct crypt_queue *queue, struct crypt_ctx *ctx, struct workqueue_struct *wq, int *next_cpu, bool sending)
{
- int cpu = cpumask_next_online(next_cpu);
+ int cpu = cpumask_next_online_dql(next_cpu, &queue->dql);
+ if (sending)
+ dql_queued(&queue->dql, skb_queue_len(&ctx->packets));
+ else
+ dql_queued(&queue->dql, 1);
queue_enqueue(queue, &ctx->per_device_head, 0);
queue_work_on(cpu, wq, &per_cpu_ptr(queue->worker, cpu)->work);
}
@@ -321,7 +345,7 @@ void packet_encrypt_worker(struct work_struct *work)
struct wireguard_peer *peer;
bool have_simd = chacha20poly1305_init_simd();
- while ((ctx = queue_dequeue_per_device(queue)) != NULL) {
+ while ((ctx = queue_dequeue_per_device(queue, true)) != NULL) {
skb_queue_walk_safe(&ctx->packets, skb, tmp) {
if (likely(skb_encrypt(skb, ctx->keypair, have_simd))) {
skb_reset(skb);
@@ -355,7 +379,7 @@ void packet_init_worker(struct work_struct *work)
}
queue_dequeue(queue);
if (likely(queue_enqueue_per_peer(&peer->send_queue, ctx)))
- queue_enqueue_per_device(&wg->send_queue, ctx, wg->packet_crypt_wq, &wg->send_queue.last_cpu);
+ queue_enqueue_per_device(&wg->send_queue, ctx, wg->packet_crypt_wq, &wg->send_queue.last_cpu, true);
else
free_ctx(ctx);
}
@@ -384,7 +408,7 @@ void packet_create_data(struct wireguard_peer *peer, struct sk_buff_head *packet
if (likely(list_empty(&peer->init_queue.list))) {
if (likely(populate_sending_ctx(ctx))) {
if (likely(queue_enqueue_per_peer(&peer->send_queue, ctx)))
- queue_enqueue_per_device(&wg->send_queue, ctx, wg->packet_crypt_wq, &wg->send_queue.last_cpu);
+ queue_enqueue_per_device(&wg->send_queue, ctx, wg->packet_crypt_wq, &wg->send_queue.last_cpu, true);
else
free_ctx(ctx);
return;
@@ -458,7 +482,7 @@ void packet_decrypt_worker(struct work_struct *work)
struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->queue;
struct wireguard_peer *peer;
- while ((ctx = queue_dequeue_per_device(queue)) != NULL) {
+ while ((ctx = queue_dequeue_per_device(queue, false)) != NULL) {
if (unlikely(socket_endpoint_from_skb(&ctx->endpoint, ctx->skb) < 0 || !skb_decrypt(ctx->skb, &ctx->keypair->receiving))) {
dev_kfree_skb(ctx->skb);
ctx->skb = NULL;
@@ -499,7 +523,7 @@ void packet_consume_data(struct sk_buff *skb, struct wireguard_device *wg)
ctx->peer = ctx->keypair->entry.peer;
if (likely(queue_enqueue_per_peer(&ctx->peer->receive_queue, ctx)))
- queue_enqueue_per_device(&wg->receive_queue, ctx, wg->packet_crypt_wq, &wg->receive_queue.last_cpu);
+ queue_enqueue_per_device(&wg->receive_queue, ctx, wg->packet_crypt_wq, &wg->receive_queue.last_cpu, false);
else {
/* TODO: replace this with a call to free_ctx when receiving uses skb_queues as well. */
noise_keypair_put(ctx->keypair);
diff --git a/src/device.c b/src/device.c
index 070e5e5..6faa7e4 100644
--- a/src/device.c
+++ b/src/device.c
@@ -311,6 +311,7 @@ static int newlink(struct net *src_net, struct net_device *dev, struct nlattr *t
INIT_LIST_HEAD(&wg->send_queue.list);
spin_lock_init(&wg->send_queue.lock);
+ dql_init(&wg->send_queue.dql, HZ);
wg->send_queue.worker = alloc_percpu(struct multicore_worker);
if (!wg->send_queue.worker)
goto error_6;
@@ -321,6 +322,7 @@ static int newlink(struct net *src_net, struct net_device *dev, struct nlattr *t
INIT_LIST_HEAD(&wg->receive_queue.list);
spin_lock_init(&wg->receive_queue.lock);
+ dql_init(&wg->receive_queue.dql, HZ);
wg->receive_queue.worker = alloc_percpu(struct multicore_worker);
if (!wg->receive_queue.worker)
goto error_7;
diff --git a/src/device.h b/src/device.h
index 9378b8b..cb9045e 100644
--- a/src/device.h
+++ b/src/device.h
@@ -30,6 +30,7 @@ struct crypt_queue {
union {
struct {
struct multicore_worker __percpu *worker;
+ struct dql dql;
int last_cpu;
};
struct work_struct work;