aboutsummaryrefslogtreecommitdiffstatshomepage
diff options
context:
space:
mode:
authorSultan Alsawaf <sultan@kerneltoast.com>2019-02-02 14:13:03 -0800
committerJason A. Donenfeld <Jason@zx2c4.com>2019-02-03 18:27:33 +0100
commitd814238650b16d9fd990ce604feb94b6ee31f9eb (patch)
tree76e3bf51b05f48308cbb2d6c24fd605eb60b266d
parentqueueing: more reasonable allocator function convention (diff)
downloadwireguard-monolithic-historical-d814238650b16d9fd990ce604feb94b6ee31f9eb.tar.xz
wireguard-monolithic-historical-d814238650b16d9fd990ce604feb94b6ee31f9eb.zip
hashtables: decouple hashtable allocations from the main device allocation
The hashtable allocations are quite large, and cause the device allocation in the net framework to stall sometimes while it tries to find a contiguous region that can fit the device struct: [<0000000000000000>] __switch_to+0x94/0xb8 [<0000000000000000>] __alloc_pages_nodemask+0x764/0x7e8 [<0000000000000000>] kmalloc_order+0x20/0x40 [<0000000000000000>] __kmalloc+0x144/0x1a0 [<0000000000000000>] alloc_netdev_mqs+0x5c/0x368 [<0000000000000000>] rtnl_create_link+0x48/0x180 [<0000000000000000>] rtnl_newlink+0x410/0x708 [<0000000000000000>] rtnetlink_rcv_msg+0x190/0x1f8 [<0000000000000000>] netlink_rcv_skb+0x4c/0xf8 [<0000000000000000>] rtnetlink_rcv+0x30/0x40 [<0000000000000000>] netlink_unicast+0x18c/0x208 [<0000000000000000>] netlink_sendmsg+0x19c/0x348 [<0000000000000000>] sock_sendmsg+0x3c/0x58 [<0000000000000000>] ___sys_sendmsg+0x290/0x2b0 [<0000000000000000>] __sys_sendmsg+0x58/0xa0 [<0000000000000000>] SyS_sendmsg+0x10/0x20 [<0000000000000000>] el0_svc_naked+0x34/0x38 [<0000000000000000>] 0xffffffffffffffff To fix the allocation stalls, decouple the hashtable allocations from the device allocation and allocate the hashtables with kvmalloc's implicit __GFP_NORETRY so that the allocations fall back to vmalloc with little resistance. Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
-rw-r--r--src/cookie.c2
-rw-r--r--src/device.c18
-rw-r--r--src/device.h4
-rw-r--r--src/hashtables.c16
-rw-r--r--src/hashtables.h4
-rw-r--r--src/netlink.c4
-rw-r--r--src/noise.c18
-rw-r--r--src/peer.c6
-rw-r--r--src/receive.c2
9 files changed, 49 insertions, 25 deletions
diff --git a/src/cookie.c b/src/cookie.c
index 97ab28a..bd23a14 100644
--- a/src/cookie.c
+++ b/src/cookie.c
@@ -202,7 +202,7 @@ void wg_cookie_message_consume(struct message_handshake_cookie *src,
u8 cookie[COOKIE_LEN];
bool ret;
- if (unlikely(!wg_index_hashtable_lookup(&wg->index_hashtable,
+ if (unlikely(!wg_index_hashtable_lookup(wg->index_hashtable,
INDEX_HASHTABLE_HANDSHAKE |
INDEX_HASHTABLE_KEYPAIR,
src->receiver_index, &peer)))
diff --git a/src/device.c b/src/device.c
index 4b4cca1..15911b2 100644
--- a/src/device.c
+++ b/src/device.c
@@ -253,6 +253,8 @@ static void wg_destruct(struct net_device *dev)
free_percpu(wg->incoming_handshakes_worker);
if (wg->have_creating_net_ref)
put_net(wg->creating_net);
+ kvfree(wg->index_hashtable);
+ kvfree(wg->peer_hashtable);
mutex_unlock(&wg->device_update_lock);
pr_debug("%s: Interface deleted\n", dev->name);
@@ -309,16 +311,22 @@ static int wg_newlink(struct net *src_net, struct net_device *dev,
mutex_init(&wg->socket_update_lock);
mutex_init(&wg->device_update_lock);
skb_queue_head_init(&wg->incoming_handshakes);
- wg_pubkey_hashtable_init(&wg->peer_hashtable);
- wg_index_hashtable_init(&wg->index_hashtable);
wg_allowedips_init(&wg->peer_allowedips);
wg_cookie_checker_init(&wg->cookie_checker, wg);
INIT_LIST_HEAD(&wg->peer_list);
wg->device_update_gen = 1;
+ wg->peer_hashtable = wg_pubkey_hashtable_alloc();
+ if (!wg->peer_hashtable)
+ return ret;
+
+ wg->index_hashtable = wg_index_hashtable_alloc();
+ if (!wg->index_hashtable)
+ goto err_free_peer_hashtable;
+
dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
if (!dev->tstats)
- return ret;
+ goto err_free_index_hashtable;
wg->incoming_handshakes_worker =
wg_packet_percpu_multicore_worker_alloc(
@@ -385,6 +393,10 @@ err_free_incoming_handshakes:
free_percpu(wg->incoming_handshakes_worker);
err_free_tstats:
free_percpu(dev->tstats);
+err_free_index_hashtable:
+ kvfree(wg->index_hashtable);
+err_free_peer_hashtable:
+ kvfree(wg->peer_hashtable);
return ret;
}
diff --git a/src/device.h b/src/device.h
index b85638c..7e7e216 100644
--- a/src/device.h
+++ b/src/device.h
@@ -48,8 +48,8 @@ struct wg_device {
int incoming_handshake_cpu;
struct multicore_worker __percpu *incoming_handshakes_worker;
struct cookie_checker cookie_checker;
- struct pubkey_hashtable peer_hashtable;
- struct index_hashtable index_hashtable;
+ struct pubkey_hashtable *peer_hashtable;
+ struct index_hashtable *index_hashtable;
struct allowedips peer_allowedips;
struct mutex device_update_lock, socket_update_lock;
struct list_head device_list, peer_list;
diff --git a/src/hashtables.c b/src/hashtables.c
index 18cac91..8aedc17 100644
--- a/src/hashtables.c
+++ b/src/hashtables.c
@@ -19,11 +19,17 @@ static struct hlist_head *pubkey_bucket(struct pubkey_hashtable *table,
return &table->hashtable[hash & (HASH_SIZE(table->hashtable) - 1)];
}
-void wg_pubkey_hashtable_init(struct pubkey_hashtable *table)
+struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void)
{
+ struct pubkey_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL);
+
+ if (!table)
+ return NULL;
+
get_random_bytes(&table->key, sizeof(table->key));
hash_init(table->hashtable);
mutex_init(&table->lock);
+ return table;
}
void wg_pubkey_hashtable_add(struct pubkey_hashtable *table,
@@ -74,10 +80,16 @@ static struct hlist_head *index_bucket(struct index_hashtable *table,
(HASH_SIZE(table->hashtable) - 1)];
}
-void wg_index_hashtable_init(struct index_hashtable *table)
+struct index_hashtable *wg_index_hashtable_alloc(void)
{
+ struct index_hashtable *table = kvmalloc(sizeof(*table), GFP_KERNEL);
+
+ if (!table)
+ return NULL;
+
hash_init(table->hashtable);
spin_lock_init(&table->lock);
+ return table;
}
/* At the moment, we limit ourselves to 2^20 total peers, which generally might
diff --git a/src/hashtables.h b/src/hashtables.h
index 7b12b86..de77537 100644
--- a/src/hashtables.h
+++ b/src/hashtables.h
@@ -21,7 +21,7 @@ struct pubkey_hashtable {
struct mutex lock;
};
-void wg_pubkey_hashtable_init(struct pubkey_hashtable *table);
+struct pubkey_hashtable *wg_pubkey_hashtable_alloc(void);
void wg_pubkey_hashtable_add(struct pubkey_hashtable *table,
struct wg_peer *peer);
void wg_pubkey_hashtable_remove(struct pubkey_hashtable *table,
@@ -48,7 +48,7 @@ struct index_hashtable_entry {
__le32 index;
};
-void wg_index_hashtable_init(struct index_hashtable *table);
+struct index_hashtable *wg_index_hashtable_alloc(void);
__le32 wg_index_hashtable_insert(struct index_hashtable *table,
struct index_hashtable_entry *entry);
bool wg_index_hashtable_replace(struct index_hashtable *table,
diff --git a/src/netlink.c b/src/netlink.c
index 3458c81..9a33192 100644
--- a/src/netlink.c
+++ b/src/netlink.c
@@ -368,7 +368,7 @@ static int set_peer(struct wg_device *wg, struct nlattr **attrs)
goto out;
}
- peer = wg_pubkey_hashtable_lookup(&wg->peer_hashtable,
+ peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable,
nla_data(attrs[WGPEER_A_PUBLIC_KEY]));
if (!peer) { /* Peer doesn't exist yet. Add a new one. */
ret = -ENODEV;
@@ -524,7 +524,7 @@ static int wg_set_device(struct sk_buff *skb, struct genl_info *info)
* two 25519-genpub ops.
*/
if (curve25519_generate_public(public_key, private_key)) {
- peer = wg_pubkey_hashtable_lookup(&wg->peer_hashtable,
+ peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable,
public_key);
if (peer) {
wg_peer_put(peer);
diff --git a/src/noise.c b/src/noise.c
index 1a85723..e1d4519 100644
--- a/src/noise.c
+++ b/src/noise.c
@@ -93,13 +93,13 @@ static void handshake_zero(struct noise_handshake *handshake)
void wg_noise_handshake_clear(struct noise_handshake *handshake)
{
wg_index_hashtable_remove(
- &handshake->entry.peer->device->index_hashtable,
+ handshake->entry.peer->device->index_hashtable,
&handshake->entry);
down_write(&handshake->lock);
handshake_zero(handshake);
up_write(&handshake->lock);
wg_index_hashtable_remove(
- &handshake->entry.peer->device->index_hashtable,
+ handshake->entry.peer->device->index_hashtable,
&handshake->entry);
}
@@ -130,7 +130,7 @@ static void keypair_free_kref(struct kref *kref)
keypair->entry.peer->device->dev->name,
keypair->internal_id,
keypair->entry.peer->internal_id);
- wg_index_hashtable_remove(&keypair->entry.peer->device->index_hashtable,
+ wg_index_hashtable_remove(keypair->entry.peer->device->index_hashtable,
&keypair->entry);
call_rcu_bh(&keypair->rcu, keypair_free_rcu);
}
@@ -141,7 +141,7 @@ void wg_noise_keypair_put(struct noise_keypair *keypair, bool unreference_now)
return;
if (unlikely(unreference_now))
wg_index_hashtable_remove(
- &keypair->entry.peer->device->index_hashtable,
+ keypair->entry.peer->device->index_hashtable,
&keypair->entry);
kref_put(&keypair->refcount, keypair_free_kref);
}
@@ -520,7 +520,7 @@ wg_noise_handshake_create_initiation(struct message_handshake_initiation *dst,
NOISE_TIMESTAMP_LEN, key, handshake->hash);
dst->sender_index = wg_index_hashtable_insert(
- &handshake->entry.peer->device->index_hashtable,
+ handshake->entry.peer->device->index_hashtable,
&handshake->entry);
handshake->state = HANDSHAKE_CREATED_INITIATION;
@@ -566,7 +566,7 @@ wg_noise_handshake_consume_initiation(struct message_handshake_initiation *src,
goto out;
/* Lookup which peer we're actually talking to */
- peer = wg_pubkey_hashtable_lookup(&wg->peer_hashtable, s);
+ peer = wg_pubkey_hashtable_lookup(wg->peer_hashtable, s);
if (!peer)
goto out;
handshake = &peer->handshake;
@@ -660,7 +660,7 @@ bool wg_noise_handshake_create_response(struct message_handshake_response *dst,
message_encrypt(dst->encrypted_nothing, NULL, 0, key, handshake->hash);
dst->sender_index = wg_index_hashtable_insert(
- &handshake->entry.peer->device->index_hashtable,
+ handshake->entry.peer->device->index_hashtable,
&handshake->entry);
handshake->state = HANDSHAKE_CREATED_RESPONSE;
@@ -693,7 +693,7 @@ wg_noise_handshake_consume_response(struct message_handshake_response *src,
goto out;
handshake = (struct noise_handshake *)wg_index_hashtable_lookup(
- &wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE,
+ wg->index_hashtable, INDEX_HASHTABLE_HANDSHAKE,
src->receiver_index, &peer);
if (unlikely(!handshake))
goto out;
@@ -793,7 +793,7 @@ bool wg_noise_handshake_begin_session(struct noise_handshake *handshake,
new_keypair->internal_id,
handshake->entry.peer->internal_id);
ret = wg_index_hashtable_replace(
- &handshake->entry.peer->device->index_hashtable,
+ handshake->entry.peer->device->index_hashtable,
&handshake->entry, &new_keypair->entry);
} else {
kzfree(new_keypair);
diff --git a/src/peer.c b/src/peer.c
index 020a97b..b786381 100644
--- a/src/peer.c
+++ b/src/peer.c
@@ -64,7 +64,7 @@ struct wg_peer *wg_peer_create(struct wg_device *wg,
NAPI_POLL_WEIGHT);
napi_enable(&peer->napi);
list_add_tail(&peer->peer_list, &wg->peer_list);
- wg_pubkey_hashtable_add(&wg->peer_hashtable, peer);
+ wg_pubkey_hashtable_add(wg->peer_hashtable, peer);
++wg->num_peers;
pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id);
return peer;
@@ -101,7 +101,7 @@ void wg_peer_remove(struct wg_peer *peer)
list_del_init(&peer->peer_list);
wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer,
&peer->device->device_update_lock);
- wg_pubkey_hashtable_remove(&peer->device->peer_hashtable, peer);
+ wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer);
/* Mark as dead, so that we don't allow jumping contexts after. */
WRITE_ONCE(peer->is_dead, true);
@@ -186,7 +186,7 @@ static void kref_release(struct kref *refcount)
/* Remove ourself from dynamic runtime lookup structures, now that the
* last reference is gone.
*/
- wg_index_hashtable_remove(&peer->device->index_hashtable,
+ wg_index_hashtable_remove(peer->device->index_hashtable,
&peer->handshake.entry);
/* Remove any lingering packets that didn't have a chance to be
diff --git a/src/receive.c b/src/receive.c
index 693ec57..51d06d3 100644
--- a/src/receive.c
+++ b/src/receive.c
@@ -538,7 +538,7 @@ static void wg_packet_consume_data(struct wg_device *wg, struct sk_buff *skb)
rcu_read_lock_bh();
PACKET_CB(skb)->keypair =
(struct noise_keypair *)wg_index_hashtable_lookup(
- &wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx,
+ wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx,
&peer);
if (unlikely(!wg_noise_keypair_get(PACKET_CB(skb)->keypair)))
goto err_keypair;