aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/src/receive.c
diff options
context:
space:
mode:
authorJason A. Donenfeld <Jason@zx2c4.com>2018-08-23 11:35:55 -0700
committerJason A. Donenfeld <Jason@zx2c4.com>2018-08-28 23:20:13 -0600
commitf103d1e2b420231c37684750cd36a825425fc313 (patch)
tree4a8d86eda58d149d85a2d99a53e9b2370fc58b5f /src/receive.c
parentwg-quick: check correct variable for route deduplication (diff)
downloadwireguard-monolithic-historical-f103d1e2b420231c37684750cd36a825425fc313.tar.xz
wireguard-monolithic-historical-f103d1e2b420231c37684750cd36a825425fc313.zip
global: run through clang-format
This is the worst commit in the whole repo, making the code much less readable, but so it goes with upstream maintainers. We are now woefully wrapped at 80 columns.
Diffstat (limited to 'src/receive.c')
-rw-r--r--src/receive.c257
1 files changed, 178 insertions, 79 deletions
diff --git a/src/receive.c b/src/receive.c
index 4e73da1..a2d1d9d 100644
--- a/src/receive.c
+++ b/src/receive.c
@@ -20,7 +20,8 @@
/* Must be called with bh disabled. */
static inline void rx_stats(struct wireguard_peer *peer, size_t len)
{
- struct pcpu_sw_netstats *tstats = get_cpu_ptr(peer->device->dev->tstats);
+ struct pcpu_sw_netstats *tstats =
+ get_cpu_ptr(peer->device->dev->tstats);
u64_stats_update_begin(&tstats->syncp);
++tstats->rx_packets;
@@ -36,38 +37,57 @@ static inline size_t validate_header_len(struct sk_buff *skb)
{
if (unlikely(skb->len < sizeof(struct message_header)))
return 0;
- if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) && skb->len >= MESSAGE_MINIMUM_LENGTH)
+ if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_DATA) &&
+ skb->len >= MESSAGE_MINIMUM_LENGTH)
return sizeof(struct message_data);
- if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) && skb->len == sizeof(struct message_handshake_initiation))
+ if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION) &&
+ skb->len == sizeof(struct message_handshake_initiation))
return sizeof(struct message_handshake_initiation);
- if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) && skb->len == sizeof(struct message_handshake_response))
+ if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE) &&
+ skb->len == sizeof(struct message_handshake_response))
return sizeof(struct message_handshake_response);
- if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) && skb->len == sizeof(struct message_handshake_cookie))
+ if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE) &&
+ skb->len == sizeof(struct message_handshake_cookie))
return sizeof(struct message_handshake_cookie);
return 0;
}
-static inline int skb_prepare_header(struct sk_buff *skb, struct wireguard_device *wg)
+static inline int skb_prepare_header(struct sk_buff *skb,
+ struct wireguard_device *wg)
{
- struct udphdr *udp;
size_t data_offset, data_len, header_len;
+ struct udphdr *udp;
- if (unlikely(skb_examine_untrusted_ip_hdr(skb) != skb->protocol || skb_transport_header(skb) < skb->head || (skb_transport_header(skb) + sizeof(struct udphdr)) > skb_tail_pointer(skb)))
+ if (unlikely(skb_examine_untrusted_ip_hdr(skb) != skb->protocol ||
+ skb_transport_header(skb) < skb->head ||
+ (skb_transport_header(skb) + sizeof(struct udphdr)) >
+ skb_tail_pointer(skb)))
return -EINVAL; /* Bogus IP header */
udp = udp_hdr(skb);
data_offset = (u8 *)udp - skb->data;
- if (unlikely(data_offset > U16_MAX || data_offset + sizeof(struct udphdr) > skb->len))
- return -EINVAL; /* Packet has offset at impossible location or isn't big enough to have UDP fields */
+ if (unlikely(data_offset > U16_MAX ||
+ data_offset + sizeof(struct udphdr) > skb->len))
+ /* Packet has offset at impossible location or isn't big enough
+ * to have UDP fields.
+ */
+ return -EINVAL;
data_len = ntohs(udp->len);
- if (unlikely(data_len < sizeof(struct udphdr) || data_len > skb->len - data_offset))
- return -EINVAL; /* UDP packet is reporting too small of a size or lying about its size */
+ if (unlikely(data_len < sizeof(struct udphdr) ||
+ data_len > skb->len - data_offset))
+ /* UDP packet is reporting too small of a size or lying about
+ * its size.
+ */
+ return -EINVAL;
data_len -= sizeof(struct udphdr);
data_offset = (u8 *)udp + sizeof(struct udphdr) - skb->data;
- if (unlikely(!pskb_may_pull(skb, data_offset + sizeof(struct message_header)) || pskb_trim(skb, data_len + data_offset) < 0))
+ if (unlikely(!pskb_may_pull(skb,
+ data_offset + sizeof(struct message_header)) ||
+ pskb_trim(skb, data_len + data_offset) < 0))
return -EINVAL;
skb_pull(skb, data_offset);
if (unlikely(skb->len != data_len))
- return -EINVAL; /* Final len does not agree with calculated len */
+ /* Final len does not agree with calculated len */
+ return -EINVAL;
header_len = validate_header_len(skb);
if (unlikely(!header_len))
return -EINVAL;
@@ -78,74 +98,96 @@ static inline int skb_prepare_header(struct sk_buff *skb, struct wireguard_devic
return 0;
}
-static void receive_handshake_packet(struct wireguard_device *wg, struct sk_buff *skb)
+static void receive_handshake_packet(struct wireguard_device *wg,
+ struct sk_buff *skb)
{
- static u64 last_under_load; /* Yes this is global, so that our load calculation applies to the whole system. */
struct wireguard_peer *peer = NULL;
- bool under_load;
enum cookie_mac_state mac_state;
+ /* This is global, so that our load calculation applies to
+ * the whole system.
+ */
+ static u64 last_under_load;
bool packet_needs_cookie;
+ bool under_load;
if (SKB_TYPE_LE32(skb) == cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE)) {
- net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n", wg->dev->name, skb);
- cookie_message_consume((struct message_handshake_cookie *)skb->data, wg);
+ net_dbg_skb_ratelimited("%s: Receiving cookie response from %pISpfsc\n",
+ wg->dev->name, skb);
+ cookie_message_consume(
+ (struct message_handshake_cookie *)skb->data, wg);
return;
}
- under_load = skb_queue_len(&wg->incoming_handshakes) >= MAX_QUEUED_INCOMING_HANDSHAKES / 8;
+ under_load = skb_queue_len(&wg->incoming_handshakes) >=
+ MAX_QUEUED_INCOMING_HANDSHAKES / 8;
if (under_load)
last_under_load = ktime_get_boot_fast_ns();
else if (last_under_load)
under_load = !has_expired(last_under_load, 1);
- mac_state = cookie_validate_packet(&wg->cookie_checker, skb, under_load);
- if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) || (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE))
+ mac_state = cookie_validate_packet(&wg->cookie_checker, skb,
+ under_load);
+ if ((under_load && mac_state == VALID_MAC_WITH_COOKIE) ||
+ (!under_load && mac_state == VALID_MAC_BUT_NO_COOKIE))
packet_needs_cookie = false;
else if (under_load && mac_state == VALID_MAC_BUT_NO_COOKIE)
packet_needs_cookie = true;
else {
- net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n", wg->dev->name, skb);
+ net_dbg_skb_ratelimited("%s: Invalid MAC of handshake, dropping packet from %pISpfsc\n",
+ wg->dev->name, skb);
return;
}
switch (SKB_TYPE_LE32(skb)) {
case cpu_to_le32(MESSAGE_HANDSHAKE_INITIATION): {
- struct message_handshake_initiation *message = (struct message_handshake_initiation *)skb->data;
+ struct message_handshake_initiation *message =
+ (struct message_handshake_initiation *)skb->data;
if (packet_needs_cookie) {
- packet_send_handshake_cookie(wg, skb, message->sender_index);
+ packet_send_handshake_cookie(wg, skb,
+ message->sender_index);
return;
}
peer = noise_handshake_consume_initiation(message, wg);
if (unlikely(!peer)) {
- net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n", wg->dev->name, skb);
+ net_dbg_skb_ratelimited("%s: Invalid handshake initiation from %pISpfsc\n",
+ wg->dev->name, skb);
return;
}
socket_set_peer_endpoint_from_skb(peer, skb);
- net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n", wg->dev->name, peer->internal_id, &peer->endpoint.addr);
+ net_dbg_ratelimited("%s: Receiving handshake initiation from peer %llu (%pISpfsc)\n",
+ wg->dev->name, peer->internal_id,
+ &peer->endpoint.addr);
packet_send_handshake_response(peer);
break;
}
case cpu_to_le32(MESSAGE_HANDSHAKE_RESPONSE): {
- struct message_handshake_response *message = (struct message_handshake_response *)skb->data;
+ struct message_handshake_response *message =
+ (struct message_handshake_response *)skb->data;
if (packet_needs_cookie) {
- packet_send_handshake_cookie(wg, skb, message->sender_index);
+ packet_send_handshake_cookie(wg, skb,
+ message->sender_index);
return;
}
peer = noise_handshake_consume_response(message, wg);
if (unlikely(!peer)) {
- net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n", wg->dev->name, skb);
+ net_dbg_skb_ratelimited("%s: Invalid handshake response from %pISpfsc\n",
+ wg->dev->name, skb);
return;
}
socket_set_peer_endpoint_from_skb(peer, skb);
- net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n", wg->dev->name, peer->internal_id, &peer->endpoint.addr);
- if (noise_handshake_begin_session(&peer->handshake, &peer->keypairs)) {
+ net_dbg_ratelimited("%s: Receiving handshake response from peer %llu (%pISpfsc)\n",
+ wg->dev->name, peer->internal_id,
+ &peer->endpoint.addr);
+ if (noise_handshake_begin_session(&peer->handshake,
+ &peer->keypairs)) {
timers_session_derived(peer);
timers_handshake_complete(peer);
- /* Calling this function will either send any existing packets in the queue
- * and not send a keepalive, which is the best case, Or, if there's nothing
- * in the queue, it will send a keepalive, in order to give immediate
- * confirmation of the session.
+ /* Calling this function will either send any existing
+ * packets in the queue and not send a keepalive, which
+ * is the best case, Or, if there's nothing in the
+ * queue, it will send a keepalive, in order to give
+ * immediate confirmation of the session.
*/
packet_send_keepalive(peer);
}
@@ -169,7 +211,8 @@ static void receive_handshake_packet(struct wireguard_device *wg, struct sk_buff
void packet_handshake_receive_worker(struct work_struct *work)
{
- struct wireguard_device *wg = container_of(work, struct multicore_worker, work)->ptr;
+ struct wireguard_device *wg =
+ container_of(work, struct multicore_worker, work)->ptr;
struct sk_buff *skb;
while ((skb = skb_dequeue(&wg->incoming_handshakes)) != NULL) {
@@ -189,8 +232,10 @@ static inline void keep_key_fresh(struct wireguard_peer *peer)
rcu_read_lock_bh();
keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
- if (likely(keypair && keypair->sending.is_valid) && keypair->i_am_the_initiator &&
- unlikely(has_expired(keypair->sending.birthdate, REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT)))
+ if (likely(keypair && keypair->sending.is_valid) &&
+ keypair->i_am_the_initiator &&
+ unlikely(has_expired(keypair->sending.birthdate,
+ REJECT_AFTER_TIME - KEEPALIVE_TIMEOUT - REKEY_TIMEOUT)))
send = true;
rcu_read_unlock_bh();
@@ -200,7 +245,9 @@ static inline void keep_key_fresh(struct wireguard_peer *peer)
}
}
-static inline bool skb_decrypt(struct sk_buff *skb, struct noise_symmetric_key *key, simd_context_t simd_context)
+static inline bool skb_decrypt(struct sk_buff *skb,
+ struct noise_symmetric_key *key,
+ simd_context_t simd_context)
{
struct scatterlist sg[MAX_SKB_FRAGS * 2 + 1];
struct sk_buff *trailer;
@@ -210,12 +257,15 @@ static inline bool skb_decrypt(struct sk_buff *skb, struct noise_symmetric_key *
if (unlikely(!key))
return false;
- if (unlikely(!key->is_valid || has_expired(key->birthdate, REJECT_AFTER_TIME) || key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) {
+ if (unlikely(!key->is_valid ||
+ has_expired(key->birthdate, REJECT_AFTER_TIME) ||
+ key->counter.receive.counter >= REJECT_AFTER_MESSAGES)) {
key->is_valid = false;
return false;
}
- PACKET_CB(skb)->nonce = le64_to_cpu(((struct message_data *)skb->data)->counter);
+ PACKET_CB(skb)->nonce =
+ le64_to_cpu(((struct message_data *)skb->data)->counter);
/* We ensure that the network header is part of the packet before we
* call skb_cow_data, so that there's no chance that data is removed
@@ -233,7 +283,9 @@ static inline bool skb_decrypt(struct sk_buff *skb, struct noise_symmetric_key *
if (skb_to_sgvec(skb, sg, 0, skb->len) <= 0)
return false;
- if (!chacha20poly1305_decrypt_sg(sg, sg, skb->len, NULL, 0, PACKET_CB(skb)->nonce, key->key, simd_context))
+ if (!chacha20poly1305_decrypt_sg(sg, sg, skb->len, NULL, 0,
+ PACKET_CB(skb)->nonce, key->key,
+ simd_context))
return false;
/* Another ugly situation of pushing and pulling the header so as to
@@ -248,33 +300,39 @@ static inline bool skb_decrypt(struct sk_buff *skb, struct noise_symmetric_key *
}
/* This is RFC6479, a replay detection bitmap algorithm that avoids bitshifts */
-static inline bool counter_validate(union noise_counter *counter, u64 their_counter)
+static inline bool counter_validate(union noise_counter *counter,
+ u64 their_counter)
{
- bool ret = false;
unsigned long index, index_current, top, i;
+ bool ret = false;
spin_lock_bh(&counter->receive.lock);
- if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 || their_counter >= REJECT_AFTER_MESSAGES))
+ if (unlikely(counter->receive.counter >= REJECT_AFTER_MESSAGES + 1 ||
+ their_counter >= REJECT_AFTER_MESSAGES))
goto out;
++their_counter;
- if (unlikely((COUNTER_WINDOW_SIZE + their_counter) < counter->receive.counter))
+ if (unlikely((COUNTER_WINDOW_SIZE + their_counter) <
+ counter->receive.counter))
goto out;
index = their_counter >> ilog2(BITS_PER_LONG);
if (likely(their_counter > counter->receive.counter)) {
index_current = counter->receive.counter >> ilog2(BITS_PER_LONG);
- top = min_t(unsigned long, index - index_current, COUNTER_BITS_TOTAL / BITS_PER_LONG);
+ top = min_t(unsigned long, index - index_current,
+ COUNTER_BITS_TOTAL / BITS_PER_LONG);
for (i = 1; i <= top; ++i)
- counter->receive.backtrack[(i + index_current) & ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0;
+ counter->receive.backtrack[(i + index_current) &
+ ((COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1)] = 0;
counter->receive.counter = their_counter;
}
index &= (COUNTER_BITS_TOTAL / BITS_PER_LONG) - 1;
- ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1), &counter->receive.backtrack[index]);
+ ret = !test_and_set_bit(their_counter & (BITS_PER_LONG - 1),
+ &counter->receive.backtrack[index]);
out:
spin_unlock_bh(&counter->receive.lock);
@@ -282,15 +340,18 @@ out:
}
#include "selftest/counter.h"
-static void packet_consume_data_done(struct wireguard_peer *peer, struct sk_buff *skb, struct endpoint *endpoint)
+static void packet_consume_data_done(struct wireguard_peer *peer,
+ struct sk_buff *skb,
+ struct endpoint *endpoint)
{
- struct wireguard_peer *routed_peer;
struct net_device *dev = peer->device->dev;
+ struct wireguard_peer *routed_peer;
unsigned int len, len_before_trim;
socket_set_peer_endpoint(peer, endpoint);
- if (unlikely(noise_received_with_keypair(&peer->keypairs, PACKET_CB(skb)->keypair))) {
+ if (unlikely(noise_received_with_keypair(&peer->keypairs,
+ PACKET_CB(skb)->keypair))) {
timers_handshake_complete(peer);
packet_send_staged_packets(peer);
}
@@ -303,7 +364,9 @@ static void packet_consume_data_done(struct wireguard_peer *peer, struct sk_buff
/* A packet with length 0 is a keepalive packet */
if (unlikely(!skb->len)) {
rx_stats(peer, message_data_len(0));
- net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n", dev->name, peer->internal_id, &peer->endpoint.addr);
+ net_dbg_ratelimited("%s: Receiving keepalive packet from peer %llu (%pISpfsc)\n",
+ dev->name, peer->internal_id,
+ &peer->endpoint.addr);
goto packet_processed;
}
@@ -311,7 +374,10 @@ static void packet_consume_data_done(struct wireguard_peer *peer, struct sk_buff
if (unlikely(skb_network_header(skb) < skb->head))
goto dishonest_packet_size;
- if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) && (ip_hdr(skb)->version == 4 || (ip_hdr(skb)->version == 6 && pskb_network_may_pull(skb, sizeof(struct ipv6hdr)))))))
+ if (unlikely(!(pskb_network_may_pull(skb, sizeof(struct iphdr)) &&
+ (ip_hdr(skb)->version == 4 ||
+ (ip_hdr(skb)->version == 6 &&
+ pskb_network_may_pull(skb, sizeof(struct ipv6hdr)))))))
goto dishonest_packet_type;
skb->dev = dev;
@@ -324,7 +390,8 @@ static void packet_consume_data_done(struct wireguard_peer *peer, struct sk_buff
if (INET_ECN_is_ce(PACKET_CB(skb)->ds))
IP_ECN_set_ce(ip_hdr(skb));
} else if (skb->protocol == htons(ETH_P_IPV6)) {
- len = ntohs(ipv6_hdr(skb)->payload_len) + sizeof(struct ipv6hdr);
+ len = ntohs(ipv6_hdr(skb)->payload_len) +
+ sizeof(struct ipv6hdr);
if (INET_ECN_is_ce(PACKET_CB(skb)->ds))
IP6_ECN_set_ce(skb, ipv6_hdr(skb));
} else
@@ -344,23 +411,29 @@ static void packet_consume_data_done(struct wireguard_peer *peer, struct sk_buff
if (unlikely(napi_gro_receive(&peer->napi, skb) == GRO_DROP)) {
++dev->stats.rx_dropped;
- net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n", dev->name, peer->internal_id, &peer->endpoint.addr);
+ net_dbg_ratelimited("%s: Failed to give packet to userspace from peer %llu (%pISpfsc)\n",
+ dev->name, peer->internal_id,
+ &peer->endpoint.addr);
} else
rx_stats(peer, message_data_len(len_before_trim));
return;
dishonest_packet_peer:
- net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n", dev->name, skb, peer->internal_id, &peer->endpoint.addr);
+ net_dbg_skb_ratelimited("%s: Packet has unallowed src IP (%pISc) from peer %llu (%pISpfsc)\n",
+ dev->name, skb, peer->internal_id,
+ &peer->endpoint.addr);
++dev->stats.rx_errors;
++dev->stats.rx_frame_errors;
goto packet_processed;
dishonest_packet_type:
- net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n", dev->name, peer->internal_id, &peer->endpoint.addr);
+ net_dbg_ratelimited("%s: Packet is neither ipv4 nor ipv6 from peer %llu (%pISpfsc)\n",
+ dev->name, peer->internal_id, &peer->endpoint.addr);
++dev->stats.rx_errors;
++dev->stats.rx_frame_errors;
goto packet_processed;
dishonest_packet_size:
- net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n", dev->name, peer->internal_id, &peer->endpoint.addr);
+ net_dbg_ratelimited("%s: Packet has incorrect size from peer %llu (%pISpfsc)\n",
+ dev->name, peer->internal_id, &peer->endpoint.addr);
++dev->stats.rx_errors;
++dev->stats.rx_length_errors;
goto packet_processed;
@@ -370,19 +443,22 @@ packet_processed:
int packet_rx_poll(struct napi_struct *napi, int budget)
{
- struct wireguard_peer *peer = container_of(napi, struct wireguard_peer, napi);
+ struct wireguard_peer *peer =
+ container_of(napi, struct wireguard_peer, napi);
struct crypt_queue *queue = &peer->rx_queue;
struct noise_keypair *keypair;
- struct sk_buff *skb;
struct endpoint endpoint;
enum packet_state state;
+ struct sk_buff *skb;
int work_done = 0;
bool free;
if (unlikely(budget <= 0))
return 0;
- while ((skb = __ptr_ring_peek(&queue->ring)) != NULL && (state = atomic_read_acquire(&PACKET_CB(skb)->state)) != PACKET_STATE_UNCRYPTED) {
+ while ((skb = __ptr_ring_peek(&queue->ring)) != NULL &&
+ (state = atomic_read_acquire(&PACKET_CB(skb)->state)) !=
+ PACKET_STATE_UNCRYPTED) {
__ptr_ring_discard_one(&queue->ring);
peer = PACKET_PEER(skb);
keypair = PACKET_CB(skb)->keypair;
@@ -391,8 +467,12 @@ int packet_rx_poll(struct napi_struct *napi, int budget)
if (unlikely(state != PACKET_STATE_CRYPTED))
goto next;
- if (unlikely(!counter_validate(&keypair->receiving.counter, PACKET_CB(skb)->nonce))) {
- net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n", peer->device->dev->name, PACKET_CB(skb)->nonce, keypair->receiving.counter.receive.counter);
+ if (unlikely(!counter_validate(&keypair->receiving.counter,
+ PACKET_CB(skb)->nonce))) {
+ net_dbg_ratelimited("%s: Packet has invalid nonce %llu (max %llu)\n",
+ peer->device->dev->name,
+ PACKET_CB(skb)->nonce,
+ keypair->receiving.counter.receive.counter);
goto next;
}
@@ -403,7 +483,7 @@ int packet_rx_poll(struct napi_struct *napi, int budget)
packet_consume_data_done(peer, skb, &endpoint);
free = false;
-next:
+ next:
noise_keypair_put(keypair, false);
peer_put(peer);
if (unlikely(free))
@@ -421,34 +501,46 @@ next:
void packet_decrypt_worker(struct work_struct *work)
{
- struct crypt_queue *queue = container_of(work, struct multicore_worker, work)->ptr;
- struct sk_buff *skb;
+ struct crypt_queue *queue =
+ container_of(work, struct multicore_worker, work)->ptr;
simd_context_t simd_context = simd_get();
+ struct sk_buff *skb;
while ((skb = ptr_ring_consume_bh(&queue->ring)) != NULL) {
- enum packet_state state = likely(skb_decrypt(skb, &PACKET_CB(skb)->keypair->receiving, simd_context)) ? PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
- queue_enqueue_per_peer_napi(&PACKET_PEER(skb)->rx_queue, skb, state);
+ enum packet_state state = likely(skb_decrypt(skb,
+ &PACKET_CB(skb)->keypair->receiving,
+ simd_context)) ?
+ PACKET_STATE_CRYPTED : PACKET_STATE_DEAD;
+ queue_enqueue_per_peer_napi(&PACKET_PEER(skb)->rx_queue, skb,
+ state);
simd_context = simd_relax(simd_context);
}
simd_put(simd_context);
}
-static void packet_consume_data(struct wireguard_device *wg, struct sk_buff *skb)
+static void packet_consume_data(struct wireguard_device *wg,
+ struct sk_buff *skb)
{
- struct wireguard_peer *peer = NULL;
__le32 idx = ((struct message_data *)skb->data)->key_idx;
+ struct wireguard_peer *peer = NULL;
int ret;
rcu_read_lock_bh();
- PACKET_CB(skb)->keypair = (struct noise_keypair *)index_hashtable_lookup(&wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx, &peer);
+ PACKET_CB(skb)->keypair =
+ (struct noise_keypair *)index_hashtable_lookup(
+ &wg->index_hashtable, INDEX_HASHTABLE_KEYPAIR, idx,
+ &peer);
if (unlikely(!noise_keypair_get(PACKET_CB(skb)->keypair)))
goto err_keypair;
if (unlikely(peer->is_dead))
goto err;
- ret = queue_enqueue_per_device_and_peer(&wg->decrypt_queue, &peer->rx_queue, skb, wg->packet_crypt_wq, &wg->decrypt_queue.last_cpu);
+ ret = queue_enqueue_per_device_and_peer(&wg->decrypt_queue,
+ &peer->rx_queue, skb,
+ wg->packet_crypt_wq,
+ &wg->decrypt_queue.last_cpu);
if (unlikely(ret == -EPIPE))
queue_enqueue_per_peer(&peer->rx_queue, skb, PACKET_STATE_DEAD);
if (likely(!ret || ret == -EPIPE)) {
@@ -473,14 +565,20 @@ void packet_receive(struct wireguard_device *wg, struct sk_buff *skb)
case cpu_to_le32(MESSAGE_HANDSHAKE_COOKIE): {
int cpu;
- if (skb_queue_len(&wg->incoming_handshakes) > MAX_QUEUED_INCOMING_HANDSHAKES || unlikely(!rng_is_initialized())) {
- net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n", wg->dev->name, skb);
+ if (skb_queue_len(&wg->incoming_handshakes) >
+ MAX_QUEUED_INCOMING_HANDSHAKES ||
+ unlikely(!rng_is_initialized())) {
+ net_dbg_skb_ratelimited("%s: Dropping handshake packet from %pISpfsc\n",
+ wg->dev->name, skb);
goto err;
}
skb_queue_tail(&wg->incoming_handshakes, skb);
- /* Queues up a call to packet_process_queued_handshake_packets(skb): */
+ /* Queues up a call to packet_process_queued_handshake_
+ * packets(skb):
+ */
cpu = cpumask_next_online(&wg->incoming_handshake_cpu);
- queue_work_on(cpu, wg->handshake_receive_wq, &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work);
+ queue_work_on(cpu, wg->handshake_receive_wq,
+ &per_cpu_ptr(wg->incoming_handshakes_worker, cpu)->work);
break;
}
case cpu_to_le32(MESSAGE_DATA):
@@ -488,7 +586,8 @@ void packet_receive(struct wireguard_device *wg, struct sk_buff *skb)
packet_consume_data(wg, skb);
break;
default:
- net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n", wg->dev->name, skb);
+ net_dbg_skb_ratelimited("%s: Invalid packet from %pISpfsc\n",
+ wg->dev->name, skb);
goto err;
}
return;