diff options
Diffstat (limited to 'src/send.c')
| -rw-r--r-- | src/send.c | 266 |
1 files changed, 266 insertions, 0 deletions
diff --git a/src/send.c b/src/send.c new file mode 100644 index 0000000..f43e4a3 --- /dev/null +++ b/src/send.c @@ -0,0 +1,266 @@ +/* Copyright 2015-2016 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. */ + +#include "wireguard.h" +#include "packets.h" +#include "timers.h" +#include "device.h" +#include "socket.h" +#include "messages.h" +#include "cookie.h" +#include <net/udp.h> +#include <net/sock.h> +#include <linux/uio.h> +#include <linux/inetdevice.h> +#include <linux/socket.h> +#include <linux/jiffies.h> + +void packet_send_handshake_initiation(struct wireguard_peer *peer) +{ + struct message_handshake_initiation packet; + + net_dbg_ratelimited("Sending handshake initiation to peer %Lu (%pISpfsc)\n", peer->internal_id, &peer->endpoint_addr); + peer->last_sent_handshake = get_jiffies_64(); + + if (noise_handshake_create_initiation(&packet, &peer->handshake)) { + cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + socket_send_buffer_to_peer(peer, &packet, sizeof(struct message_handshake_initiation), HANDSHAKE_DSCP); + timers_handshake_initiated(peer); + } +} + +void packet_send_handshake_response(struct wireguard_peer *peer) +{ + struct message_handshake_response packet; + + net_dbg_ratelimited("Sending handshake response to peer %Lu (%pISpfsc)\n", peer->internal_id, &peer->endpoint_addr); + peer->last_sent_handshake = get_jiffies_64(); + + if (noise_handshake_create_response(&packet, &peer->handshake)) { + cookie_add_mac_to_packet(&packet, sizeof(packet), peer); + if (noise_handshake_begin_session(&peer->handshake, &peer->keypairs, false)) { + timers_ephemeral_key_created(peer); + socket_send_buffer_to_peer(peer, &packet, sizeof(struct message_handshake_response), HANDSHAKE_DSCP); + } + } +} + +void packet_send_queued_handshakes(struct work_struct *work) +{ + struct wireguard_peer *peer = container_of(work, struct wireguard_peer, transmit_handshake_work); + peer->last_sent_handshake = get_jiffies_64(); + packet_send_handshake_initiation(peer); + peer_put(peer); +} + +void packet_queue_send_handshake_initiation(struct wireguard_peer *peer) +{ + rcu_read_lock(); + peer = peer_get(peer); + rcu_read_unlock(); + if (!peer) + return; + /* Queues up calling packet_send_queued_handshakes(peer), where we do a peer_put(peer) after: */ + if (!queue_work(peer->device->workqueue, &peer->transmit_handshake_work)) + peer_put(peer); /* If the work was already queued, we want to drop the extra reference */ +} + +static inline void ratelimit_packet_send_handshake_initiation(struct wireguard_peer *peer) +{ + if (time_is_before_jiffies64(peer->last_sent_handshake + REKEY_TIMEOUT)) + packet_queue_send_handshake_initiation(peer); +} + +void packet_send_handshake_cookie(struct wireguard_device *wg, struct sk_buff *initiating_skb, void *data, size_t data_len, __le32 sender_index) +{ + struct message_handshake_cookie packet; + +#ifdef DEBUG + struct sockaddr_storage addr = { 0 }; + if (initiating_skb) + socket_addr_from_skb(&addr, initiating_skb); + net_dbg_ratelimited("Sending cookie response for denied handshake message for %pISpfsc\n", &addr); +#endif + cookie_message_create(&packet, initiating_skb, data, data_len, sender_index, &wg->cookie_checker); + socket_send_buffer_as_reply_to_skb(initiating_skb, &packet, sizeof(packet), wg); +} + +static inline void keep_key_fresh(struct wireguard_peer *peer) +{ + struct noise_keypair *keypair; + unsigned long rekey_after_time = REKEY_AFTER_TIME; + + rcu_read_lock(); + keypair = rcu_dereference(peer->keypairs.current_keypair); + if (unlikely(!keypair || !keypair->sending.is_valid)) { + rcu_read_unlock(); + return; + } + + /* We don't want both peers initiating a new handshake at the same time */ + if (!keypair->i_am_the_initiator) + rekey_after_time += REKEY_TIMEOUT * 2; + + if (atomic64_read(&keypair->sending.counter.counter) > REKEY_AFTER_MESSAGES || + time_is_before_eq_jiffies64(keypair->sending.birthdate + rekey_after_time)) { + rcu_read_unlock(); + ratelimit_packet_send_handshake_initiation(peer); + } else + rcu_read_unlock(); +} + +void packet_send_keepalive(struct wireguard_peer *peer) +{ + struct sk_buff *skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH, GFP_ATOMIC); + if (unlikely(!skb)) + return; + skb_reserve(skb, DATA_PACKET_HEAD_ROOM); + skb->dev = netdev_pub(peer->device); + skb_queue_tail(&peer->tx_packet_queue, skb); + packet_send_queue(peer); +} + +struct packet_bundle { + atomic_t count; + struct sk_buff *first; +}; + +static inline void send_off_bundle(struct packet_bundle *bundle, struct wireguard_peer *peer) +{ + struct sk_buff *skb, *next; + for (skb = bundle->first; skb; skb = next) { + /* We store the next pointer locally because socket_send_skb_to_peer + * consumes the packet before the top of the loop comes again. */ + next = skb->next; + if (likely(!socket_send_skb_to_peer(peer, skb, 0 /* TODO: Should we copy the DSCP value from the enclosed packet? */))) + timers_data_sent(peer); + } +} + +static void message_create_data_done(struct sk_buff *skb, struct wireguard_peer *peer) +{ + struct packet_bundle *bundle = *((struct packet_bundle **)skb->cb); + /* A packet completed successfully, so we deincrement the counter of packets + * remaining, and if we hit zero we can send it off. */ + if (atomic_dec_and_test(&bundle->count)) + send_off_bundle(bundle, peer); + keep_key_fresh(peer); +} + +int packet_send_queue(struct wireguard_peer *peer) +{ + struct packet_bundle *bundle; + struct sk_buff_head local_queue; + struct sk_buff *skb, *next, *first; + unsigned long flags; + bool parallel = true; + + /* Steal the current queue into our local one. */ + skb_queue_head_init(&local_queue); + spin_lock_irqsave(&peer->tx_packet_queue.lock, flags); + skb_queue_splice_init(&peer->tx_packet_queue, &local_queue); + spin_unlock_irqrestore(&peer->tx_packet_queue.lock, flags); + + first = skb_peek(&local_queue); + if (unlikely(!first)) + goto out; + + /* Remove the circularity from the queue, so that we can iterate on + * on the skbs themselves. */ + local_queue.prev->next = local_queue.next->prev = NULL; + + /* The first pointer of the control block is a pointer to the bundle + * and after that, in the first packet only, is where we actually store + * the bundle data. This saves us a call to kmalloc. */ + bundle = (struct packet_bundle *)(first->cb + sizeof(void *)); + atomic_set(&bundle->count, skb_queue_len(&local_queue)); + bundle->first = first; + + /* Non-parallel path for the case of only one packet that's small */ + if (skb_queue_len(&local_queue) == 1 && first->len <= 256) + parallel = false; + + for (skb = first; skb; skb = next) { + /* We store the next pointer locally because we might free skb + * before the top of the loop comes again. */ + next = skb->next; + + /* We set the first pointer in cb to point to the bundle data. */ + *(struct packet_bundle **)skb->cb = bundle; + + /* We submit it for encryption and sending. */ + switch (packet_create_data(skb, peer, message_create_data_done, parallel)) { + case 0: + /* If all goes well, we can simply deincrement the queue counter. Even + * though skb_dequeue() would do this for us, we don't want to break the + * links between packets, so we just traverse the list normally and + * deincrement the counter manually each time a packet is consumed. */ + --local_queue.qlen; + break; + case -ENOKEY: + /* ENOKEY means that we don't have a valid session for the peer, which + * means we should initiate a session, and then requeue everything. */ + ratelimit_packet_send_handshake_initiation(peer); + /* Fall through */ + case -EBUSY: + /* EBUSY happens when the parallel workers are all filled up, in which + * case we should requeue everything. */ + if (skb->prev) { + /* Since we're requeuing skb and everything after skb, we make + * sure that the previously successfully sent packets don't link + * to the requeued packets, which will be sent independently the + * next time this function is called. */ + skb->prev->next = NULL; + skb->prev = NULL; + } + if (atomic_sub_and_test(local_queue.qlen, &bundle->count)) { + /* We remove the requeued packets from the count of total packets + * that were successfully submitted, which means we then must see + * if we were the ones to get it to zero. If we are at zero, we + * only send the previous successful packets if there actually were + * packets that succeeded before skb. */ + if (skb != first) + send_off_bundle(bundle, peer); + } + /* We stick the remaining skbs from local_queue at the top of the peer's + * queue again, setting the top of local_queue to be the skb that begins + * the requeueing. */ + local_queue.next = skb; + spin_lock_irqsave(&peer->tx_packet_queue.lock, flags); + skb_queue_splice(&local_queue, &peer->tx_packet_queue); + spin_unlock_irqrestore(&peer->tx_packet_queue.lock, flags); + goto out; + default: + /* If we failed for any other reason, we want to just free the packet and + * forget about it, so we first deincrement the queue counter as in the + * successful case above. */ + --local_queue.qlen; + if (skb == first && next) { + /* If it's the first one that failed, we need to move the bundle data + * to the next packet. Then, all subsequent assignments of the bundle + * pointer will be to the moved data. */ + *(struct packet_bundle *)(next->cb + sizeof(void *)) = *bundle; + bundle = (struct packet_bundle *)(next->cb + sizeof(void *)); + bundle->first = next; + } + /* We remove the skb from the list and free it. */ + if (skb->prev) + skb->prev->next = skb->next; + if (skb->next) + skb->next->prev = skb->prev; + kfree_skb(skb); + if (atomic_dec_and_test(&bundle->count)) { + /* As above, if this failed packet pushes the count to zero, we have to + * be the ones to send it off only in the case that there's something to + * send. */ + if (skb != first) + send_off_bundle(bundle, peer); + } + /* Only at the bottom do we update our local `first` variable, because we need it + * in the check above. But it's important that bundle->first is updated earlier when + * actually moving the bundle. */ + first = bundle->first; + } + } +out: + return NETDEV_TX_OK; +} |