aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/net/tun.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/tun.c')
-rw-r--r--drivers/net/tun.c97
1 files changed, 74 insertions, 23 deletions
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index 005020042be9..a4fdad475594 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -188,6 +188,11 @@ struct tun_file {
struct xdp_rxq_info xdp_rxq;
};
+struct tun_page {
+ struct page *page;
+ int count;
+};
+
struct tun_flow_entry {
struct hlist_node hash_link;
struct rcu_head rcu;
@@ -196,7 +201,7 @@ struct tun_flow_entry {
u32 rxhash;
u32 rps_rxhash;
int queue_index;
- unsigned long updated;
+ unsigned long updated ____cacheline_aligned_in_smp;
};
#define TUN_NUM_FLOW_ENTRIES 1024
@@ -524,18 +529,17 @@ static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
unsigned long delay = tun->ageing_time;
u16 queue_index = tfile->queue_index;
- if (!rxhash)
- return;
- else
- head = &tun->flows[tun_hashfn(rxhash)];
+ head = &tun->flows[tun_hashfn(rxhash)];
rcu_read_lock();
e = tun_flow_find(head, rxhash);
if (likely(e)) {
/* TODO: keep queueing to old queue until it's empty? */
- e->queue_index = queue_index;
- e->updated = jiffies;
+ if (e->queue_index != queue_index)
+ e->queue_index = queue_index;
+ if (e->updated != jiffies)
+ e->updated = jiffies;
sock_rps_record_flow_hash(e->rps_rxhash);
} else {
spin_lock_bh(&tun->lock);
@@ -1249,6 +1253,21 @@ static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
}
}
+static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
+{
+ if (new_carrier) {
+ struct tun_struct *tun = netdev_priv(dev);
+
+ if (!tun->numqueues)
+ return -EPERM;
+
+ netif_carrier_on(dev);
+ } else {
+ netif_carrier_off(dev);
+ }
+ return 0;
+}
+
static const struct net_device_ops tun_netdev_ops = {
.ndo_uninit = tun_net_uninit,
.ndo_open = tun_net_open,
@@ -1258,6 +1277,7 @@ static const struct net_device_ops tun_netdev_ops = {
.ndo_select_queue = tun_select_queue,
.ndo_set_rx_headroom = tun_set_headroom,
.ndo_get_stats64 = tun_net_get_stats64,
+ .ndo_change_carrier = tun_net_change_carrier,
};
static void __tun_xdp_flush_tfile(struct tun_file *tfile)
@@ -1340,6 +1360,7 @@ static const struct net_device_ops tap_netdev_ops = {
.ndo_get_stats64 = tun_net_get_stats64,
.ndo_bpf = tun_xdp,
.ndo_xdp_xmit = tun_xdp_xmit,
+ .ndo_change_carrier = tun_net_change_carrier,
};
static void tun_flow_init(struct tun_struct *tun)
@@ -1473,23 +1494,22 @@ static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
skb->truesize += skb->data_len;
for (i = 1; i < it->nr_segs; i++) {
- struct page_frag *pfrag = &current->task_frag;
size_t fragsz = it->iov[i].iov_len;
+ struct page *page;
+ void *frag;
if (fragsz == 0 || fragsz > PAGE_SIZE) {
err = -EINVAL;
goto free;
}
-
- if (!skb_page_frag_refill(fragsz, pfrag, GFP_KERNEL)) {
+ frag = netdev_alloc_frag(fragsz);
+ if (!frag) {
err = -ENOMEM;
goto free;
}
-
- skb_fill_page_desc(skb, i - 1, pfrag->page,
- pfrag->offset, fragsz);
- page_ref_inc(pfrag->page);
- pfrag->offset += fragsz;
+ page = virt_to_head_page(frag);
+ skb_fill_page_desc(skb, i - 1, page,
+ frag - page_address(page), fragsz);
}
return skb;
@@ -2381,9 +2401,16 @@ static void tun_sock_write_space(struct sock *sk)
kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
}
+static void tun_put_page(struct tun_page *tpage)
+{
+ if (tpage->page)
+ __page_frag_cache_drain(tpage->page, tpage->count);
+}
+
static int tun_xdp_one(struct tun_struct *tun,
struct tun_file *tfile,
- struct xdp_buff *xdp, int *flush)
+ struct xdp_buff *xdp, int *flush,
+ struct tun_page *tpage)
{
unsigned int datasize = xdp->data_end - xdp->data;
struct tun_xdp_hdr *hdr = xdp->data_hard_start;
@@ -2395,6 +2422,7 @@ static int tun_xdp_one(struct tun_struct *tun,
int buflen = hdr->buflen;
int err = 0;
bool skb_xdp = false;
+ struct page *page;
xdp_prog = rcu_dereference(tun->xdp_prog);
if (xdp_prog) {
@@ -2421,7 +2449,14 @@ static int tun_xdp_one(struct tun_struct *tun,
case XDP_PASS:
break;
default:
- put_page(virt_to_head_page(xdp->data));
+ page = virt_to_head_page(xdp->data);
+ if (tpage->page == page) {
+ ++tpage->count;
+ } else {
+ tun_put_page(tpage);
+ tpage->page = page;
+ tpage->count = 1;
+ }
return 0;
}
}
@@ -2453,18 +2488,21 @@ build:
goto out;
}
- if (!rcu_dereference(tun->steering_prog))
+ if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&
+ !tfile->detached)
rxhash = __skb_get_hash_symmetric(skb);
skb_record_rx_queue(skb, tfile->queue_index);
netif_receive_skb(skb);
- stats = get_cpu_ptr(tun->pcpu_stats);
+ /* No need for get_cpu_ptr() here since this function is
+ * always called with bh disabled
+ */
+ stats = this_cpu_ptr(tun->pcpu_stats);
u64_stats_update_begin(&stats->syncp);
stats->rx_packets++;
stats->rx_bytes += datasize;
u64_stats_update_end(&stats->syncp);
- put_cpu_ptr(stats);
if (rxhash)
tun_flow_update(tun, rxhash, tfile);
@@ -2485,15 +2523,18 @@ static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
return -EBADFD;
if (ctl && (ctl->type == TUN_MSG_PTR)) {
+ struct tun_page tpage;
int n = ctl->num;
int flush = 0;
+ memset(&tpage, 0, sizeof(tpage));
+
local_bh_disable();
rcu_read_lock();
for (i = 0; i < n; i++) {
xdp = &((struct xdp_buff *)ctl->ptr)[i];
- tun_xdp_one(tun, tfile, xdp, &flush);
+ tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
}
if (flush)
@@ -2502,6 +2543,8 @@ static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
rcu_read_unlock();
local_bh_enable();
+ tun_put_page(&tpage);
+
ret = total_len;
goto out;
}
@@ -2978,12 +3021,12 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
struct net *net = sock_net(&tfile->sk);
struct tun_struct *tun;
void __user* argp = (void __user*)arg;
+ unsigned int ifindex, carrier;
struct ifreq ifr;
kuid_t owner;
kgid_t group;
int sndbuf;
int vnet_hdr_sz;
- unsigned int ifindex;
int le;
int ret;
bool do_notify = false;
@@ -3161,7 +3204,7 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
tun_debug(KERN_DEBUG, tun, "set hw address: %pM\n",
ifr.ifr_hwaddr.sa_data);
- ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr);
+ ret = dev_set_mac_address(tun->dev, &ifr.ifr_hwaddr, NULL);
break;
case TUNGETSNDBUF:
@@ -3267,6 +3310,14 @@ static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
break;
+ case TUNSETCARRIER:
+ ret = -EFAULT;
+ if (copy_from_user(&carrier, argp, sizeof(carrier)))
+ goto unlock;
+
+ ret = tun_net_change_carrier(tun->dev, (bool)carrier);
+ break;
+
default:
ret = -EINVAL;
break;
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.