aboutsummaryrefslogtreecommitdiffstats
path: root/net/unix/af_unix.c
diff options
context:
space:
mode:
Diffstat (limited to 'net/unix/af_unix.c')
-rw-r--r--net/unix/af_unix.c1096
1 files changed, 718 insertions, 378 deletions
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index b0bfc78e421c..b3545fc68097 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -89,6 +89,7 @@
#include <linux/socket.h>
#include <linux/un.h>
#include <linux/fcntl.h>
+#include <linux/filter.h>
#include <linux/termios.h>
#include <linux/sockios.h>
#include <linux/net.h>
@@ -117,24 +118,70 @@
#include "scm.h"
-struct hlist_head unix_socket_table[2 * UNIX_HASH_SIZE];
-EXPORT_SYMBOL_GPL(unix_socket_table);
-DEFINE_SPINLOCK(unix_table_lock);
-EXPORT_SYMBOL_GPL(unix_table_lock);
static atomic_long_t unix_nr_socks;
+static struct hlist_head bsd_socket_buckets[UNIX_HASH_SIZE / 2];
+static spinlock_t bsd_socket_locks[UNIX_HASH_SIZE / 2];
+/* SMP locking strategy:
+ * hash table is protected with spinlock.
+ * each socket state is protected by separate spinlock.
+ */
-static struct hlist_head *unix_sockets_unbound(void *addr)
+static unsigned int unix_unbound_hash(struct sock *sk)
{
- unsigned long hash = (unsigned long)addr;
+ unsigned long hash = (unsigned long)sk;
hash ^= hash >> 16;
hash ^= hash >> 8;
- hash %= UNIX_HASH_SIZE;
- return &unix_socket_table[UNIX_HASH_SIZE + hash];
+ hash ^= sk->sk_type;
+
+ return hash & UNIX_HASH_MOD;
}
-#define UNIX_ABSTRACT(sk) (unix_sk(sk)->addr->hash < UNIX_HASH_SIZE)
+static unsigned int unix_bsd_hash(struct inode *i)
+{
+ return i->i_ino & UNIX_HASH_MOD;
+}
+
+static unsigned int unix_abstract_hash(struct sockaddr_un *sunaddr,
+ int addr_len, int type)
+{
+ __wsum csum = csum_partial(sunaddr, addr_len, 0);
+ unsigned int hash;
+
+ hash = (__force unsigned int)csum_fold(csum);
+ hash ^= hash >> 8;
+ hash ^= type;
+
+ return UNIX_HASH_MOD + 1 + (hash & UNIX_HASH_MOD);
+}
+
+static void unix_table_double_lock(struct net *net,
+ unsigned int hash1, unsigned int hash2)
+{
+ if (hash1 == hash2) {
+ spin_lock(&net->unx.table.locks[hash1]);
+ return;
+ }
+
+ if (hash1 > hash2)
+ swap(hash1, hash2);
+
+ spin_lock(&net->unx.table.locks[hash1]);
+ spin_lock_nested(&net->unx.table.locks[hash2], SINGLE_DEPTH_NESTING);
+}
+
+static void unix_table_double_unlock(struct net *net,
+ unsigned int hash1, unsigned int hash2)
+{
+ if (hash1 == hash2) {
+ spin_unlock(&net->unx.table.locks[hash1]);
+ return;
+ }
+
+ spin_unlock(&net->unx.table.locks[hash1]);
+ spin_unlock(&net->unx.table.locks[hash2]);
+}
#ifdef CONFIG_SECURITY_NETWORK
static void unix_get_secdata(struct scm_cookie *scm, struct sk_buff *skb)
@@ -164,20 +211,6 @@ static inline bool unix_secdata_eq(struct scm_cookie *scm, struct sk_buff *skb)
}
#endif /* CONFIG_SECURITY_NETWORK */
-/*
- * SMP locking strategy:
- * hash table is protected with spinlock unix_table_lock
- * each socket state is protected by separate spin lock.
- */
-
-static inline unsigned int unix_hash_fold(__wsum n)
-{
- unsigned int hash = (__force unsigned int)csum_fold(n);
-
- hash ^= hash>>8;
- return hash&(UNIX_HASH_SIZE-1);
-}
-
#define unix_peer(sk) (unix_sk(sk)->peer)
static inline int unix_our_peer(struct sock *sk, struct sock *osk)
@@ -214,6 +247,22 @@ struct sock *unix_peer_get(struct sock *s)
}
EXPORT_SYMBOL_GPL(unix_peer_get);
+static struct unix_address *unix_create_addr(struct sockaddr_un *sunaddr,
+ int addr_len)
+{
+ struct unix_address *addr;
+
+ addr = kmalloc(sizeof(*addr) + addr_len, GFP_KERNEL);
+ if (!addr)
+ return NULL;
+
+ refcount_set(&addr->refcnt, 1);
+ addr->len = addr_len;
+ memcpy(addr->name, sunaddr, addr_len);
+
+ return addr;
+}
+
static inline void unix_release_addr(struct unix_address *addr)
{
if (refcount_dec_and_test(&addr->refcnt))
@@ -227,29 +276,29 @@ static inline void unix_release_addr(struct unix_address *addr)
* - if started by zero, it is abstract name.
*/
-static int unix_mkname(struct sockaddr_un *sunaddr, int len, unsigned int *hashp)
+static int unix_validate_addr(struct sockaddr_un *sunaddr, int addr_len)
{
- *hashp = 0;
-
- if (len <= sizeof(short) || len > sizeof(*sunaddr))
+ if (addr_len <= offsetof(struct sockaddr_un, sun_path) ||
+ addr_len > sizeof(*sunaddr))
return -EINVAL;
- if (!sunaddr || sunaddr->sun_family != AF_UNIX)
+
+ if (sunaddr->sun_family != AF_UNIX)
return -EINVAL;
- if (sunaddr->sun_path[0]) {
- /*
- * This may look like an off by one error but it is a bit more
- * subtle. 108 is the longest valid AF_UNIX path for a binding.
- * sun_path[108] doesn't as such exist. However in kernel space
- * we are guaranteed that it is a valid memory location in our
- * kernel address buffer.
- */
- ((char *)sunaddr)[len] = 0;
- len = strlen(sunaddr->sun_path)+1+sizeof(short);
- return len;
- }
- *hashp = unix_hash_fold(csum_partial(sunaddr, len, 0));
- return len;
+ return 0;
+}
+
+static void unix_mkname_bsd(struct sockaddr_un *sunaddr, int addr_len)
+{
+ /* This may look like an off by one error but it is a bit more
+ * subtle. 108 is the longest valid AF_UNIX path for a binding.
+ * sun_path[108] doesn't as such exist. However in kernel space
+ * we are guaranteed that it is a valid memory location in our
+ * kernel address buffer because syscall functions always pass
+ * a pointer of struct sockaddr_storage which has a bigger buffer
+ * than 108.
+ */
+ ((char *)sunaddr)[addr_len] = 0;
}
static void __unix_remove_socket(struct sock *sk)
@@ -257,32 +306,52 @@ static void __unix_remove_socket(struct sock *sk)
sk_del_node_init(sk);
}
-static void __unix_insert_socket(struct hlist_head *list, struct sock *sk)
+static void __unix_insert_socket(struct net *net, struct sock *sk)
{
- WARN_ON(!sk_unhashed(sk));
- sk_add_node(sk, list);
+ DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk));
+ sk_add_node(sk, &net->unx.table.buckets[sk->sk_hash]);
}
-static void __unix_set_addr(struct sock *sk, struct unix_address *addr,
- unsigned hash)
+static void __unix_set_addr_hash(struct net *net, struct sock *sk,
+ struct unix_address *addr, unsigned int hash)
{
__unix_remove_socket(sk);
smp_store_release(&unix_sk(sk)->addr, addr);
- __unix_insert_socket(&unix_socket_table[hash], sk);
+
+ sk->sk_hash = hash;
+ __unix_insert_socket(net, sk);
}
-static inline void unix_remove_socket(struct sock *sk)
+static void unix_remove_socket(struct net *net, struct sock *sk)
{
- spin_lock(&unix_table_lock);
+ spin_lock(&net->unx.table.locks[sk->sk_hash]);
__unix_remove_socket(sk);
- spin_unlock(&unix_table_lock);
+ spin_unlock(&net->unx.table.locks[sk->sk_hash]);
+}
+
+static void unix_insert_unbound_socket(struct net *net, struct sock *sk)
+{
+ spin_lock(&net->unx.table.locks[sk->sk_hash]);
+ __unix_insert_socket(net, sk);
+ spin_unlock(&net->unx.table.locks[sk->sk_hash]);
+}
+
+static void unix_insert_bsd_socket(struct sock *sk)
+{
+ spin_lock(&bsd_socket_locks[sk->sk_hash]);
+ sk_add_bind_node(sk, &bsd_socket_buckets[sk->sk_hash]);
+ spin_unlock(&bsd_socket_locks[sk->sk_hash]);
}
-static inline void unix_insert_socket(struct hlist_head *list, struct sock *sk)
+static void unix_remove_bsd_socket(struct sock *sk)
{
- spin_lock(&unix_table_lock);
- __unix_insert_socket(list, sk);
- spin_unlock(&unix_table_lock);
+ if (!hlist_unhashed(&sk->sk_bind_node)) {
+ spin_lock(&bsd_socket_locks[sk->sk_hash]);
+ __sk_del_bind_node(sk);
+ spin_unlock(&bsd_socket_locks[sk->sk_hash]);
+
+ sk_node_init(&sk->sk_bind_node);
+ }
}
static struct sock *__unix_find_socket_byname(struct net *net,
@@ -291,12 +360,9 @@ static struct sock *__unix_find_socket_byname(struct net *net,
{
struct sock *s;
- sk_for_each(s, &unix_socket_table[hash]) {
+ sk_for_each(s, &net->unx.table.buckets[hash]) {
struct unix_sock *u = unix_sk(s);
- if (!net_eq(sock_net(s), net))
- continue;
-
if (u->addr->len == len &&
!memcmp(u->addr->name, sunname, len))
return s;
@@ -310,32 +376,31 @@ static inline struct sock *unix_find_socket_byname(struct net *net,
{
struct sock *s;
- spin_lock(&unix_table_lock);
+ spin_lock(&net->unx.table.locks[hash]);
s = __unix_find_socket_byname(net, sunname, len, hash);
if (s)
sock_hold(s);
- spin_unlock(&unix_table_lock);
+ spin_unlock(&net->unx.table.locks[hash]);
return s;
}
static struct sock *unix_find_socket_byinode(struct inode *i)
{
+ unsigned int hash = unix_bsd_hash(i);
struct sock *s;
- spin_lock(&unix_table_lock);
- sk_for_each(s,
- &unix_socket_table[i->i_ino & (UNIX_HASH_SIZE - 1)]) {
+ spin_lock(&bsd_socket_locks[hash]);
+ sk_for_each_bound(s, &bsd_socket_buckets[hash]) {
struct dentry *dentry = unix_sk(s)->path.dentry;
if (dentry && d_backing_inode(dentry) == i) {
sock_hold(s);
- goto found;
+ spin_unlock(&bsd_socket_locks[hash]);
+ return s;
}
}
- s = NULL;
-found:
- spin_unlock(&unix_table_lock);
- return s;
+ spin_unlock(&bsd_socket_locks[hash]);
+ return NULL;
}
/* Support code for asymmetrically connected dgram sockets
@@ -446,7 +511,7 @@ static int unix_dgram_peer_wake_me(struct sock *sk, struct sock *other)
* -ECONNREFUSED. Otherwise, if we haven't queued any skbs
* to other and its full, we will hang waiting for POLLOUT.
*/
- if (unix_recvq_full(other) && !sock_flag(other, SOCK_DEAD))
+ if (unix_recvq_full_lockless(other) && !sock_flag(other, SOCK_DEAD))
return 1;
if (connected)
@@ -504,15 +569,9 @@ static void unix_sock_destructor(struct sock *sk)
skb_queue_purge(&sk->sk_receive_queue);
-#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
- if (u->oob_skb) {
- kfree_skb(u->oob_skb);
- u->oob_skb = NULL;
- }
-#endif
- WARN_ON(refcount_read(&sk->sk_wmem_alloc));
- WARN_ON(!sk_unhashed(sk));
- WARN_ON(sk->sk_socket);
+ DEBUG_NET_WARN_ON_ONCE(refcount_read(&sk->sk_wmem_alloc));
+ DEBUG_NET_WARN_ON_ONCE(!sk_unhashed(sk));
+ DEBUG_NET_WARN_ON_ONCE(sk->sk_socket);
if (!sock_flag(sk, SOCK_DEAD)) {
pr_info("Attempt to release alive unix socket: %p\n", sk);
return;
@@ -522,9 +581,7 @@ static void unix_sock_destructor(struct sock *sk)
unix_release_addr(u->addr);
atomic_long_dec(&unix_nr_socks);
- local_bh_disable();
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
- local_bh_enable();
#ifdef UNIX_REFCNT_DEBUG
pr_debug("UNIX %p is destroyed, %ld are still alive.\n", sk,
atomic_long_read(&unix_nr_socks));
@@ -534,12 +591,13 @@ static void unix_sock_destructor(struct sock *sk)
static void unix_release_sock(struct sock *sk, int embrion)
{
struct unix_sock *u = unix_sk(sk);
- struct path path;
struct sock *skpair;
struct sk_buff *skb;
+ struct path path;
int state;
- unix_remove_socket(sk);
+ unix_remove_socket(sock_net(sk), sk);
+ unix_remove_bsd_socket(sk);
/* Clear state */
unix_state_lock(sk);
@@ -556,6 +614,13 @@ static void unix_release_sock(struct sock *sk, int embrion)
unix_state_unlock(sk);
+#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
+ if (u->oob_skb) {
+ kfree_skb(u->oob_skb);
+ u->oob_skb = NULL;
+ }
+#endif
+
wake_up_interruptible_all(&u->peer_wait);
if (skpair != NULL) {
@@ -699,10 +764,8 @@ static ssize_t unix_stream_splice_read(struct socket *, loff_t *ppos,
unsigned int flags);
static int unix_dgram_sendmsg(struct socket *, struct msghdr *, size_t);
static int unix_dgram_recvmsg(struct socket *, struct msghdr *, size_t, int);
-static int unix_read_sock(struct sock *sk, read_descriptor_t *desc,
- sk_read_actor_t recv_actor);
-static int unix_stream_read_sock(struct sock *sk, read_descriptor_t *desc,
- sk_read_actor_t recv_actor);
+static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
+static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
static int unix_dgram_connect(struct socket *, struct sockaddr *,
int, int);
static int unix_seqpacket_sendmsg(struct socket *, struct msghdr *, size_t);
@@ -723,15 +786,45 @@ static int unix_set_peek_off(struct sock *sk, int val)
}
#ifdef CONFIG_PROC_FS
+static int unix_count_nr_fds(struct sock *sk)
+{
+ struct sk_buff *skb;
+ struct unix_sock *u;
+ int nr_fds = 0;
+
+ spin_lock(&sk->sk_receive_queue.lock);
+ skb = skb_peek(&sk->sk_receive_queue);
+ while (skb) {
+ u = unix_sk(skb->sk);
+ nr_fds += atomic_read(&u->scm_stat.nr_fds);
+ skb = skb_peek_next(skb, &sk->sk_receive_queue);
+ }
+ spin_unlock(&sk->sk_receive_queue.lock);
+
+ return nr_fds;
+}
+
static void unix_show_fdinfo(struct seq_file *m, struct socket *sock)
{
struct sock *sk = sock->sk;
struct unix_sock *u;
+ int nr_fds;
if (sk) {
- u = unix_sk(sock->sk);
- seq_printf(m, "scm_fds: %u\n",
- atomic_read(&u->scm_stat.nr_fds));
+ u = unix_sk(sk);
+ if (sock->type == SOCK_DGRAM) {
+ nr_fds = atomic_read(&u->scm_stat.nr_fds);
+ goto out_print;
+ }
+
+ unix_state_lock(sk);
+ if (sk->sk_state != TCP_LISTEN)
+ nr_fds = atomic_read(&u->scm_stat.nr_fds);
+ else
+ nr_fds = unix_count_nr_fds(sk);
+ unix_state_unlock(sk);
+out_print:
+ seq_printf(m, "scm_fds: %u\n", nr_fds);
}
}
#else
@@ -756,7 +849,7 @@ static const struct proto_ops unix_stream_ops = {
.shutdown = unix_shutdown,
.sendmsg = unix_stream_sendmsg,
.recvmsg = unix_stream_recvmsg,
- .read_sock = unix_stream_read_sock,
+ .read_skb = unix_stream_read_skb,
.mmap = sock_no_mmap,
.sendpage = unix_stream_sendpage,
.splice_read = unix_stream_splice_read,
@@ -781,7 +874,7 @@ static const struct proto_ops unix_dgram_ops = {
.listen = sock_no_listen,
.shutdown = unix_shutdown,
.sendmsg = unix_dgram_sendmsg,
- .read_sock = unix_read_sock,
+ .read_skb = unix_read_skb,
.recvmsg = unix_dgram_recvmsg,
.mmap = sock_no_mmap,
.sendpage = sock_no_sendpage,
@@ -872,6 +965,7 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern,
sock_init_data(sock, sk);
+ sk->sk_hash = unix_unbound_hash(sk);
sk->sk_allocation = GFP_KERNEL_ACCOUNT;
sk->sk_write_space = unix_write_space;
sk->sk_max_ack_backlog = net->unx.sysctl_max_dgram_qlen;
@@ -887,11 +981,9 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern,
init_waitqueue_head(&u->peer_wait);
init_waitqueue_func_entry(&u->peer_wake, unix_dgram_peer_wake_relay);
memset(&u->scm_stat, 0, sizeof(struct scm_stat));
- unix_insert_socket(unix_sockets_unbound(sk), sk);
+ unix_insert_unbound_socket(net, sk);
- local_bh_disable();
- sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
- local_bh_enable();
+ sock_prot_inuse_add(net, sk->sk_prot, 1);
return sk;
@@ -952,15 +1044,91 @@ static int unix_release(struct socket *sock)
return 0;
}
-static int unix_autobind(struct socket *sock)
+static struct sock *unix_find_bsd(struct sockaddr_un *sunaddr, int addr_len,
+ int type)
{
- struct sock *sk = sock->sk;
- struct net *net = sock_net(sk);
+ struct inode *inode;
+ struct path path;
+ struct sock *sk;
+ int err;
+
+ unix_mkname_bsd(sunaddr, addr_len);
+ err = kern_path(sunaddr->sun_path, LOOKUP_FOLLOW, &path);
+ if (err)
+ goto fail;
+
+ err = path_permission(&path, MAY_WRITE);
+ if (err)
+ goto path_put;
+
+ err = -ECONNREFUSED;
+ inode = d_backing_inode(path.dentry);
+ if (!S_ISSOCK(inode->i_mode))
+ goto path_put;
+
+ sk = unix_find_socket_byinode(inode);
+ if (!sk)
+ goto path_put;
+
+ err = -EPROTOTYPE;
+ if (sk->sk_type == type)
+ touch_atime(&path);
+ else
+ goto sock_put;
+
+ path_put(&path);
+
+ return sk;
+
+sock_put:
+ sock_put(sk);
+path_put:
+ path_put(&path);
+fail:
+ return ERR_PTR(err);
+}
+
+static struct sock *unix_find_abstract(struct net *net,
+ struct sockaddr_un *sunaddr,
+ int addr_len, int type)
+{
+ unsigned int hash = unix_abstract_hash(sunaddr, addr_len, type);
+ struct dentry *dentry;
+ struct sock *sk;
+
+ sk = unix_find_socket_byname(net, sunaddr, addr_len, hash);
+ if (!sk)
+ return ERR_PTR(-ECONNREFUSED);
+
+ dentry = unix_sk(sk)->path.dentry;
+ if (dentry)
+ touch_atime(&unix_sk(sk)->path);
+
+ return sk;
+}
+
+static struct sock *unix_find_other(struct net *net,
+ struct sockaddr_un *sunaddr,
+ int addr_len, int type)
+{
+ struct sock *sk;
+
+ if (sunaddr->sun_path[0])
+ sk = unix_find_bsd(sunaddr, addr_len, type);
+ else
+ sk = unix_find_abstract(net, sunaddr, addr_len, type);
+
+ return sk;
+}
+
+static int unix_autobind(struct sock *sk)
+{
+ unsigned int new_hash, old_hash = sk->sk_hash;
struct unix_sock *u = unix_sk(sk);
- static u32 ordernum = 1;
+ struct net *net = sock_net(sk);
struct unix_address *addr;
+ u32 lastnum, ordernum;
int err;
- unsigned int retries = 0;
err = mutex_lock_interruptible(&u->bindlock);
if (err)
@@ -970,141 +1138,104 @@ static int unix_autobind(struct socket *sock)
goto out;
err = -ENOMEM;
- addr = kzalloc(sizeof(*addr) + sizeof(short) + 16, GFP_KERNEL);
+ addr = kzalloc(sizeof(*addr) +
+ offsetof(struct sockaddr_un, sun_path) + 16, GFP_KERNEL);
if (!addr)
goto out;
+ addr->len = offsetof(struct sockaddr_un, sun_path) + 6;
addr->name->sun_family = AF_UNIX;
refcount_set(&addr->refcnt, 1);
+ ordernum = get_random_u32();
+ lastnum = ordernum & 0xFFFFF;
retry:
- addr->len = sprintf(addr->name->sun_path+1, "%05x", ordernum) + 1 + sizeof(short);
- addr->hash = unix_hash_fold(csum_partial(addr->name, addr->len, 0));
- addr->hash ^= sk->sk_type;
+ ordernum = (ordernum + 1) & 0xFFFFF;
+ sprintf(addr->name->sun_path + 1, "%05x", ordernum);
- spin_lock(&unix_table_lock);
- ordernum = (ordernum+1)&0xFFFFF;
+ new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type);
+ unix_table_double_lock(net, old_hash, new_hash);
- if (__unix_find_socket_byname(net, addr->name, addr->len, addr->hash)) {
- spin_unlock(&unix_table_lock);
- /*
- * __unix_find_socket_byname() may take long time if many names
+ if (__unix_find_socket_byname(net, addr->name, addr->len, new_hash)) {
+ unix_table_double_unlock(net, old_hash, new_hash);
+
+ /* __unix_find_socket_byname() may take long time if many names
* are already in use.
*/
cond_resched();
- /* Give up if all names seems to be in use. */
- if (retries++ == 0xFFFFF) {
+
+ if (ordernum == lastnum) {
+ /* Give up if all names seems to be in use. */
err = -ENOSPC;
- kfree(addr);
+ unix_release_addr(addr);
goto out;
}
+
goto retry;
}
- __unix_set_addr(sk, addr, addr->hash);
- spin_unlock(&unix_table_lock);
+ __unix_set_addr_hash(net, sk, addr, new_hash);
+ unix_table_double_unlock(net, old_hash, new_hash);
err = 0;
out: mutex_unlock(&u->bindlock);
return err;
}
-static struct sock *unix_find_other(struct net *net,
- struct sockaddr_un *sunname, int len,
- int type, unsigned int hash, int *error)
-{
- struct sock *u;
- struct path path;
- int err = 0;
-
- if (sunname->sun_path[0]) {
- struct inode *inode;
- err = kern_path(sunname->sun_path, LOOKUP_FOLLOW, &path);
- if (err)
- goto fail;
- inode = d_backing_inode(path.dentry);
- err = path_permission(&path, MAY_WRITE);
- if (err)
- goto put_fail;
-
- err = -ECONNREFUSED;
- if (!S_ISSOCK(inode->i_mode))
- goto put_fail;
- u = unix_find_socket_byinode(inode);
- if (!u)
- goto put_fail;
-
- if (u->sk_type == type)
- touch_atime(&path);
-
- path_put(&path);
-
- err = -EPROTOTYPE;
- if (u->sk_type != type) {
- sock_put(u);
- goto fail;
- }
- } else {
- err = -ECONNREFUSED;
- u = unix_find_socket_byname(net, sunname, len, type ^ hash);
- if (u) {
- struct dentry *dentry;
- dentry = unix_sk(u)->path.dentry;
- if (dentry)
- touch_atime(&unix_sk(u)->path);
- } else
- goto fail;
- }
- return u;
-
-put_fail:
- path_put(&path);
-fail:
- *error = err;
- return NULL;
-}
-
-static int unix_bind_bsd(struct sock *sk, struct unix_address *addr)
+static int unix_bind_bsd(struct sock *sk, struct sockaddr_un *sunaddr,
+ int addr_len)
{
- struct unix_sock *u = unix_sk(sk);
umode_t mode = S_IFSOCK |
(SOCK_INODE(sk->sk_socket)->i_mode & ~current_umask());
+ unsigned int new_hash, old_hash = sk->sk_hash;
+ struct unix_sock *u = unix_sk(sk);
+ struct net *net = sock_net(sk);
struct user_namespace *ns; // barf...
- struct path parent;
+ struct unix_address *addr;
struct dentry *dentry;
- unsigned int hash;
+ struct path parent;
int err;
+ unix_mkname_bsd(sunaddr, addr_len);
+ addr_len = strlen(sunaddr->sun_path) +
+ offsetof(struct sockaddr_un, sun_path) + 1;
+
+ addr = unix_create_addr(sunaddr, addr_len);
+ if (!addr)
+ return -ENOMEM;
+
/*
* Get the parent directory, calculate the hash for last
* component.
*/
dentry = kern_path_create(AT_FDCWD, addr->name->sun_path, &parent, 0);
- if (IS_ERR(dentry))
- return PTR_ERR(dentry);
- ns = mnt_user_ns(parent.mnt);
+ if (IS_ERR(dentry)) {
+ err = PTR_ERR(dentry);
+ goto out;
+ }
/*
* All right, let's create it.
*/
+ ns = mnt_user_ns(parent.mnt);
err = security_path_mknod(&parent, dentry, mode, 0);
if (!err)
err = vfs_mknod(ns, d_inode(parent.dentry), dentry, mode, 0);
if (err)
- goto out;
+ goto out_path;
err = mutex_lock_interruptible(&u->bindlock);
if (err)
goto out_unlink;
if (u->addr)
goto out_unlock;
- addr->hash = UNIX_HASH_SIZE;
- hash = d_backing_inode(dentry)->i_ino & (UNIX_HASH_SIZE - 1);
- spin_lock(&unix_table_lock);
+ new_hash = unix_bsd_hash(d_backing_inode(dentry));
+ unix_table_double_lock(net, old_hash, new_hash);
u->path.mnt = mntget(parent.mnt);
u->path.dentry = dget(dentry);
- __unix_set_addr(sk, addr, hash);
- spin_unlock(&unix_table_lock);
+ __unix_set_addr_hash(net, sk, addr, new_hash);
+ unix_table_double_unlock(net, old_hash, new_hash);
+ unix_insert_bsd_socket(sk);
mutex_unlock(&u->bindlock);
done_path_create(&parent, dentry);
return 0;
@@ -1115,74 +1246,76 @@ out_unlock:
out_unlink:
/* failed after successful mknod? unlink what we'd created... */
vfs_unlink(ns, d_inode(parent.dentry), dentry, NULL);
-out:
+out_path:
done_path_create(&parent, dentry);
- return err;
+out:
+ unix_release_addr(addr);
+ return err == -EEXIST ? -EADDRINUSE : err;
}
-static int unix_bind_abstract(struct sock *sk, struct unix_address *addr)
+static int unix_bind_abstract(struct sock *sk, struct sockaddr_un *sunaddr,
+ int addr_len)
{
+ unsigned int new_hash, old_hash = sk->sk_hash;
struct unix_sock *u = unix_sk(sk);
+ struct net *net = sock_net(sk);
+ struct unix_address *addr;
int err;
+ addr = unix_create_addr(sunaddr, addr_len);
+ if (!addr)
+ return -ENOMEM;
+
err = mutex_lock_interruptible(&u->bindlock);
if (err)
- return err;
+ goto out;
if (u->addr) {
- mutex_unlock(&u->bindlock);
- return -EINVAL;
+ err = -EINVAL;
+ goto out_mutex;
}
- spin_lock(&unix_table_lock);
- if (__unix_find_socket_byname(sock_net(sk), addr->name, addr->len,
- addr->hash)) {
- spin_unlock(&unix_table_lock);
- mutex_unlock(&u->bindlock);
- return -EADDRINUSE;
- }
- __unix_set_addr(sk, addr, addr->hash);
- spin_unlock(&unix_table_lock);
+ new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type);
+ unix_table_double_lock(net, old_hash, new_hash);
+
+ if (__unix_find_socket_byname(net, addr->name, addr->len, new_hash))
+ goto out_spin;
+
+ __unix_set_addr_hash(net, sk, addr, new_hash);
+ unix_table_double_unlock(net, old_hash, new_hash);
mutex_unlock(&u->bindlock);
return 0;
+
+out_spin:
+ unix_table_double_unlock(net, old_hash, new_hash);
+ err = -EADDRINUSE;
+out_mutex:
+ mutex_unlock(&u->bindlock);
+out:
+ unix_release_addr(addr);
+ return err;
}
static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
- struct sock *sk = sock->sk;
struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr;
- char *sun_path = sunaddr->sun_path;
+ struct sock *sk = sock->sk;
int err;
- unsigned int hash;
- struct unix_address *addr;
- if (addr_len < offsetofend(struct sockaddr_un, sun_family) ||
- sunaddr->sun_family != AF_UNIX)
- return -EINVAL;
-
- if (addr_len == sizeof(short))
- return unix_autobind(sock);
+ if (addr_len == offsetof(struct sockaddr_un, sun_path) &&
+ sunaddr->sun_family == AF_UNIX)
+ return unix_autobind(sk);
- err = unix_mkname(sunaddr, addr_len, &hash);
- if (err < 0)
+ err = unix_validate_addr(sunaddr, addr_len);
+ if (err)
return err;
- addr_len = err;
- addr = kmalloc(sizeof(*addr)+addr_len, GFP_KERNEL);
- if (!addr)
- return -ENOMEM;
- memcpy(addr->name, sunaddr, addr_len);
- addr->len = addr_len;
- addr->hash = hash ^ sk->sk_type;
- refcount_set(&addr->refcnt, 1);
-
- if (sun_path[0])
- err = unix_bind_bsd(sk, addr);
+ if (sunaddr->sun_path[0])
+ err = unix_bind_bsd(sk, sunaddr, addr_len);
else
- err = unix_bind_abstract(sk, addr);
- if (err)
- unix_release_addr(addr);
- return err == -EEXIST ? -EADDRINUSE : err;
+ err = unix_bind_abstract(sk, sunaddr, addr_len);
+
+ return err;
}
static void unix_state_double_lock(struct sock *sk1, struct sock *sk2)
@@ -1213,11 +1346,9 @@ static void unix_state_double_unlock(struct sock *sk1, struct sock *sk2)
static int unix_dgram_connect(struct socket *sock, struct sockaddr *addr,
int alen, int flags)
{
- struct sock *sk = sock->sk;
- struct net *net = sock_net(sk);
struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr;
+ struct sock *sk = sock->sk;
struct sock *other;
- unsigned int hash;
int err;
err = -EINVAL;
@@ -1225,19 +1356,23 @@ static int unix_dgram_connect(struct socket *sock, struct sockaddr *addr,
goto out;
if (addr->sa_family != AF_UNSPEC) {
- err = unix_mkname(sunaddr, alen, &hash);
- if (err < 0)
+ err = unix_validate_addr(sunaddr, alen);
+ if (err)
goto out;
- alen = err;
if (test_bit(SOCK_PASSCRED, &sock->flags) &&
- !unix_sk(sk)->addr && (err = unix_autobind(sock)) != 0)
- goto out;
+ !unix_sk(sk)->addr) {
+ err = unix_autobind(sk);
+ if (err)
+ goto out;
+ }
restart:
- other = unix_find_other(net, sunaddr, alen, sock->type, hash, &err);
- if (!other)
+ other = unix_find_other(sock_net(sk), sunaddr, alen, sock->type);
+ if (IS_ERR(other)) {
+ err = PTR_ERR(other);
goto out;
+ }
unix_state_double_lock(sk, other);
@@ -1321,25 +1456,23 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr,
int addr_len, int flags)
{
struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr;
- struct sock *sk = sock->sk;
- struct net *net = sock_net(sk);
+ struct sock *sk = sock->sk, *newsk = NULL, *other = NULL;
struct unix_sock *u = unix_sk(sk), *newu, *otheru;
- struct sock *newsk = NULL;
- struct sock *other = NULL;
+ struct net *net = sock_net(sk);
struct sk_buff *skb = NULL;
- unsigned int hash;
- int st;
- int err;
long timeo;
+ int err;
+ int st;
- err = unix_mkname(sunaddr, addr_len, &hash);
- if (err < 0)
+ err = unix_validate_addr(sunaddr, addr_len);
+ if (err)
goto out;
- addr_len = err;
- if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr &&
- (err = unix_autobind(sock)) != 0)
- goto out;
+ if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr) {
+ err = unix_autobind(sk);
+ if (err)
+ goto out;
+ }
timeo = sock_sndtimeo(sk, flags & O_NONBLOCK);
@@ -1349,7 +1482,7 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr,
*/
/* create new sock for complete connection */
- newsk = unix_create1(sock_net(sk), NULL, 0, sock->type);
+ newsk = unix_create1(net, NULL, 0, sock->type);
if (IS_ERR(newsk)) {
err = PTR_ERR(newsk);
newsk = NULL;
@@ -1365,9 +1498,12 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr,
restart:
/* Find listening sock. */
- other = unix_find_other(net, sunaddr, addr_len, sk->sk_type, hash, &err);
- if (!other)
+ other = unix_find_other(net, sunaddr, addr_len, sk->sk_type);
+ if (IS_ERR(other)) {
+ err = PTR_ERR(other);
+ other = NULL;
goto out;
+ }
/* Latch state of peer */
unix_state_lock(other);
@@ -1455,9 +1591,9 @@ restart:
*
* The contents of *(otheru->addr) and otheru->path
* are seen fully set up here, since we have found
- * otheru in hash under unix_table_lock. Insertion
- * into the hash chain we'd found it in had been done
- * in an earlier critical area protected by unix_table_lock,
+ * otheru in hash under its lock. Insertion into the
+ * hash chain we'd found it in had been done in an
+ * earlier critical area protected by the chain's lock,
* the same one where we'd set *(otheru->addr) contents,
* as well as otheru->path and otheru->addr itself.
*
@@ -1557,7 +1693,8 @@ static int unix_accept(struct socket *sock, struct socket *newsock, int flags,
* so that no locks are necessary.
*/
- skb = skb_recv_datagram(sk, 0, flags&O_NONBLOCK, &err);
+ skb = skb_recv_datagram(sk, (flags & O_NONBLOCK) ? MSG_DONTWAIT : 0,
+ &err);
if (!skb) {
/* This means receive shutdown. */
if (err == 0)
@@ -1604,7 +1741,7 @@ static int unix_getname(struct socket *sock, struct sockaddr *uaddr, int peer)
if (!addr) {
sunaddr->sun_family = AF_UNIX;
sunaddr->sun_path[0] = 0;
- err = sizeof(short);
+ err = offsetof(struct sockaddr_un, sun_path);
} else {
err = addr->len;
memcpy(sunaddr, addr->name, addr->len);
@@ -1722,11 +1859,9 @@ static int maybe_init_creds(struct scm_cookie *scm,
static bool unix_skb_scm_eq(struct sk_buff *skb,
struct scm_cookie *scm)
{
- const struct unix_skb_parms *u = &UNIXCB(skb);
-
- return u->pid == scm->pid &&
- uid_eq(u->uid, scm->creds.uid) &&
- gid_eq(u->gid, scm->creds.gid) &&
+ return UNIXCB(skb).pid == scm->pid &&
+ uid_eq(UNIXCB(skb).uid, scm->creds.uid) &&
+ gid_eq(UNIXCB(skb).gid, scm->creds.gid) &&
unix_secdata_eq(scm, skb);
}
@@ -1755,19 +1890,15 @@ static void scm_stat_del(struct sock *sk, struct sk_buff *skb)
static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg,
size_t len)
{
- struct sock *sk = sock->sk;
- struct net *net = sock_net(sk);
- struct unix_sock *u = unix_sk(sk);
DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, msg->msg_name);
- struct sock *other = NULL;
- int namelen = 0; /* fake GCC */
- int err;
- unsigned int hash;
- struct sk_buff *skb;
- long timeo;
+ struct sock *sk = sock->sk, *other = NULL;
+ struct unix_sock *u = unix_sk(sk);
struct scm_cookie scm;
+ struct sk_buff *skb;
int data_len = 0;
int sk_locked;
+ long timeo;
+ int err;
wait_for_unix_gc();
err = scm_send(sock, msg, &scm, false);
@@ -1779,10 +1910,9 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg,
goto out;
if (msg->msg_namelen) {
- err = unix_mkname(sunaddr, msg->msg_namelen, &hash);
- if (err < 0)
+ err = unix_validate_addr(sunaddr, msg->msg_namelen);
+ if (err)
goto out;
- namelen = err;
} else {
sunaddr = NULL;
err = -ENOTCONN;
@@ -1791,9 +1921,11 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg,
goto out;
}
- if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr
- && (err = unix_autobind(sock)) != 0)
- goto out;
+ if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr) {
+ err = unix_autobind(sk);
+ if (err)
+ goto out;
+ }
err = -EMSGSIZE;
if (len > sk->sk_sndbuf - 32)
@@ -1833,10 +1965,13 @@ restart:
if (sunaddr == NULL)
goto out_free;
- other = unix_find_other(net, sunaddr, namelen, sk->sk_type,
- hash, &err);
- if (other == NULL)
+ other = unix_find_other(sock_net(sk), sunaddr, msg->msg_namelen,
+ sk->sk_type);
+ if (IS_ERR(other)) {
+ err = PTR_ERR(other);
+ other = NULL;
goto out_free;
+ }
}
if (sk_filter(other, skb) < 0) {
@@ -1961,7 +2096,7 @@ out:
*/
#define UNIX_SKB_FRAGS_SZ (PAGE_SIZE << get_order(32768))
-#if (IS_ENABLED(CONFIG_AF_UNIX_OOB))
+#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
static int queue_oob(struct socket *sock, struct msghdr *msg, struct sock *other)
{
struct unix_sock *ousk = unix_sk(other);
@@ -1996,7 +2131,7 @@ static int queue_oob(struct socket *sock, struct msghdr *msg, struct sock *other
if (ousk->oob_skb)
consume_skb(ousk->oob_skb);
- ousk->oob_skb = skb;
+ WRITE_ONCE(ousk->oob_skb, skb);
scm_stat_add(other, skb);
skb_queue_tail(&other->sk_receive_queue, skb);
@@ -2027,7 +2162,7 @@ static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg,
err = -EOPNOTSUPP;
if (msg->msg_flags & MSG_OOB) {
-#if (IS_ENABLED(CONFIG_AF_UNIX_OOB))
+#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
if (len)
len--;
else
@@ -2098,7 +2233,7 @@ static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg,
sent += size;
}
-#if (IS_ENABLED(CONFIG_AF_UNIX_OOB))
+#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
if (msg->msg_flags & MSG_OOB) {
err = queue_oob(sock, msg, other);
if (err)
@@ -2395,42 +2530,25 @@ static int unix_dgram_recvmsg(struct socket *sock, struct msghdr *msg, size_t si
const struct proto *prot = READ_ONCE(sk->sk_prot);
if (prot != &unix_dgram_proto)
- return prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
- flags & ~MSG_DONTWAIT, NULL);
+ return prot->recvmsg(sk, msg, size, flags, NULL);
#endif
return __unix_dgram_recvmsg(sk, msg, size, flags);
}
-static int unix_read_sock(struct sock *sk, read_descriptor_t *desc,
- sk_read_actor_t recv_actor)
+static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
{
- int copied = 0;
-
- while (1) {
- struct unix_sock *u = unix_sk(sk);
- struct sk_buff *skb;
- int used, err;
-
- mutex_lock(&u->iolock);
- skb = skb_recv_datagram(sk, 0, 1, &err);
- mutex_unlock(&u->iolock);
- if (!skb)
- return err;
+ struct unix_sock *u = unix_sk(sk);
+ struct sk_buff *skb;
+ int err, copied;
- used = recv_actor(desc, skb, 0, skb->len);
- if (used <= 0) {
- if (!copied)
- copied = used;
- kfree_skb(skb);
- break;
- } else if (used <= skb->len) {
- copied += used;
- }
+ mutex_lock(&u->iolock);
+ skb = skb_recv_datagram(sk, MSG_DONTWAIT, &err);
+ mutex_unlock(&u->iolock);
+ if (!skb)
+ return err;
- kfree_skb(skb);
- if (!desc->count)
- break;
- }
+ copied = recv_actor(sk, skb);
+ kfree_skb(skb);
return copied;
}
@@ -2442,13 +2560,14 @@ static long unix_stream_data_wait(struct sock *sk, long timeo,
struct sk_buff *last, unsigned int last_len,
bool freezable)
{
+ unsigned int state = TASK_INTERRUPTIBLE | freezable * TASK_FREEZABLE;
struct sk_buff *tail;
DEFINE_WAIT(wait);
unix_state_lock(sk);
for (;;) {
- prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
+ prepare_to_wait(sk_sleep(sk), &wait, state);
tail = skb_peek_tail(&sk->sk_receive_queue);
if (tail != last ||
@@ -2461,10 +2580,7 @@ static long unix_stream_data_wait(struct sock *sk, long timeo,
sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
unix_state_unlock(sk);
- if (freezable)
- timeo = freezable_schedule_timeout(timeo);
- else
- timeo = schedule_timeout(timeo);
+ timeo = schedule_timeout(timeo);
unix_state_lock(sk);
if (sock_flag(sk, SOCK_DEAD))
@@ -2514,9 +2630,8 @@ static int unix_stream_recv_urg(struct unix_stream_read_state *state)
oob_skb = u->oob_skb;
- if (!(state->flags & MSG_PEEK)) {
- u->oob_skb = NULL;
- }
+ if (!(state->flags & MSG_PEEK))
+ WRITE_ONCE(u->oob_skb, NULL);
unix_state_unlock(sk);
@@ -2551,7 +2666,7 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk,
skb = NULL;
} else if (sock_flag(sk, SOCK_URGINLINE)) {
if (!(flags & MSG_PEEK)) {
- u->oob_skb = NULL;
+ WRITE_ONCE(u->oob_skb, NULL);
consume_skb(skb);
}
} else if (!(flags & MSG_PEEK)) {
@@ -2565,13 +2680,12 @@ static struct sk_buff *manage_oob(struct sk_buff *skb, struct sock *sk,
}
#endif
-static int unix_stream_read_sock(struct sock *sk, read_descriptor_t *desc,
- sk_read_actor_t recv_actor)
+static int unix_stream_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
{
if (unlikely(sk->sk_state != TCP_ESTABLISHED))
return -ENOTCONN;
- return unix_read_sock(sk, desc, recv_actor);
+ return unix_read_skb(sk, recv_actor);
}
static int unix_stream_read_generic(struct unix_stream_read_state *state,
@@ -2829,8 +2943,7 @@ static int unix_stream_recvmsg(struct socket *sock, struct msghdr *msg,
const struct proto *prot = READ_ONCE(sk->sk_prot);
if (prot != &unix_stream_proto)
- return prot->recvmsg(sk, msg, size, flags & MSG_DONTWAIT,
- flags & ~MSG_DONTWAIT, NULL);
+ return prot->recvmsg(sk, msg, size, flags, NULL);
#endif
return unix_stream_read_generic(&state, true);
}
@@ -3006,11 +3119,10 @@ static int unix_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
case SIOCATMARK:
{
struct sk_buff *skb;
- struct unix_sock *u = unix_sk(sk);
int answ = 0;
skb = skb_peek(&sk->sk_receive_queue);
- if (skb && skb == u->oob_skb)
+ if (skb && skb == READ_ONCE(unix_sk(sk)->oob_skb))
answ = 1;
err = put_user(answ, (int __user *)arg);
}
@@ -3051,6 +3163,10 @@ static __poll_t unix_poll(struct file *file, struct socket *sock, poll_table *wa
mask |= EPOLLIN | EPOLLRDNORM;
if (sk_is_readable(sk))
mask |= EPOLLIN | EPOLLRDNORM;
+#if IS_ENABLED(CONFIG_AF_UNIX_OOB)
+ if (READ_ONCE(unix_sk(sk)->oob_skb))
+ mask |= EPOLLPRI;
+#endif
/* Connection-based need to check for termination and startup */
if ((sk->sk_type == SOCK_STREAM || sk->sk_type == SOCK_SEQPACKET) &&
@@ -3132,19 +3248,18 @@ static __poll_t unix_dgram_poll(struct file *file, struct socket *sock,
#define BUCKET_SPACE (BITS_PER_LONG - (UNIX_HASH_BITS + 1) - 1)
#define get_bucket(x) ((x) >> BUCKET_SPACE)
-#define get_offset(x) ((x) & ((1L << BUCKET_SPACE) - 1))
+#define get_offset(x) ((x) & ((1UL << BUCKET_SPACE) - 1))
#define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o))
static struct sock *unix_from_bucket(struct seq_file *seq, loff_t *pos)
{
unsigned long offset = get_offset(*pos);
unsigned long bucket = get_bucket(*pos);
- struct sock *sk;
unsigned long count = 0;
+ struct sock *sk;
- for (sk = sk_head(&unix_socket_table[bucket]); sk; sk = sk_next(sk)) {
- if (sock_net(sk) != seq_file_net(seq))
- continue;
+ for (sk = sk_head(&seq_file_net(seq)->unx.table.buckets[bucket]);
+ sk; sk = sk_next(sk)) {
if (++count == offset)
break;
}
@@ -3152,57 +3267,68 @@ static struct sock *unix_from_bucket(struct seq_file *seq, loff_t *pos)
return sk;
}
-static struct sock *unix_next_socket(struct seq_file *seq,
- struct sock *sk,
- loff_t *pos)
+static struct sock *unix_get_first(struct seq_file *seq, loff_t *pos)
{
- unsigned long bucket;
+ unsigned long bucket = get_bucket(*pos);
+ struct net *net = seq_file_net(seq);
+ struct sock *sk;
- while (sk > (struct sock *)SEQ_START_TOKEN) {
- sk = sk_next(sk);
- if (!sk)
- goto next_bucket;
- if (sock_net(sk) == seq_file_net(seq))
- return sk;
- }
+ while (bucket < UNIX_HASH_SIZE) {
+ spin_lock(&net->unx.table.locks[bucket]);
- do {
sk = unix_from_bucket(seq, pos);
if (sk)
return sk;
-next_bucket:
- bucket = get_bucket(*pos) + 1;
- *pos = set_bucket_offset(bucket, 1);
- } while (bucket < ARRAY_SIZE(unix_socket_table));
+ spin_unlock(&net->unx.table.locks[bucket]);
+
+ *pos = set_bucket_offset(++bucket, 1);
+ }
return NULL;
}
-static void *unix_seq_start(struct seq_file *seq, loff_t *pos)
- __acquires(unix_table_lock)
+static struct sock *unix_get_next(struct seq_file *seq, struct sock *sk,
+ loff_t *pos)
{
- spin_lock(&unix_table_lock);
+ unsigned long bucket = get_bucket(*pos);
+
+ sk = sk_next(sk);
+ if (sk)
+ return sk;
+
+ spin_unlock(&seq_file_net(seq)->unx.table.locks[bucket]);
+
+ *pos = set_bucket_offset(++bucket, 1);
+
+ return unix_get_first(seq, pos);
+}
+
+static void *unix_seq_start(struct seq_file *seq, loff_t *pos)
+{
if (!*pos)
return SEQ_START_TOKEN;
- if (get_bucket(*pos) >= ARRAY_SIZE(unix_socket_table))
- return NULL;
-
- return unix_next_socket(seq, NULL, pos);
+ return unix_get_first(seq, pos);
}
static void *unix_seq_next(struct seq_file *seq, void *v, loff_t *pos)
{
++*pos;
- return unix_next_socket(seq, v, pos);
+
+ if (v == SEQ_START_TOKEN)
+ return unix_get_first(seq, pos);
+
+ return unix_get_next(seq, v, pos);
}
static void unix_seq_stop(struct seq_file *seq, void *v)
- __releases(unix_table_lock)
{
- spin_unlock(&unix_table_lock);
+ struct sock *sk = v;
+
+ if (sk)
+ spin_unlock(&seq_file_net(seq)->unx.table.locks[sk->sk_hash]);
}
static int unix_seq_show(struct seq_file *seq, void *v)
@@ -3227,15 +3353,16 @@ static int unix_seq_show(struct seq_file *seq, void *v)
(s->sk_state == TCP_ESTABLISHED ? SS_CONNECTING : SS_DISCONNECTING),
sock_i_ino(s));
- if (u->addr) { // under unix_table_lock here
+ if (u->addr) { // under a hash table lock here
int i, len;
seq_putc(seq, ' ');
i = 0;
- len = u->addr->len - sizeof(short);
- if (!UNIX_ABSTRACT(s))
+ len = u->addr->len -
+ offsetof(struct sockaddr_un, sun_path);
+ if (u->addr->name->sun_path[0]) {
len--;
- else {
+ } else {
seq_putc(seq, '@');
i++;
}
@@ -3258,6 +3385,15 @@ static const struct seq_operations unix_seq_ops = {
};
#if IS_BUILTIN(CONFIG_UNIX) && defined(CONFIG_BPF_SYSCALL)
+struct bpf_unix_iter_state {
+ struct seq_net_private p;
+ unsigned int cur_sk;
+ unsigned int end_sk;
+ unsigned int max_sk;
+ struct sock **batch;
+ bool st_bucket_done;
+};
+
struct bpf_iter__unix {
__bpf_md_ptr(struct bpf_iter_meta *, meta);
__bpf_md_ptr(struct unix_sock *, unix_sk);
@@ -3276,24 +3412,153 @@ static int unix_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
return bpf_iter_run_prog(prog, &ctx);
}
+static int bpf_iter_unix_hold_batch(struct seq_file *seq, struct sock *start_sk)
+
+{
+ struct bpf_unix_iter_state *iter = seq->private;
+ unsigned int expected = 1;
+ struct sock *sk;
+
+ sock_hold(start_sk);
+ iter->batch[iter->end_sk++] = start_sk;
+
+ for (sk = sk_next(start_sk); sk; sk = sk_next(sk)) {
+ if (iter->end_sk < iter->max_sk) {
+ sock_hold(sk);
+ iter->batch[iter->end_sk++] = sk;
+ }
+
+ expected++;
+ }
+
+ spin_unlock(&seq_file_net(seq)->unx.table.locks[start_sk->sk_hash]);
+
+ return expected;
+}
+
+static void bpf_iter_unix_put_batch(struct bpf_unix_iter_state *iter)
+{
+ while (iter->cur_sk < iter->end_sk)
+ sock_put(iter->batch[iter->cur_sk++]);
+}
+
+static int bpf_iter_unix_realloc_batch(struct bpf_unix_iter_state *iter,
+ unsigned int new_batch_sz)
+{
+ struct sock **new_batch;
+
+ new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
+ GFP_USER | __GFP_NOWARN);
+ if (!new_batch)
+ return -ENOMEM;
+
+ bpf_iter_unix_put_batch(iter);
+ kvfree(iter->batch);
+ iter->batch = new_batch;
+ iter->max_sk = new_batch_sz;
+
+ return 0;
+}
+
+static struct sock *bpf_iter_unix_batch(struct seq_file *seq,
+ loff_t *pos)
+{
+ struct bpf_unix_iter_state *iter = seq->private;
+ unsigned int expected;
+ bool resized = false;
+ struct sock *sk;
+
+ if (iter->st_bucket_done)
+ *pos = set_bucket_offset(get_bucket(*pos) + 1, 1);
+
+again:
+ /* Get a new batch */
+ iter->cur_sk = 0;
+ iter->end_sk = 0;
+
+ sk = unix_get_first(seq, pos);
+ if (!sk)
+ return NULL; /* Done */
+
+ expected = bpf_iter_unix_hold_batch(seq, sk);
+
+ if (iter->end_sk == expected) {
+ iter->st_bucket_done = true;
+ return sk;
+ }
+
+ if (!resized && !bpf_iter_unix_realloc_batch(iter, expected * 3 / 2)) {
+ resized = true;
+ goto again;
+ }
+
+ return sk;
+}
+
+static void *bpf_iter_unix_seq_start(struct seq_file *seq, loff_t *pos)
+{
+ if (!*pos)
+ return SEQ_START_TOKEN;
+
+ /* bpf iter does not support lseek, so it always
+ * continue from where it was stop()-ped.
+ */
+ return bpf_iter_unix_batch(seq, pos);
+}
+
+static void *bpf_iter_unix_seq_next(struct seq_file *seq, void *v, loff_t *pos)
+{
+ struct bpf_unix_iter_state *iter = seq->private;
+ struct sock *sk;
+
+ /* Whenever seq_next() is called, the iter->cur_sk is
+ * done with seq_show(), so advance to the next sk in
+ * the batch.
+ */
+ if (iter->cur_sk < iter->end_sk)
+ sock_put(iter->batch[iter->cur_sk++]);
+
+ ++*pos;
+
+ if (iter->cur_sk < iter->end_sk)
+ sk = iter->batch[iter->cur_sk];
+ else
+ sk = bpf_iter_unix_batch(seq, pos);
+
+ return sk;
+}
+
static int bpf_iter_unix_seq_show(struct seq_file *seq, void *v)
{
struct bpf_iter_meta meta;
struct bpf_prog *prog;
struct sock *sk = v;
uid_t uid;
+ bool slow;
+ int ret;
if (v == SEQ_START_TOKEN)
return 0;
+ slow = lock_sock_fast(sk);
+
+ if (unlikely(sk_unhashed(sk))) {
+ ret = SEQ_SKIP;
+ goto unlock;
+ }
+
uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
meta.seq = seq;
prog = bpf_iter_get_info(&meta, false);
- return unix_prog_seq_show(prog, &meta, v, uid);
+ ret = unix_prog_seq_show(prog, &meta, v, uid);
+unlock:
+ unlock_sock_fast(sk, slow);
+ return ret;
}
static void bpf_iter_unix_seq_stop(struct seq_file *seq, void *v)
{
+ struct bpf_unix_iter_state *iter = seq->private;
struct bpf_iter_meta meta;
struct bpf_prog *prog;
@@ -3304,12 +3569,13 @@ static void bpf_iter_unix_seq_stop(struct seq_file *seq, void *v)
(void)unix_prog_seq_show(prog, &meta, v, 0);
}
- unix_seq_stop(seq, v);
+ if (iter->cur_sk < iter->end_sk)
+ bpf_iter_unix_put_batch(iter);
}
static const struct seq_operations bpf_iter_unix_seq_ops = {
- .start = unix_seq_start,
- .next = unix_seq_next,
+ .start = bpf_iter_unix_seq_start,
+ .next = bpf_iter_unix_seq_next,
.stop = bpf_iter_unix_seq_stop,
.show = bpf_iter_unix_seq_show,
};
@@ -3325,7 +3591,7 @@ static const struct net_proto_family unix_family_ops = {
static int __net_init unix_net_init(struct net *net)
{
- int error = -ENOMEM;
+ int i;
net->unx.sysctl_max_dgram_qlen = 10;
if (unix_sysctl_register(net))
@@ -3333,18 +3599,44 @@ static int __net_init unix_net_init(struct net *net)
#ifdef CONFIG_PROC_FS
if (!proc_create_net("unix", 0, net->proc_net, &unix_seq_ops,
- sizeof(struct seq_net_private))) {
- unix_sysctl_unregister(net);
- goto out;
+ sizeof(struct seq_net_private)))
+ goto err_sysctl;
+#endif
+
+ net->unx.table.locks = kvmalloc_array(UNIX_HASH_SIZE,
+ sizeof(spinlock_t), GFP_KERNEL);
+ if (!net->unx.table.locks)
+ goto err_proc;
+
+ net->unx.table.buckets = kvmalloc_array(UNIX_HASH_SIZE,
+ sizeof(struct hlist_head),
+ GFP_KERNEL);
+ if (!net->unx.table.buckets)
+ goto free_locks;
+
+ for (i = 0; i < UNIX_HASH_SIZE; i++) {
+ spin_lock_init(&net->unx.table.locks[i]);
+ INIT_HLIST_HEAD(&net->unx.table.buckets[i]);
}
+
+ return 0;
+
+free_locks:
+ kvfree(net->unx.table.locks);
+err_proc:
+#ifdef CONFIG_PROC_FS
+ remove_proc_entry("unix", net->proc_net);
+err_sysctl:
#endif
- error = 0;
+ unix_sysctl_unregister(net);
out:
- return error;
+ return -ENOMEM;
}
static void __net_exit unix_net_exit(struct net *net)
{
+ kvfree(net->unx.table.buckets);
+ kvfree(net->unx.table.locks);
unix_sysctl_unregister(net);
remove_proc_entry("unix", net->proc_net);
}
@@ -3358,13 +3650,55 @@ static struct pernet_operations unix_net_ops = {
DEFINE_BPF_ITER_FUNC(unix, struct bpf_iter_meta *meta,
struct unix_sock *unix_sk, uid_t uid)
+#define INIT_BATCH_SZ 16
+
+static int bpf_iter_init_unix(void *priv_data, struct bpf_iter_aux_info *aux)
+{
+ struct bpf_unix_iter_state *iter = priv_data;
+ int err;
+
+ err = bpf_iter_init_seq_net(priv_data, aux);
+ if (err)
+ return err;
+
+ err = bpf_iter_unix_realloc_batch(iter, INIT_BATCH_SZ);
+ if (err) {
+ bpf_iter_fini_seq_net(priv_data);
+ return err;
+ }
+
+ return 0;
+}
+
+static void bpf_iter_fini_unix(void *priv_data)
+{
+ struct bpf_unix_iter_state *iter = priv_data;
+
+ bpf_iter_fini_seq_net(priv_data);
+ kvfree(iter->batch);
+}
+
static const struct bpf_iter_seq_info unix_seq_info = {
.seq_ops = &bpf_iter_unix_seq_ops,
- .init_seq_private = bpf_iter_init_seq_net,
- .fini_seq_private = bpf_iter_fini_seq_net,
- .seq_priv_size = sizeof(struct seq_net_private),
+ .init_seq_private = bpf_iter_init_unix,
+ .fini_seq_private = bpf_iter_fini_unix,
+ .seq_priv_size = sizeof(struct bpf_unix_iter_state),
};
+static const struct bpf_func_proto *
+bpf_iter_unix_get_func_proto(enum bpf_func_id func_id,
+ const struct bpf_prog *prog)
+{
+ switch (func_id) {
+ case BPF_FUNC_setsockopt:
+ return &bpf_sk_setsockopt_proto;
+ case BPF_FUNC_getsockopt:
+ return &bpf_sk_getsockopt_proto;
+ default:
+ return NULL;
+ }
+}
+
static struct bpf_iter_reg unix_reg_info = {
.target = "unix",
.ctx_arg_info_size = 1,
@@ -3372,6 +3706,7 @@ static struct bpf_iter_reg unix_reg_info = {
{ offsetof(struct bpf_iter__unix, unix_sk),
PTR_TO_BTF_ID_OR_NULL },
},
+ .get_func_proto = bpf_iter_unix_get_func_proto,
.seq_info = &unix_seq_info,
};
@@ -3385,10 +3720,15 @@ static void __init bpf_iter_register(void)
static int __init af_unix_init(void)
{
- int rc = -1;
+ int i, rc = -1;
BUILD_BUG_ON(sizeof(struct unix_skb_parms) > sizeof_field(struct sk_buff, cb));
+ for (i = 0; i < UNIX_HASH_SIZE / 2; i++) {
+ spin_lock_init(&bsd_socket_locks[i]);
+ INIT_HLIST_HEAD(&bsd_socket_buckets[i]);
+ }
+
rc = proto_register(&unix_dgram_proto, 1);
if (rc != 0) {
pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__);
Copyright © 1996 – 2023 Jason A. Donenfeld. All Rights Reverse Engineered.