diff options
Diffstat (limited to '')
| -rw-r--r-- | ipc/msg.c | 39 | ||||
| -rw-r--r-- | ipc/namespace.c | 20 | ||||
| -rw-r--r-- | ipc/sem.c | 41 | ||||
| -rw-r--r-- | ipc/shm.c | 123 | ||||
| -rw-r--r-- | ipc/util.c | 215 | ||||
| -rw-r--r-- | ipc/util.h | 33 |
6 files changed, 236 insertions, 235 deletions
diff --git a/ipc/msg.c b/ipc/msg.c index 3b6545302598..883642cf2b27 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -38,6 +38,7 @@ #include <linux/rwsem.h> #include <linux/nsproxy.h> #include <linux/ipc_namespace.h> +#include <linux/rhashtable.h> #include <asm/current.h> #include <linux/uaccess.h> @@ -162,7 +163,7 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params) /* ipc_addid() locks msq upon success. */ retval = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni); if (retval < 0) { - call_rcu(&msq->q_perm.rcu, msg_rcu_free); + ipc_rcu_putref(&msq->q_perm, msg_rcu_free); return retval; } @@ -385,7 +386,7 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd, down_write(&msg_ids(ns).rwsem); rcu_read_lock(); - ipcp = ipcctl_pre_down_nolock(ns, &msg_ids(ns), msqid, cmd, + ipcp = ipcctl_obtain_check(ns, &msg_ids(ns), msqid, cmd, &msqid64->msg_perm, msqid64->msg_qbytes); if (IS_ERR(ipcp)) { err = PTR_ERR(ipcp); @@ -455,7 +456,7 @@ static int msgctl_info(struct ipc_namespace *ns, int msqid, int cmd, struct msginfo *msginfo) { int err; - int max_id; + int max_idx; /* * We must not return kernel stack data. @@ -482,16 +483,15 @@ static int msgctl_info(struct ipc_namespace *ns, int msqid, msginfo->msgpool = MSGPOOL; msginfo->msgtql = MSGTQL; } - max_id = ipc_get_maxid(&msg_ids(ns)); + max_idx = ipc_get_maxidx(&msg_ids(ns)); up_read(&msg_ids(ns).rwsem); - return (max_id < 0) ? 0 : max_id; + return (max_idx < 0) ? 0 : max_idx; } static int msgctl_stat(struct ipc_namespace *ns, int msqid, int cmd, struct msqid64_ds *p) { struct msg_queue *msq; - int id = 0; int err; memset(p, 0, sizeof(*p)); @@ -503,7 +503,6 @@ static int msgctl_stat(struct ipc_namespace *ns, int msqid, err = PTR_ERR(msq); goto out_unlock; } - id = msq->q_perm.id; } else { /* IPC_STAT */ msq = msq_obtain_object_check(ns, msqid); if (IS_ERR(msq)) { @@ -548,10 +547,21 @@ static int msgctl_stat(struct ipc_namespace *ns, int msqid, p->msg_lspid = pid_vnr(msq->q_lspid); p->msg_lrpid = pid_vnr(msq->q_lrpid); - ipc_unlock_object(&msq->q_perm); - rcu_read_unlock(); - return id; + if (cmd == IPC_STAT) { + /* + * As defined in SUS: + * Return 0 on success + */ + err = 0; + } else { + /* + * MSG_STAT and MSG_STAT_ANY (both Linux specific) + * Return the full id, including the sequence number + */ + err = msq->q_perm.id; + } + ipc_unlock_object(&msq->q_perm); out_unlock: rcu_read_unlock(); return err; @@ -1228,7 +1238,7 @@ COMPAT_SYSCALL_DEFINE5(msgrcv, int, msqid, compat_uptr_t, msgp, } #endif -int msg_init_ns(struct ipc_namespace *ns) +void msg_init_ns(struct ipc_namespace *ns) { ns->msg_ctlmax = MSGMAX; ns->msg_ctlmnb = MSGMNB; @@ -1236,7 +1246,7 @@ int msg_init_ns(struct ipc_namespace *ns) atomic_set(&ns->msg_bytes, 0); atomic_set(&ns->msg_hdrs, 0); - return ipc_init_ids(&ns->ids[IPC_MSG_IDS]); + ipc_init_ids(&ns->ids[IPC_MSG_IDS]); } #ifdef CONFIG_IPC_NS @@ -1277,12 +1287,11 @@ static int sysvipc_msg_proc_show(struct seq_file *s, void *it) } #endif -int __init msg_init(void) +void __init msg_init(void) { - const int err = msg_init_ns(&init_ipc_ns); + msg_init_ns(&init_ipc_ns); ipc_init_proc_interface("sysvipc/msg", " key msqid perms cbytes qnum lspid lrpid uid gid cuid cgid stime rtime ctime\n", IPC_MSG_IDS, sysvipc_msg_proc_show); - return err; } diff --git a/ipc/namespace.c b/ipc/namespace.c index f59a89966f92..21607791d62c 100644 --- a/ipc/namespace.c +++ b/ipc/namespace.c @@ -55,28 +55,16 @@ static struct ipc_namespace *create_ipc_ns(struct user_namespace *user_ns, ns->user_ns = get_user_ns(user_ns); ns->ucounts = ucounts; - err = sem_init_ns(ns); + err = mq_init_ns(ns); if (err) goto fail_put; - err = msg_init_ns(ns); - if (err) - goto fail_destroy_sem; - err = shm_init_ns(ns); - if (err) - goto fail_destroy_msg; - err = mq_init_ns(ns); - if (err) - goto fail_destroy_shm; + sem_init_ns(ns); + msg_init_ns(ns); + shm_init_ns(ns); return ns; -fail_destroy_shm: - shm_exit_ns(ns); -fail_destroy_msg: - msg_exit_ns(ns); -fail_destroy_sem: - sem_exit_ns(ns); fail_put: put_user_ns(ns->user_ns); ns_free_inum(&ns->ns); diff --git a/ipc/sem.c b/ipc/sem.c index 5af1943ad782..26f8e37fcdcb 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -86,6 +86,7 @@ #include <linux/ipc_namespace.h> #include <linux/sched/wake_q.h> #include <linux/nospec.h> +#include <linux/rhashtable.h> #include <linux/uaccess.h> #include "util.h" @@ -220,14 +221,14 @@ static int sysvipc_sem_proc_show(struct seq_file *s, void *it); #define sc_semopm sem_ctls[2] #define sc_semmni sem_ctls[3] -int sem_init_ns(struct ipc_namespace *ns) +void sem_init_ns(struct ipc_namespace *ns) { ns->sc_semmsl = SEMMSL; ns->sc_semmns = SEMMNS; ns->sc_semopm = SEMOPM; ns->sc_semmni = SEMMNI; ns->used_sems = 0; - return ipc_init_ids(&ns->ids[IPC_SEM_IDS]); + ipc_init_ids(&ns->ids[IPC_SEM_IDS]); } #ifdef CONFIG_IPC_NS @@ -239,14 +240,12 @@ void sem_exit_ns(struct ipc_namespace *ns) } #endif -int __init sem_init(void) +void __init sem_init(void) { - const int err = sem_init_ns(&init_ipc_ns); - + sem_init_ns(&init_ipc_ns); ipc_init_proc_interface("sysvipc/sem", " key semid perms nsems uid gid cuid cgid otime ctime\n", IPC_SEM_IDS, sysvipc_sem_proc_show); - return err; } /** @@ -556,7 +555,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params) /* ipc_addid() locks sma upon success. */ retval = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni); if (retval < 0) { - call_rcu(&sma->sem_perm.rcu, sem_rcu_free); + ipc_rcu_putref(&sma->sem_perm, sem_rcu_free); return retval; } ns->used_sems += nsems; @@ -1222,7 +1221,6 @@ static int semctl_stat(struct ipc_namespace *ns, int semid, { struct sem_array *sma; time64_t semotime; - int id = 0; int err; memset(semid64, 0, sizeof(*semid64)); @@ -1234,7 +1232,6 @@ static int semctl_stat(struct ipc_namespace *ns, int semid, err = PTR_ERR(sma); goto out_unlock; } - id = sma->sem_perm.id; } else { /* IPC_STAT */ sma = sem_obtain_object_check(ns, semid); if (IS_ERR(sma)) { @@ -1274,10 +1271,20 @@ static int semctl_stat(struct ipc_namespace *ns, int semid, #endif semid64->sem_nsems = sma->sem_nsems; + if (cmd == IPC_STAT) { + /* + * As defined in SUS: + * Return 0 on success + */ + err = 0; + } else { + /* + * SEM_STAT and SEM_STAT_ANY (both Linux specific) + * Return the full id, including the sequence number + */ + err = sma->sem_perm.id; + } ipc_unlock_object(&sma->sem_perm); - rcu_read_unlock(); - return id; - out_unlock: rcu_read_unlock(); return err; @@ -1287,7 +1294,7 @@ static int semctl_info(struct ipc_namespace *ns, int semid, int cmd, void __user *p) { struct seminfo seminfo; - int max_id; + int max_idx; int err; err = security_sem_semctl(NULL, cmd); @@ -1311,11 +1318,11 @@ static int semctl_info(struct ipc_namespace *ns, int semid, seminfo.semusz = SEMUSZ; seminfo.semaem = SEMAEM; } - max_id = ipc_get_maxid(&sem_ids(ns)); + max_idx = ipc_get_maxidx(&sem_ids(ns)); up_read(&sem_ids(ns).rwsem); if (copy_to_user(p, &seminfo, sizeof(struct seminfo))) return -EFAULT; - return (max_id < 0) ? 0 : max_id; + return (max_idx < 0) ? 0 : max_idx; } static int semctl_setval(struct ipc_namespace *ns, int semid, int semnum, @@ -1587,7 +1594,7 @@ static int semctl_down(struct ipc_namespace *ns, int semid, down_write(&sem_ids(ns).rwsem); rcu_read_lock(); - ipcp = ipcctl_pre_down_nolock(ns, &sem_ids(ns), semid, cmd, + ipcp = ipcctl_obtain_check(ns, &sem_ids(ns), semid, cmd, &semid64->sem_perm, 0); if (IS_ERR(ipcp)) { err = PTR_ERR(ipcp); @@ -2118,7 +2125,7 @@ static long do_semtimedop(int semid, struct sembuf __user *tsops, } do { - queue.status = -EINTR; + WRITE_ONCE(queue.status, -EINTR); queue.sleeper = current; __set_current_state(TASK_INTERRUPTIBLE); diff --git a/ipc/shm.c b/ipc/shm.c index 051a3e1fb8df..b0eb3757ab89 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -43,6 +43,7 @@ #include <linux/nsproxy.h> #include <linux/mount.h> #include <linux/ipc_namespace.h> +#include <linux/rhashtable.h> #include <linux/uaccess.h> @@ -95,14 +96,14 @@ static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp); static int sysvipc_shm_proc_show(struct seq_file *s, void *it); #endif -int shm_init_ns(struct ipc_namespace *ns) +void shm_init_ns(struct ipc_namespace *ns) { ns->shm_ctlmax = SHMMAX; ns->shm_ctlall = SHMALL; ns->shm_ctlmni = SHMMNI; ns->shm_rmid_forced = 0; ns->shm_tot = 0; - return ipc_init_ids(&shm_ids(ns)); + ipc_init_ids(&shm_ids(ns)); } /* @@ -135,9 +136,8 @@ void shm_exit_ns(struct ipc_namespace *ns) static int __init ipc_ns_init(void) { - const int err = shm_init_ns(&init_ipc_ns); - WARN(err, "ipc: sysv shm_init_ns failed: %d\n", err); - return err; + shm_init_ns(&init_ipc_ns); + return 0; } pure_initcall(ipc_ns_init); @@ -179,16 +179,33 @@ static inline struct shmid_kernel *shm_obtain_object_check(struct ipc_namespace */ static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id) { - struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id); + struct kern_ipc_perm *ipcp; + rcu_read_lock(); + ipcp = ipc_obtain_object_idr(&shm_ids(ns), id); + if (IS_ERR(ipcp)) + goto err; + + ipc_lock_object(ipcp); + /* + * ipc_rmid() may have already freed the ID while ipc_lock_object() + * was spinning: here verify that the structure is still valid. + * Upon races with RMID, return -EIDRM, thus indicating that + * the ID points to a removed identifier. + */ + if (ipc_valid_object(ipcp)) { + /* return a locked ipc object upon success */ + return container_of(ipcp, struct shmid_kernel, shm_perm); + } + + ipc_unlock_object(ipcp); +err: + rcu_read_unlock(); /* * Callers of shm_lock() must validate the status of the returned ipc - * object pointer (as returned by ipc_lock()), and error out as - * appropriate. + * object pointer and error out as appropriate. */ - if (IS_ERR(ipcp)) - return (void *)ipcp; - return container_of(ipcp, struct shmid_kernel, shm_perm); + return (void *)ipcp; } static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp) @@ -427,6 +444,17 @@ static int shm_split(struct vm_area_struct *vma, unsigned long addr) return 0; } +static unsigned long shm_pagesize(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct shm_file_data *sfd = shm_file_data(file); + + if (sfd->vm_ops->pagesize) + return sfd->vm_ops->pagesize(vma); + + return PAGE_SIZE; +} + #ifdef CONFIG_NUMA static int shm_set_policy(struct vm_area_struct *vma, struct mempolicy *new) { @@ -554,6 +582,7 @@ static const struct vm_operations_struct shm_vm_ops = { .close = shm_close, /* callback for when the vm-area is released */ .fault = shm_fault, .split = shm_split, + .pagesize = shm_pagesize, #if defined(CONFIG_NUMA) .set_policy = shm_set_policy, .get_policy = shm_get_policy, @@ -671,6 +700,8 @@ no_id: if (is_file_hugepages(file) && shp->mlock_user) user_shm_unlock(size, shp->mlock_user); fput(file); + ipc_rcu_putref(&shp->shm_perm, shm_rcu_free); + return error; no_file: call_rcu(&shp->shm_perm.rcu, shm_rcu_free); return error; @@ -866,7 +897,7 @@ static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd, down_write(&shm_ids(ns).rwsem); rcu_read_lock(); - ipcp = ipcctl_pre_down_nolock(ns, &shm_ids(ns), shmid, cmd, + ipcp = ipcctl_obtain_check(ns, &shm_ids(ns), shmid, cmd, &shmid64->shm_perm, 0); if (IS_ERR(ipcp)) { err = PTR_ERR(ipcp); @@ -917,7 +948,7 @@ static int shmctl_ipc_info(struct ipc_namespace *ns, shminfo->shmall = ns->shm_ctlall; shminfo->shmmin = SHMMIN; down_read(&shm_ids(ns).rwsem); - err = ipc_get_maxid(&shm_ids(ns)); + err = ipc_get_maxidx(&shm_ids(ns)); up_read(&shm_ids(ns).rwsem); if (err < 0) err = 0; @@ -937,7 +968,7 @@ static int shmctl_shm_info(struct ipc_namespace *ns, shm_info->shm_tot = ns->shm_tot; shm_info->swap_attempts = 0; shm_info->swap_successes = 0; - err = ipc_get_maxid(&shm_ids(ns)); + err = ipc_get_maxidx(&shm_ids(ns)); up_read(&shm_ids(ns).rwsem); if (err < 0) err = 0; @@ -949,7 +980,6 @@ static int shmctl_stat(struct ipc_namespace *ns, int shmid, int cmd, struct shmid64_ds *tbuf) { struct shmid_kernel *shp; - int id = 0; int err; memset(tbuf, 0, sizeof(*tbuf)); @@ -961,7 +991,6 @@ static int shmctl_stat(struct ipc_namespace *ns, int shmid, err = PTR_ERR(shp); goto out_unlock; } - id = shp->shm_perm.id; } else { /* IPC_STAT */ shp = shm_obtain_object_check(ns, shmid); if (IS_ERR(shp)) { @@ -1011,10 +1040,21 @@ static int shmctl_stat(struct ipc_namespace *ns, int shmid, tbuf->shm_lpid = pid_vnr(shp->shm_lprid); tbuf->shm_nattch = shp->shm_nattch; - ipc_unlock_object(&shp->shm_perm); - rcu_read_unlock(); - return id; + if (cmd == IPC_STAT) { + /* + * As defined in SUS: + * Return 0 on success + */ + err = 0; + } else { + /* + * SHM_STAT and SHM_STAT_ANY (both Linux specific) + * Return the full id, including the sequence number + */ + err = shp->shm_perm.id; + } + ipc_unlock_object(&shp->shm_perm); out_unlock: rcu_read_unlock(); return err; @@ -1354,15 +1394,14 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, struct shmid_kernel *shp; unsigned long addr = (unsigned long)shmaddr; unsigned long size; - struct file *file; + struct file *file, *base; int err; unsigned long flags = MAP_SHARED; unsigned long prot; int acc_mode; struct ipc_namespace *ns; struct shm_file_data *sfd; - struct path path; - fmode_t f_mode; + int f_flags; unsigned long populate = 0; err = -EINVAL; @@ -1395,11 +1434,11 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, if (shmflg & SHM_RDONLY) { prot = PROT_READ; acc_mode = S_IRUGO; - f_mode = FMODE_READ; + f_flags = O_RDONLY; } else { prot = PROT_READ | PROT_WRITE; acc_mode = S_IRUGO | S_IWUGO; - f_mode = FMODE_READ | FMODE_WRITE; + f_flags = O_RDWR; } if (shmflg & SHM_EXEC) { prot |= PROT_EXEC; @@ -1435,46 +1474,44 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, goto out_unlock; } - path = shp->shm_file->f_path; - path_get(&path); + /* + * We need to take a reference to the real shm file to prevent the + * pointer from becoming stale in cases where the lifetime of the outer + * file extends beyond that of the shm segment. It's not usually + * possible, but it can happen during remap_file_pages() emulation as + * that unmaps the memory, then does ->mmap() via file reference only. + * We'll deny the ->mmap() if the shm segment was since removed, but to + * detect shm ID reuse we need to compare the file pointers. + */ + base = get_file(shp->shm_file); shp->shm_nattch++; - size = i_size_read(d_inode(path.dentry)); + size = i_size_read(file_inode(base)); ipc_unlock_object(&shp->shm_perm); rcu_read_unlock(); err = -ENOMEM; sfd = kzalloc(sizeof(*sfd), GFP_KERNEL); if (!sfd) { - path_put(&path); + fput(base); goto out_nattch; } - file = alloc_file(&path, f_mode, - is_file_hugepages(shp->shm_file) ? + file = alloc_file_clone(base, f_flags, + is_file_hugepages(base) ? &shm_file_operations_huge : &shm_file_operations); err = PTR_ERR(file); if (IS_ERR(file)) { kfree(sfd); - path_put(&path); + fput(base); goto out_nattch; } - file->private_data = sfd; - file->f_mapping = shp->shm_file->f_mapping; sfd->id = shp->shm_perm.id; sfd->ns = get_ipc_ns(ns); - /* - * We need to take a reference to the real shm file to prevent the - * pointer from becoming stale in cases where the lifetime of the outer - * file extends beyond that of the shm segment. It's not usually - * possible, but it can happen during remap_file_pages() emulation as - * that unmaps the memory, then does ->mmap() via file reference only. - * We'll deny the ->mmap() if the shm segment was since removed, but to - * detect shm ID reuse we need to compare the file pointers. - */ - sfd->file = get_file(shp->shm_file); + sfd->file = base; sfd->vm_ops = NULL; + file->private_data = sfd; err = security_mmap_file(file, prot, flags); if (err) diff --git a/ipc/util.c b/ipc/util.c index 4e81182fa0ac..0af05752969f 100644 --- a/ipc/util.c +++ b/ipc/util.c @@ -63,6 +63,7 @@ #include <linux/rwsem.h> #include <linux/memory.h> #include <linux/ipc_namespace.h> +#include <linux/rhashtable.h> #include <asm/unistd.h> @@ -87,16 +88,12 @@ struct ipc_proc_iface { */ static int __init ipc_init(void) { - int err_sem, err_msg; - proc_mkdir("sysvipc", NULL); - err_sem = sem_init(); - WARN(err_sem, "ipc: sysv sem_init failed: %d\n", err_sem); - err_msg = msg_init(); - WARN(err_msg, "ipc: sysv msg_init failed: %d\n", err_msg); + sem_init(); + msg_init(); shm_init(); - return err_msg ? err_msg : err_sem; + return 0; } device_initcall(ipc_init); @@ -115,22 +112,17 @@ static const struct rhashtable_params ipc_kht_params = { * Set up the sequence range to use for the ipc identifier range (limited * below IPCMNI) then initialise the keys hashtable and ids idr. */ -int ipc_init_ids(struct ipc_ids *ids) +void ipc_init_ids(struct ipc_ids *ids) { - int err; ids->in_use = 0; ids->seq = 0; init_rwsem(&ids->rwsem); - err = rhashtable_init(&ids->key_ht, &ipc_kht_params); - if (err) - return err; + rhashtable_init(&ids->key_ht, &ipc_kht_params); idr_init(&ids->ipcs_idr); - ids->tables_initialized = true; - ids->max_id = -1; + ids->max_idx = -1; #ifdef CONFIG_CHECKPOINT_RESTORE ids->next_id = -1; #endif - return 0; } #ifdef CONFIG_PROC_FS @@ -178,61 +170,66 @@ void __init ipc_init_proc_interface(const char *path, const char *header, */ static struct kern_ipc_perm *ipc_findkey(struct ipc_ids *ids, key_t key) { - struct kern_ipc_perm *ipcp = NULL; + struct kern_ipc_perm *ipcp; - if (likely(ids->tables_initialized)) - ipcp = rhashtable_lookup_fast(&ids->key_ht, &key, + ipcp = rhashtable_lookup_fast(&ids->key_ht, &key, ipc_kht_params); + if (!ipcp) + return NULL; - if (ipcp) { - rcu_read_lock(); - ipc_lock_object(ipcp); - return ipcp; - } - - return NULL; + rcu_read_lock(); + ipc_lock_object(ipcp); + return ipcp; } -#ifdef CONFIG_CHECKPOINT_RESTORE /* - * Specify desired id for next allocated IPC object. + * Insert new IPC object into idr tree, and set sequence number and id + * in the correct order. + * Especially: + * - the sequence number must be set before inserting the object into the idr, + * because the sequence number is accessed without a lock. + * - the id can/must be set after inserting the object into the idr. + * All accesses must be done after getting kern_ipc_perm.lock. + * + * The caller must own kern_ipc_perm.lock.of the new object. + * On error, the function returns a (negative) error code. */ -#define ipc_idr_alloc(ids, new) \ - idr_alloc(&(ids)->ipcs_idr, (new), \ - (ids)->next_id < 0 ? 0 : ipcid_to_idx((ids)->next_id),\ - 0, GFP_NOWAIT) - -static inline int ipc_buildid(int id, struct ipc_ids *ids, - struct kern_ipc_perm *new) +static inline int ipc_idr_alloc(struct ipc_ids *ids, struct kern_ipc_perm *new) { - if (ids->next_id < 0) { /* default, behave as !CHECKPOINT_RESTORE */ + int idx, next_id = -1; + +#ifdef CONFIG_CHECKPOINT_RESTORE + next_id = ids->next_id; + ids->next_id = -1; +#endif + + /* + * As soon as a new object is inserted into the idr, + * ipc_obtain_object_idr() or ipc_obtain_object_check() can find it, + * and the lockless preparations for ipc operations can start. + * This means especially: permission checks, audit calls, allocation + * of undo structures, ... + * + * Thus the object must be fully initialized, and if something fails, + * then the full tear-down sequence must be followed. + * (i.e.: set new->deleted, reduce refcount, call_rcu()) + */ + + if (next_id < 0) { /* !CHECKPOINT_RESTORE or next_id is unset */ new->seq = ids->seq++; if (ids->seq > IPCID_SEQ_MAX) ids->seq = 0; + idx = idr_alloc(&ids->ipcs_idr, new, 0, 0, GFP_NOWAIT); } else { - new->seq = ipcid_to_seqx(ids->next_id); - ids->next_id = -1; + new->seq = ipcid_to_seqx(next_id); + idx = idr_alloc(&ids->ipcs_idr, new, ipcid_to_idx(next_id), + 0, GFP_NOWAIT); } - - return SEQ_MULTIPLIER * new->seq + id; + if (idx >= 0) + new->id = SEQ_MULTIPLIER * new->seq + idx; + return idx; } -#else -#define ipc_idr_alloc(ids, new) \ - idr_alloc(&(ids)->ipcs_idr, (new), 0, 0, GFP_NOWAIT) - -static inline int ipc_buildid(int id, struct ipc_ids *ids, - struct kern_ipc_perm *new) -{ - new->seq = ids->seq++; - if (ids->seq > IPCID_SEQ_MAX) - ids->seq = 0; - - return SEQ_MULTIPLIER * new->seq + id; -} - -#endif /* CONFIG_CHECKPOINT_RESTORE */ - /** * ipc_addid - add an ipc identifier * @ids: ipc identifier set @@ -240,9 +237,11 @@ static inline int ipc_buildid(int id, struct ipc_ids *ids, * @limit: limit for the number of used ids * * Add an entry 'new' to the ipc ids idr. The permissions object is - * initialised and the first free entry is set up and the id assigned + * initialised and the first free entry is set up and the index assigned * is returned. The 'new' entry is returned in a locked state on success. + * * On failure the entry is not locked and a negative err-code is returned. + * The caller must use ipc_rcu_putref() to free the identifier. * * Called with writer ipc_ids.rwsem held. */ @@ -250,19 +249,20 @@ int ipc_addid(struct ipc_ids *ids, struct kern_ipc_perm *new, int limit) { kuid_t euid; kgid_t egid; - int id, err; + int idx, err; + + /* 1) Initialize the refcount so that ipc_rcu_putref works */ + refcount_set(&new->refcount, 1); if (limit > IPCMNI) limit = IPCMNI; - if (!ids->tables_initialized || ids->in_use >= limit) + if (ids->in_use >= limit) return -ENOSPC; idr_preload(GFP_KERNEL); - refcount_set(&new->refcount, 1); spin_lock_init(&new->lock); - new->deleted = false; rcu_read_lock(); spin_lock(&new->lock); @@ -270,30 +270,30 @@ int ipc_addid(struct ipc_ids *ids, struct kern_ipc_perm *new, int limit) new->cuid = new->uid = euid; new->gid = new->cgid = egid; - id = ipc_idr_alloc(ids, new); + new->deleted = false; + + idx = ipc_idr_alloc(ids, new); idr_preload_end(); - if (id >= 0 && new->key != IPC_PRIVATE) { + if (idx >= 0 && new->key != IPC_PRIVATE) { err = rhashtable_insert_fast(&ids->key_ht, &new->khtnode, ipc_kht_params); if (err < 0) { - idr_remove(&ids->ipcs_idr, id); - id = err; + idr_remove(&ids->ipcs_idr, idx); + idx = err; } } - if (id < 0) { + if (idx < 0) { + new->deleted = true; spin_unlock(&new->lock); rcu_read_unlock(); - return id; + return idx; } ids->in_use++; - if (id > ids->max_id) - ids->max_id = id; - - new->id = ipc_buildid(id, ids, new); - - return id; + if (idx > ids->max_idx) + ids->max_idx = idx; + return idx; } /** @@ -431,20 +431,20 @@ static void ipc_kht_remove(struct ipc_ids *ids, struct kern_ipc_perm *ipcp) */ void ipc_rmid(struct ipc_ids *ids, struct kern_ipc_perm *ipcp) { - int lid = ipcid_to_idx(ipcp->id); + int idx = ipcid_to_idx(ipcp->id); - idr_remove(&ids->ipcs_idr, lid); + idr_remove(&ids->ipcs_idr, idx); ipc_kht_remove(ids, ipcp); ids->in_use--; ipcp->deleted = true; - if (unlikely(lid == ids->max_id)) { + if (unlikely(idx == ids->max_idx)) { do { - lid--; - if (lid == -1) + idx--; + if (idx == -1) break; - } while (!idr_find(&ids->ipcs_idr, lid)); - ids->max_id = lid; + } while (!idr_find(&ids->ipcs_idr, idx)); + ids->max_idx = idx; } } @@ -462,7 +462,7 @@ void ipc_set_key_private(struct ipc_ids *ids, struct kern_ipc_perm *ipcp) ipcp->key = IPC_PRIVATE; } -int ipc_rcu_getref(struct kern_ipc_perm *ptr) +bool ipc_rcu_getref(struct kern_ipc_perm *ptr) { return refcount_inc_not_zero(&ptr->refcount); } @@ -564,12 +564,9 @@ void ipc64_perm_to_ipc_perm(struct ipc64_perm *in, struct ipc_perm *out) struct kern_ipc_perm *ipc_obtain_object_idr(struct ipc_ids *ids, int id) { struct kern_ipc_perm *out; - int lid = ipcid_to_idx(id); - - if (unlikely(!ids->tables_initialized)) - return ERR_PTR(-EINVAL); + int idx = ipcid_to_idx(id); - out = idr_find(&ids->ipcs_idr, lid); + out = idr_find(&ids->ipcs_idr, idx); if (!out) return ERR_PTR(-EINVAL); @@ -577,48 +574,12 @@ struct kern_ipc_perm *ipc_obtain_object_idr(struct ipc_ids *ids, int id) } /** - * ipc_lock - lock an ipc structure without rwsem held - * @ids: ipc identifier set - * @id: ipc id to look for - * - * Look for an id in the ipc ids idr and lock the associated ipc object. - * - * The ipc object is locked on successful exit. - */ -struct kern_ipc_perm *ipc_lock(struct ipc_ids *ids, int id) -{ - struct kern_ipc_perm *out; - - rcu_read_lock(); - out = ipc_obtain_object_idr(ids, id); - if (IS_ERR(out)) - goto err; - - spin_lock(&out->lock); - - /* - * ipc_rmid() may have already freed the ID while ipc_lock() - * was spinning: here verify that the structure is still valid. - * Upon races with RMID, return -EIDRM, thus indicating that - * the ID points to a removed identifier. - */ - if (ipc_valid_object(out)) - return out; - - spin_unlock(&out->lock); - out = ERR_PTR(-EIDRM); -err: - rcu_read_unlock(); - return out; -} - -/** * ipc_obtain_object_check * @ids: ipc identifier set * @id: ipc id to look for * - * Similar to ipc_obtain_object_idr() but also checks - * the ipc object reference counter. + * Similar to ipc_obtain_object_idr() but also checks the ipc object + * sequence number. * * Call inside the RCU critical section. * The ipc object is *not* locked on exit. @@ -676,7 +637,7 @@ int ipc_update_perm(struct ipc64_perm *in, struct kern_ipc_perm *out) } /** - * ipcctl_pre_down_nolock - retrieve an ipc and check permissions for some IPC_XXX cmd + * ipcctl_obtain_check - retrieve an ipc object and check permissions * @ns: ipc namespace * @ids: the table of ids where to look for the ipc * @id: the id of the ipc to retrieve @@ -686,16 +647,16 @@ int ipc_update_perm(struct ipc64_perm *in, struct kern_ipc_perm *out) * * This function does some common audit and permissions check for some IPC_XXX * cmd and is called from semctl_down, shmctl_down and msgctl_down. - * It must be called without any lock held and: * - * - retrieves the ipc with the given id in the given table. + * It: + * - retrieves the ipc object with the given id in the given table. * - performs some audit and permission check, depending on the given cmd * - returns a pointer to the ipc object or otherwise, the corresponding * error. * * Call holding the both the rwsem and the rcu read lock. */ -struct kern_ipc_perm *ipcctl_pre_down_nolock(struct ipc_namespace *ns, +struct kern_ipc_perm *ipcctl_obtain_check(struct ipc_namespace *ns, struct ipc_ids *ids, int id, int cmd, struct ipc64_perm *perm, int extra_perm) { diff --git a/ipc/util.h b/ipc/util.h index 0aba3230d007..0a159f69b3bb 100644 --- a/ipc/util.h +++ b/ipc/util.h @@ -18,8 +18,8 @@ #define IPCMNI 32768 /* <= MAX_INT limit for ipc arrays (including sysctl changes) */ #define SEQ_MULTIPLIER (IPCMNI) -int sem_init(void); -int msg_init(void); +void sem_init(void); +void msg_init(void); void shm_init(void); struct ipc_namespace; @@ -34,17 +34,17 @@ static inline void mq_put_mnt(struct ipc_namespace *ns) { } #endif #ifdef CONFIG_SYSVIPC -int sem_init_ns(struct ipc_namespace *ns); -int msg_init_ns(struct ipc_namespace *ns); -int shm_init_ns(struct ipc_namespace *ns); +void sem_init_ns(struct ipc_namespace *ns); +void msg_init_ns(struct ipc_namespace *ns); +void shm_init_ns(struct ipc_namespace *ns); void sem_exit_ns(struct ipc_namespace *ns); void msg_exit_ns(struct ipc_namespace *ns); void shm_exit_ns(struct ipc_namespace *ns); #else -static inline int sem_init_ns(struct ipc_namespace *ns) { return 0; } -static inline int msg_init_ns(struct ipc_namespace *ns) { return 0; } -static inline int shm_init_ns(struct ipc_namespace *ns) { return 0; } +static inline void sem_init_ns(struct ipc_namespace *ns) { } +static inline void msg_init_ns(struct ipc_namespace *ns) { } +static inline void shm_init_ns(struct ipc_namespace *ns) { } static inline void sem_exit_ns(struct ipc_namespace *ns) { } static inline void msg_exit_ns(struct ipc_namespace *ns) { } @@ -83,7 +83,7 @@ struct ipc_ops { struct seq_file; struct ipc_ids; -int ipc_init_ids(struct ipc_ids *); +void ipc_init_ids(struct ipc_ids *ids); #ifdef CONFIG_PROC_FS void __init ipc_init_proc_interface(const char *path, const char *header, int ids, int (*show)(struct seq_file *, void *)); @@ -113,12 +113,12 @@ void ipc_set_key_private(struct ipc_ids *, struct kern_ipc_perm *); int ipcperms(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp, short flg); /** - * ipc_get_maxid - get the last assigned id + * ipc_get_maxidx - get the highest assigned index * @ids: ipc identifier set * * Called with ipc_ids.rwsem held for reading. */ -static inline int ipc_get_maxid(struct ipc_ids *ids) +static inline int ipc_get_maxidx(struct ipc_ids *ids) { if (ids->in_use == 0) return -1; @@ -126,7 +126,7 @@ static inline int ipc_get_maxid(struct ipc_ids *ids) if (ids->in_use == IPCMNI) return IPCMNI - 1; - return ids->max_id; + return ids->max_idx; } /* @@ -138,17 +138,16 @@ static inline int ipc_get_maxid(struct ipc_ids *ids) * refcount is initialized by ipc_addid(), before that point call_rcu() * must be used. */ -int ipc_rcu_getref(struct kern_ipc_perm *ptr); +bool ipc_rcu_getref(struct kern_ipc_perm *ptr); void ipc_rcu_putref(struct kern_ipc_perm *ptr, void (*func)(struct rcu_head *head)); -struct kern_ipc_perm *ipc_lock(struct ipc_ids *, int); struct kern_ipc_perm *ipc_obtain_object_idr(struct ipc_ids *ids, int id); void kernel_to_ipc64_perm(struct kern_ipc_perm *in, struct ipc64_perm *out); void ipc64_perm_to_ipc_perm(struct ipc64_perm *in, struct ipc_perm *out); int ipc_update_perm(struct ipc64_perm *in, struct kern_ipc_perm *out); -struct kern_ipc_perm *ipcctl_pre_down_nolock(struct ipc_namespace *ns, +struct kern_ipc_perm *ipcctl_obtain_check(struct ipc_namespace *ns, struct ipc_ids *ids, int id, int cmd, struct ipc64_perm *perm, int extra_perm); @@ -173,9 +172,9 @@ extern struct msg_msg *load_msg(const void __user *src, size_t len); extern struct msg_msg *copy_msg(struct msg_msg *src, struct msg_msg *dst); extern int store_msg(void __user *dest, struct msg_msg *msg, size_t len); -static inline int ipc_checkid(struct kern_ipc_perm *ipcp, int uid) +static inline int ipc_checkid(struct kern_ipc_perm *ipcp, int id) { - return uid / SEQ_MULTIPLIER != ipcp->seq; + return ipcid_to_seqx(id) != ipcp->seq; } static inline void ipc_lock_object(struct kern_ipc_perm *perm) |