Put file descriptors on shared data structures when they are completely

setup, take 3.

LARVAL fd still exist, but they are no longer marked with a flag and no
longer reachable via `fd_ofiles[]' or the global linked list.  This allows
us to simplifies a lot code grabbing new references to fds.

All of this is now possible because dup2(2) refuses to clone LARVAL fds.

Note that the `fdplock' could now be release in all open(2)-like syscalls,
just like it is done in accept(2).

With inputs from Mathieu Masson, visa@, guenther@ and art@

Previous version ok bluhm@, ok visa@, sthen@

12 files changed, 127 insertions, 115 deletions

diff --git a/sys/kern/exec_script.c b/sys/kern/exec_script.c
index d6ac23ba26f..788260254f5 100644
--- a/sys/kern/exec_script.c
+++ b/sys/kern/exec_script.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: exec_script.c,v 1.46 2018/06/05 09:29:05 mpi Exp $	*/
+/*	$OpenBSD: exec_script.c,v 1.47 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: exec_script.c,v 1.13 1996/02/04 02:15:06 christos Exp $	*/
 
 /*
@@ -170,17 +170,20 @@ check_shell:
 #endif
 
 		fdplock(p->p_fd);
-		error = falloc(p, 0, &fp, &epp->ep_fd);
-		fdpunlock(p->p_fd);
-		if (error)
+		error = falloc(p, &fp, &epp->ep_fd);
+		if (error) {
+			fdpunlock(p->p_fd);
 			goto fail;
+		}
 
 		epp->ep_flags |= EXEC_HASFD;
 		fp->f_type = DTYPE_VNODE;
 		fp->f_ops = &vnops;
 		fp->f_data = (caddr_t) scriptvp;
 		fp->f_flag = FREAD;
-		FILE_SET_MATURE(fp, p);
+		fdinsert(p->p_fd, epp->ep_fd, 0, fp);
+		fdpunlock(p->p_fd);
+		FRELE(fp, p);
 	}
 
 	/* set up the parameters for the recursive check_exec() call */
diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 4d7b0a44016..7ccc3e49f03 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_descrip.c,v 1.165 2018/06/17 08:22:02 anton Exp $	*/
+/*	$OpenBSD: kern_descrip.c,v 1.166 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: kern_descrip.c,v 1.42 1996/03/30 22:24:38 christos Exp $	*/
 
 /*
@@ -73,6 +73,7 @@ int numfiles;			/* actual number of open files */
 static __inline void fd_used(struct filedesc *, int);
 static __inline void fd_unused(struct filedesc *, int);
 static __inline int find_next_zero(u_int *, int, u_int);
+static __inline int fd_inuse(struct filedesc *, int);
 int finishdup(struct proc *, struct file *, int, int, register_t *, int);
 int find_last_set(struct filedesc *, int);
 int dodup3(struct proc *, int, int, int, register_t *);
@@ -125,7 +126,6 @@ int
 find_last_set(struct filedesc *fd, int last)
 {
 	int off, i;
-	struct file **ofiles = fd->fd_ofiles;
 	u_int *bitmap = fd->fd_lomap;
 
 	off = (last - 1) >> NDENTRYSHIFT;
@@ -139,11 +139,22 @@ find_last_set(struct filedesc *fd, int last)
 	if (i >= last)
 		i = last - 1;
 
-	while (i > 0 && ofiles[i] == NULL)
+	while (i > 0 && !fd_inuse(fd, i))
 		i--;
 	return i;
 }
 
+static __inline int
+fd_inuse(struct filedesc *fdp, int fd)
+{
+	u_int off = fd >> NDENTRYSHIFT;
+
+	if (fdp->fd_lomap[off] & (1 << (fd & NDENTRYMASK)))
+		return 1;
+
+	return 0;
+}
+
 static __inline void
 fd_used(struct filedesc *fdp, int fd)
 {
@@ -190,7 +201,7 @@ fd_iterfile(struct file *fp, struct proc *p)
 		nfp = LIST_NEXT(fp, f_list);
 
 	/* don't FREF when f_count == 0 to avoid race in fdrop() */
-	while (nfp != NULL && (nfp->f_count == 0 || !FILE_IS_USABLE(nfp)))
+	while (nfp != NULL && nfp->f_count == 0)
 		nfp = LIST_NEXT(nfp, f_list);
 	if (nfp != NULL)
 		FREF(nfp);
@@ -209,9 +220,6 @@ fd_getfile(struct filedesc *fdp, int fd)
 	if ((u_int)fd >= fdp->fd_nfiles || (fp = fdp->fd_ofiles[fd]) == NULL)
 		return (NULL);
 
-	if (!FILE_IS_USABLE(fp))
-		return (NULL);
-
 	FREF(fp);
 	return (fp);
 }
@@ -629,24 +637,24 @@ finishdup(struct proc *p, struct file *fp, int old, int new,
 	struct filedesc *fdp = p->p_fd;
 
 	fdpassertlocked(fdp);
+	KASSERT(fp->f_iflags & FIF_INSERTED);
+
 	if (fp->f_count == LONG_MAX-2) {
 		FRELE(fp, p);
 		return (EDEADLK);
 	}
 
-	oldfp = fdp->fd_ofiles[new];
-	if (oldfp != NULL) {
-		if (!FILE_IS_USABLE(oldfp)) {
-			FRELE(fp, p);
-			return (EBUSY);
-		}
-		FREF(oldfp);
-	}
+	oldfp = fd_getfile(fdp, new);
+	if (dup2 && oldfp == NULL) {
+		if (fd_inuse(fdp, new)) {
+ 			FRELE(fp, p);
+ 			return (EBUSY);
+ 		}
+		fd_used(fdp, new);
+ 	}
 
 	fdp->fd_ofiles[new] = fp;
 	fdp->fd_ofileflags[new] = fdp->fd_ofileflags[old] & ~UF_EXCLOSE;
-	if (dup2 && oldfp == NULL)
-		fd_used(fdp, new);
 	*retval = new;
 
 	if (oldfp != NULL) {
@@ -658,6 +666,23 @@ finishdup(struct proc *p, struct file *fp, int old, int new,
 }
 
 void
+fdinsert(struct filedesc *fdp, int fd, int flags, struct file *fp)
+{
+	struct file *fq;
+
+	fdpassertlocked(fdp);
+	if ((fq = fdp->fd_ofiles[0]) != NULL) {
+		LIST_INSERT_AFTER(fq, fp, f_list);
+	} else {
+		LIST_INSERT_HEAD(&filehead, fp, f_list);
+	}
+	KASSERT(fdp->fd_ofiles[fd] == NULL);
+	fdp->fd_ofiles[fd] = fp;
+	fdp->fd_ofileflags[fd] |= (flags & UF_EXCLOSE);
+	fp->f_iflags |= FIF_INSERTED;
+}
+
+void
 fdremove(struct filedesc *fdp, int fd)
 {
 	fdpassertlocked(fdp);
@@ -670,21 +695,14 @@ int
 fdrelease(struct proc *p, int fd)
 {
 	struct filedesc *fdp = p->p_fd;
-	struct file **fpp, *fp;
+	struct file *fp;
 
 	fdpassertlocked(fdp);
 
-	/*
-	 * Don't fd_getfile here. We want to closef LARVAL files and closef
-	 * can deal with that.
-	 */
-	fpp = &fdp->fd_ofiles[fd];
-	fp = *fpp;
+	fp = fd_getfile(fdp, fd);
 	if (fp == NULL)
 		return (EBADF);
-	FREF(fp);
-	*fpp = NULL;
-	fd_unused(fdp, fd);
+	fdremove(fdp, fd);
 	knote_fdclose(p, fd);
 	return (closef(fp, p));
 }
@@ -700,12 +718,6 @@ sys_close(struct proc *p, void *v, register_t *retval)
 	} */ *uap = v;
 	int fd = SCARG(uap, fd), error;
 	struct filedesc *fdp = p->p_fd;
-	struct file *fp;
-
-	fp = fd_getfile(fdp, fd);
-	if (fp == NULL)
-		return (EBADF);
-	FRELE(fp, p);
 
 	fdplock(fdp);
 	error = fdrelease(p, fd);
@@ -926,9 +938,9 @@ fdexpand(struct proc *p)
  * a file descriptor for the process that refers to it.
  */
 int
-falloc(struct proc *p, int flags, struct file **resultfp, int *resultfd)
+falloc(struct proc *p, struct file **resultfp, int *resultfd)
 {
-	struct file *fp, *fq;
+	struct file *fp;
 	int error, i;
 
 	KASSERT(resultfp != NULL);
@@ -961,14 +973,6 @@ restart:
 	 * with and without the KERNEL_LOCK().
 	 */
 	mtx_init(&fp->f_mtx, IPL_MPFLOOR);
-	fp->f_iflags = FIF_LARVAL;
-	if ((fq = p->p_fd->fd_ofiles[0]) != NULL) {
-		LIST_INSERT_AFTER(fq, fp, f_list);
-	} else {
-		LIST_INSERT_HEAD(&filehead, fp, f_list);
-	}
-	p->p_fd->fd_ofiles[i] = fp;
-	p->p_fd->fd_ofileflags[i] |= (flags & UF_EXCLOSE);
 	fp->f_count = 1;
 	fp->f_cred = p->p_ucred;
 	crhold(fp->f_cred);
@@ -1191,8 +1195,8 @@ fdrop(struct file *fp, struct proc *p)
 	else
 		error = 0;
 
-	/* Free fp */
-	LIST_REMOVE(fp, f_list);
+	if (fp->f_iflags & FIF_INSERTED)
+		LIST_REMOVE(fp, f_list);
 	crfree(fp->f_cred);
 	numfiles--;
 	pool_put(&file_pool, fp);
@@ -1307,7 +1311,7 @@ dupfdopen(struct proc *p, int indx, int mode)
 	 * of file descriptors, or the fd to be dup'd has already been
 	 * closed, reject. Note, there is no need to check for new == old
 	 * because fd_getfile will return NULL if the file at indx is
-	 * newly created by falloc (FIF_LARVAL).
+	 * newly created by falloc.
 	 */
 	if ((wfp = fd_getfile(fdp, dupfd)) == NULL)
 		return (EBADF);
diff --git a/sys/kern/kern_event.c b/sys/kern/kern_event.c
index 809ed1f03a2..2ea8dbf45f8 100644
--- a/sys/kern/kern_event.c
+++ b/sys/kern/kern_event.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_event.c,v 1.93 2018/06/17 08:22:02 anton Exp $	*/
+/*	$OpenBSD: kern_event.c,v 1.94 2018/06/18 09:15:05 mpi Exp $	*/
 
 /*-
  * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
@@ -445,10 +445,9 @@ sys_kqueue(struct proc *p, void *v, register_t *retval)
 	int fd, error;
 
 	fdplock(fdp);
-	error = falloc(p, 0, &fp, &fd);
-	fdpunlock(fdp);
+	error = falloc(p, &fp, &fd);
 	if (error)
-		return (error);
+		goto out;
 	fp->f_flag = FREAD | FWRITE;
 	fp->f_type = DTYPE_KQUEUE;
 	fp->f_ops = &kqueueops;
@@ -458,9 +457,12 @@ sys_kqueue(struct proc *p, void *v, register_t *retval)
 	KQREF(kq);
 	*retval = fd;
 	kq->kq_fdp = fdp;
-	FILE_SET_MATURE(fp, p);
 	LIST_INSERT_HEAD(&p->p_p->ps_kqlist, kq, kq_next);
-	return (0);
+	fdinsert(fdp, fd, 0, fp);
+	FRELE(fp, p);
+out:
+	fdpunlock(fdp);
+	return (error);
 }
 
 int
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 9051cf4d054..7ccbab6a374 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_exec.c,v 1.197 2018/06/05 09:29:05 mpi Exp $	*/
+/*	$OpenBSD: kern_exec.c,v 1.198 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: kern_exec.c,v 1.75 1996/02/09 18:59:28 christos Exp $	*/
 
 /*-
@@ -584,7 +584,7 @@ sys_execve(struct proc *p, void *v, register_t *retval)
 				struct vnode *vp;
 				int indx;
 
-				if ((error = falloc(p, 0, &fp, &indx)) != 0)
+				if ((error = falloc(p, &fp, &indx)) != 0)
 					break;
 #ifdef DIAGNOSTIC
 				if (indx != i)
@@ -607,10 +607,9 @@ sys_execve(struct proc *p, void *v, register_t *retval)
 				fp->f_type = DTYPE_VNODE;
 				fp->f_ops = &vnops;
 				fp->f_data = (caddr_t)vp;
-				FILE_SET_MATURE(fp, p);
-			} else {
-				FRELE(fp, p);
+				fdinsert(p->p_fd, indx, 0, fp);
 			}
+			FRELE(fp, p);
 		}
 		fdpunlock(p->p_fd);
 		if (error)
diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index f34192d2335..b1a6c51137f 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sys_pipe.c,v 1.80 2018/06/05 09:29:05 mpi Exp $	*/
+/*	$OpenBSD: sys_pipe.c,v 1.81 2018/06/18 09:15:05 mpi Exp $	*/
 
 /*
  * Copyright (c) 1996 John S. Dyson
@@ -154,7 +154,7 @@ dopipe(struct proc *p, int *ufds, int flags)
 
 	fdplock(fdp);
 
-	error = falloc(p, cloexec, &rf, &fds[0]);
+	error = falloc(p, &rf, &fds[0]);
 	if (error != 0)
 		goto free2;
 	rf->f_flag = FREAD | FWRITE | (flags & FNONBLOCK);
@@ -162,7 +162,7 @@ dopipe(struct proc *p, int *ufds, int flags)
 	rf->f_data = rpipe;
 	rf->f_ops = &pipeops;
 
-	error = falloc(p, cloexec, &wf, &fds[1]);
+	error = falloc(p, &wf, &fds[1]);
 	if (error != 0)
 		goto free3;
 	wf->f_flag = FREAD | FWRITE | (flags & FNONBLOCK);
@@ -173,8 +173,8 @@ dopipe(struct proc *p, int *ufds, int flags)
 	rpipe->pipe_peer = wpipe;
 	wpipe->pipe_peer = rpipe;
 
-	FILE_SET_MATURE(rf, p);
-	FILE_SET_MATURE(wf, p);
+	fdinsert(fdp, fds[0], cloexec, rf);
+	fdinsert(fdp, fds[1], cloexec, wf);
 
 	error = copyout(fds, ufds, sizeof(fds));
 	if (error != 0) {
@@ -186,6 +186,9 @@ dopipe(struct proc *p, int *ufds, int flags)
 		ktrfds(p, fds, 2);
 #endif
 	fdpunlock(fdp);
+
+	FRELE(rf, p);
+	FRELE(wf, p);
 	return (error);
 
 free3:
diff --git a/sys/kern/tty_pty.c b/sys/kern/tty_pty.c
index db69167b1fb..3b17c5250f0 100644
--- a/sys/kern/tty_pty.c
+++ b/sys/kern/tty_pty.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: tty_pty.c,v 1.86 2018/06/05 09:29:05 mpi Exp $	*/
+/*	$OpenBSD: tty_pty.c,v 1.87 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: tty_pty.c,v 1.33.4.1 1996/06/02 09:08:11 mrg Exp $	*/
 
 /*
@@ -1070,11 +1070,11 @@ ptmioctl(dev_t dev, u_long cmd, caddr_t data, int flag, struct proc *p)
 	case PTMGET:
 		fdplock(fdp);
 		/* Grab two filedescriptors. */
-		if ((error = falloc(p, 0, &cfp, &cindx)) != 0) {
+		if ((error = falloc(p, &cfp, &cindx)) != 0) {
 			fdpunlock(fdp);
 			break;
 		}
-		if ((error = falloc(p, 0, &sfp, &sindx)) != 0) {
+		if ((error = falloc(p, &sfp, &sindx)) != 0) {
 			fdremove(fdp, cindx);
 			closef(cfp, p);
 			fdpunlock(fdp);
@@ -1166,11 +1166,12 @@ retry:
 		memcpy(ptm->cn, pti->pty_pn, sizeof(pti->pty_pn));
 		memcpy(ptm->sn, pti->pty_sn, sizeof(pti->pty_sn));
 
-		/* mark the files mature now that we've passed all errors */
-		FILE_SET_MATURE(cfp, p);
-		FILE_SET_MATURE(sfp, p);
-
+		/* insert files now that we've passed all errors */
+		fdinsert(fdp, cindx, 0, cfp);
+		fdinsert(fdp, sindx, 0, sfp);
 		fdpunlock(fdp);
+		FRELE(cfp, p);
+		FRELE(sfp, p);
 		break;
 	default:
 		error = EINVAL;
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 6e676f9a898..09ae2fdf5ab 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: uipc_syscalls.c,v 1.175 2018/06/06 06:55:22 mpi Exp $	*/
+/*	$OpenBSD: uipc_syscalls.c,v 1.176 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: uipc_syscalls.c,v 1.19 1996/02/09 19:00:48 christos Exp $	*/
 
 /*
@@ -106,9 +106,9 @@ sys_socket(struct proc *p, void *v, register_t *retval)
 
 	KERNEL_LOCK();
 	fdplock(fdp);
-	error = falloc(p, cloexec, &fp, &fd);
-	fdpunlock(fdp);
+	error = falloc(p, &fp, &fd);
 	if (error) {
+		fdpunlock(fdp);
 		soclose(so);
 	} else {
 		fp->f_flag = fflag;
@@ -118,7 +118,9 @@ sys_socket(struct proc *p, void *v, register_t *retval)
 			so->so_state |= SS_NBIO;
 		so->so_state |= ss;
 		fp->f_data = so;
-		FILE_SET_MATURE(fp, p);
+		fdinsert(fdp, fd, cloexec, fp);
+		fdpunlock(fdp);
+		FRELE(fp, p);
 		*retval = fd;
 	}
 	KERNEL_UNLOCK();
@@ -273,7 +275,9 @@ doaccept(struct proc *p, int sock, struct sockaddr *name, socklen_t *anamelen,
 	socklen_t namelen;
 	int error, s, tmpfd;
 	struct socket *head, *so;
-	int nflag;
+	int cloexec, nflag;
+
+	cloexec = (flags & SOCK_CLOEXEC) ? UF_EXCLOSE : 0;
 
 	if (name && (error = copyin(anamelen, &namelen, sizeof (namelen))))
 		return (error);
@@ -283,7 +287,7 @@ doaccept(struct proc *p, int sock, struct sockaddr *name, socklen_t *anamelen,
 	headfp = fp;
 
 	fdplock(fdp);
-	error = falloc(p, (flags & SOCK_CLOEXEC) ? UF_EXCLOSE : 0, &fp, &tmpfd);
+	error = falloc(p, &fp, &tmpfd);
 	fdpunlock(fdp);
 	if (error) {
 		FRELE(headfp, p);
@@ -348,8 +352,11 @@ out:
 		else
 			so->so_state &= ~SS_NBIO;
 		sounlock(head, s);
+		fdplock(fdp);
 		fp->f_data = so;
-		FILE_SET_MATURE(fp, p);
+		fdinsert(fdp, tmpfd, cloexec, fp);
+		fdpunlock(fdp);
+		FRELE(fp, p);
 		*retval = tmpfd;
 	} else {
 		sounlock(head, s);
@@ -478,13 +485,13 @@ sys_socketpair(struct proc *p, void *v, register_t *retval)
 	}
 	KERNEL_LOCK();
 	fdplock(fdp);
-	if ((error = falloc(p, cloexec, &fp1, &sv[0])) != 0)
+	if ((error = falloc(p, &fp1, &sv[0])) != 0)
 		goto free3;
 	fp1->f_flag = fflag;
 	fp1->f_type = DTYPE_SOCKET;
 	fp1->f_ops = &socketops;
 	fp1->f_data = so1;
-	if ((error = falloc(p, cloexec, &fp2, &sv[1])) != 0)
+	if ((error = falloc(p, &fp2, &sv[1])) != 0)
 		goto free4;
 	fp2->f_flag = fflag;
 	fp2->f_type = DTYPE_SOCKET;
@@ -502,9 +509,11 @@ sys_socketpair(struct proc *p, void *v, register_t *retval)
 			(*fp2->f_ops->fo_ioctl)(fp2, FIONBIO, (caddr_t)&type,
 			    p);
 		}
-		FILE_SET_MATURE(fp1, p);
-		FILE_SET_MATURE(fp2, p);
+		fdinsert(fdp, sv[0], cloexec, fp1);
+		fdinsert(fdp, sv[1], cloexec, fp2);
 		fdpunlock(fdp);
+		FRELE(fp1, p);
+		FRELE(fp2, p);
 		KERNEL_UNLOCK();
 		return (0);
 	}
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 4d4c3364124..aa031b6f8b7 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vfs_syscalls.c,v 1.289 2018/06/14 00:23:36 millert Exp $	*/
+/*	$OpenBSD: vfs_syscalls.c,v 1.290 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: vfs_syscalls.c,v 1.71 1996/04/23 10:29:02 mycroft Exp $	*/
 
 /*
@@ -899,7 +899,7 @@ doopenat(struct proc *p, int fd, const char *path, int oflags, mode_t mode,
 	struct file *fp;
 	struct vnode *vp;
 	struct vattr vattr;
-	int flags, cmode;
+	int flags, cloexec, cmode;
 	int type, indx, error, localtrunc = 0;
 	struct flock lf;
 	struct nameidata nd;
@@ -911,10 +911,10 @@ doopenat(struct proc *p, int fd, const char *path, int oflags, mode_t mode,
 			return (error);
 	}
 
-	fdplock(fdp);
+	cloexec = (oflags & O_CLOEXEC) ? UF_EXCLOSE : 0;
 
-	if ((error = falloc(p, (oflags & O_CLOEXEC) ? UF_EXCLOSE : 0, &fp,
-	    &indx)) != 0)
+	fdplock(fdp);
+	if ((error = falloc(p, &fp, &indx)) != 0)
 		goto out;
 	flags = FFLAGS(oflags);
 	if (flags & FREAD)
@@ -999,7 +999,8 @@ doopenat(struct proc *p, int fd, const char *path, int oflags, mode_t mode,
 	}
 	VOP_UNLOCK(vp);
 	*retval = indx;
-	FILE_SET_MATURE(fp, p);
+	fdinsert(fdp, indx, cloexec, fp);
+	FRELE(fp, p);
 out:
 	fdpunlock(fdp);
 	return (error);
@@ -1060,7 +1061,7 @@ sys_fhopen(struct proc *p, void *v, register_t *retval)
 	struct vnode *vp = NULL;
 	struct mount *mp;
 	struct ucred *cred = p->p_ucred;
-	int flags;
+	int flags, cloexec;
 	int type, indx, error=0;
 	struct flock lf;
 	struct vattr va;
@@ -1078,9 +1079,10 @@ sys_fhopen(struct proc *p, void *v, register_t *retval)
 	if ((flags & O_CREAT))
 		return (EINVAL);
 
+	cloexec = (flags & O_CLOEXEC) ? UF_EXCLOSE : 0;
+
 	fdplock(fdp);
-	if ((error = falloc(p, (flags & O_CLOEXEC) ? UF_EXCLOSE : 0, &fp,
-	    &indx)) != 0) {
+	if ((error = falloc(p, &fp, &indx)) != 0) {
 		fp = NULL;
 		goto bad;
 	}
@@ -1160,9 +1162,9 @@ sys_fhopen(struct proc *p, void *v, register_t *retval)
 	}
 	VOP_UNLOCK(vp);
 	*retval = indx;
-	FILE_SET_MATURE(fp, p);
-
+	fdinsert(fdp, indx, cloexec, fp);
 	fdpunlock(fdp);
+	FRELE(fp, p);
 	return (0);
 
 bad:
diff --git a/sys/sys/file.h b/sys/sys/file.h
index e4fd595eae3..49f325cc8d0 100644
--- a/sys/sys/file.h
+++ b/sys/sys/file.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: file.h,v 1.47 2018/06/05 09:29:05 mpi Exp $	*/
+/*	$OpenBSD: file.h,v 1.48 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: file.h,v 1.11 1995/03/26 20:24:13 jtc Exp $	*/
 
 /*
@@ -91,10 +91,7 @@ struct file {
 };
 
 #define FIF_HASLOCK		0x01	/* descriptor holds advisory lock */
-#define FIF_LARVAL		0x02	/* not fully constructed, don't use */
-
-#define FILE_IS_USABLE(fp) \
-	(((fp)->f_iflags & FIF_LARVAL) == 0)
+#define FIF_INSERTED		0x80	/* present in `filehead' */
 
 #define FREF(fp) \
 	do { \
@@ -104,11 +101,6 @@ struct file {
 	} while (0)
 #define FRELE(fp,p)	(--(fp)->f_count == 0 ? fdrop(fp, p) : 0)
 
-#define FILE_SET_MATURE(fp,p) do {				\
-	(fp)->f_iflags &= ~FIF_LARVAL;				\
-	FRELE(fp, p);						\
-} while (0)
-
 int	fdrop(struct file *, struct proc *);
 
 LIST_HEAD(filelist, file);
diff --git a/sys/sys/filedesc.h b/sys/sys/filedesc.h
index eab3ba7e885..f2f22a8ffdd 100644
--- a/sys/sys/filedesc.h
+++ b/sys/sys/filedesc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: filedesc.h,v 1.38 2018/06/17 08:22:02 anton Exp $	*/
+/*	$OpenBSD: filedesc.h,v 1.39 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: filedesc.h,v 1.14 1996/04/09 20:55:28 cgd Exp $	*/
 
 /*
@@ -120,12 +120,13 @@ void	filedesc_init(void);
 int	dupfdopen(struct proc *, int, int);
 int	fdalloc(struct proc *p, int want, int *result);
 void	fdexpand(struct proc *);
-int	falloc(struct proc *_p, int _flags, struct file **_rfp, int *_rfd);
+int	falloc(struct proc *_p, struct file **_rfp, int *_rfd);
 struct	filedesc *fdinit(void);
 struct	filedesc *fdshare(struct process *);
 struct	filedesc *fdcopy(struct process *);
 void	fdfree(struct proc *p);
 int	fdrelease(struct proc *p, int);
+void	fdinsert(struct filedesc *, int, int, struct file *);
 void	fdremove(struct filedesc *, int);
 void	fdcloseexec(struct proc *);
 struct file *fd_iterfile(struct file *, struct proc *);
diff --git a/usr.sbin/pstat/pstat.8 b/usr.sbin/pstat/pstat.8
index 52b59f7df93..7809d4f43c2 100644
--- a/usr.sbin/pstat/pstat.8
+++ b/usr.sbin/pstat/pstat.8
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: pstat.8,v 1.54 2018/06/05 09:29:05 mpi Exp $
+.\"	$OpenBSD: pstat.8,v 1.55 2018/06/18 09:15:05 mpi Exp $
 .\"	$NetBSD: pstat.8,v 1.9.4.1 1996/06/02 09:08:17 mrg Exp $
 .\"
 .\" Copyright (c) 1980, 1991, 1993, 1994
@@ -30,7 +30,7 @@
 .\"
 .\"     from: @(#)pstat.8	8.4 (Berkeley) 4/19/94
 .\"
-.Dd $Mdocdate: June 5 2018 $
+.Dd $Mdocdate: June 18 2018 $
 .Dt PSTAT 8
 .Os
 .Sh NAME
@@ -101,8 +101,6 @@ open for appending
 exclusive or shared lock present
 .It I
 signal pgrp when data ready
-.It l
-file descriptor slot is larval
 .El
 .It CNT
 Number of processes that know this open file.
diff --git a/usr.sbin/pstat/pstat.c b/usr.sbin/pstat/pstat.c
index 207f8123ba8..e0e3f33d61e 100644
--- a/usr.sbin/pstat/pstat.c
+++ b/usr.sbin/pstat/pstat.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pstat.c,v 1.116 2018/06/05 09:29:05 mpi Exp $	*/
+/*	$OpenBSD: pstat.c,v 1.117 2018/06/18 09:15:05 mpi Exp $	*/
 /*	$NetBSD: pstat.c,v 1.27 1996/10/23 22:50:06 cgd Exp $	*/
 
 /*-
@@ -1044,8 +1044,6 @@ filemode(void)
 
 		if (kf->f_iflags & FIF_HASLOCK)
 			*fbp++ = 'L';
-		if (kf->f_iflags & FIF_LARVAL)
-			*fbp++ = 'l';
 
 		*fbp = '\0';
 		(void)printf("%6s  %3ld", flagbuf, (long)kf->f_count);