Change pthread_cleanup_{push,pop} to macros that store the cleanup info

on the stack instead of mallocing the list and move the APIs from libpthread
to libc so that they can be used inside libc.

Note: the standard was explicitly written to permit/support this
"macro with unmatched brace" style and it's what basically everyone
else already does.  We xor the info with random cookies with a
random magic to detect/trip-up overwrites.

Major bump to both libc and libpthread due to the API move.

ok mpi@

7 files changed, 98 insertions, 19 deletions

diff --git a/lib/libc/Symbols.list b/lib/libc/Symbols.list
index 56545aef0ac..f595eebc6db 100644
--- a/lib/libc/Symbols.list
+++ b/lib/libc/Symbols.list
@@ -1664,6 +1664,8 @@ _spinlock
 _spinlocktry
 _spinunlock
 _thread_atfork
+_thread_cleanup_pop
+_thread_cleanup_push
 _thread_dofork
 _thread_set_callbacks
 pthread_atfork
diff --git a/lib/libc/hidden/pthread.h b/lib/libc/hidden/pthread.h
index 534020101c5..5991bc4cfee 100644
--- a/lib/libc/hidden/pthread.h
+++ b/lib/libc/hidden/pthread.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pthread.h,v 1.2 2017/09/05 02:40:54 guenther Exp $	*/
+/*	$OpenBSD: pthread.h,v 1.3 2017/10/28 21:23:14 guenther Exp $	*/
 /*
  * Copyright (c) 2015 Philip Guenther <guenther@openbsd.org>
  *
@@ -20,6 +20,8 @@
 
 #include_next <pthread.h>
 
+PROTO_NORMAL(_thread_cleanup_pop);
+PROTO_NORMAL(_thread_cleanup_push);
 PROTO_STD_DEPRECATED(pthread_atfork);
 PROTO_STD_DEPRECATED(pthread_cond_broadcast);
 PROTO_STD_DEPRECATED(pthread_cond_destroy);
diff --git a/lib/libc/include/thread_private.h b/lib/libc/include/thread_private.h
index 23faa73faaf..6c695ee5ee8 100644
--- a/lib/libc/include/thread_private.h
+++ b/lib/libc/include/thread_private.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: thread_private.h,v 1.30 2017/09/05 02:40:54 guenther Exp $ */
+/* $OpenBSD: thread_private.h,v 1.31 2017/10/28 21:23:14 guenther Exp $ */
 
 /* PUBLIC DOMAIN: No Rights Reserved. Marco S Hyman <marc@snafu.org> */
 
@@ -344,12 +344,6 @@ struct rthread_storage {
 	void *data;
 };
 
-struct rthread_cleanup_fn {
-	void (*fn)(void *);
-	void *arg;
-	struct rthread_cleanup_fn *next;
-};
-
 struct tib;
 struct stack;
 struct pthread {
@@ -367,7 +361,7 @@ struct pthread {
 	pthread_cond_t blocking_cond;
 	struct pthread_attr attr;
 	struct rthread_storage *local_storage;
-	struct rthread_cleanup_fn *cleanup_fns;
+	struct __thread_cleanup *cleanup_fns;
 
 	/* cancel received in a delayed cancel block? */
 	int delayed_cancel;
diff --git a/lib/libc/shlib_version b/lib/libc/shlib_version
index c15952cf991..589abf589f1 100644
--- a/lib/libc/shlib_version
+++ b/lib/libc/shlib_version
@@ -1,4 +1,4 @@
-major=90
+major=91
 minor=0
 # note: If changes were made to include/thread_private.h or if system
 # calls were added/changed then librthread/shlib_version also be updated.
diff --git a/lib/libc/thread/Makefile.inc b/lib/libc/thread/Makefile.inc
index b409f578e3e..9eb1622f883 100644
--- a/lib/libc/thread/Makefile.inc
+++ b/lib/libc/thread/Makefile.inc
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile.inc,v 1.11 2017/09/05 02:40:54 guenther Exp $
+#	$OpenBSD: Makefile.inc,v 1.12 2017/10/28 21:23:14 guenther Exp $
 
 .PATH: ${LIBCSRCDIR}/thread
 
@@ -6,6 +6,7 @@ SRCS+=	callbacks.c atfork.c
 
 # threads infrastructure
 SRCS+=	rthread.c \
+	rthread_cleanup.c \
 	rthread_condattr.c \
 	rthread_debug.c \
 	rthread_file.c \
diff --git a/lib/libc/thread/rthread.c b/lib/libc/thread/rthread.c
index 0f19338b086..a92d3a20328 100644
--- a/lib/libc/thread/rthread.c
+++ b/lib/libc/thread/rthread.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: rthread.c,v 1.4 2017/09/05 02:40:54 guenther Exp $ */
+/*	$OpenBSD: rthread.c,v 1.5 2017/10/28 21:23:14 guenther Exp $ */
 /*
  * Copyright (c) 2004,2005 Ted Unangst <tedu@openbsd.org>
  * All Rights Reserved.
@@ -112,7 +112,7 @@ DEF_STRONG(pthread_self);
 void
 pthread_exit(void *retval)
 {
-	struct rthread_cleanup_fn *clfn;
+	struct __thread_cleanup  *clfn;
 	struct tib *tib;
 	pthread_t thread = pthread_self();
 
@@ -131,12 +131,8 @@ pthread_exit(void *retval)
 
 	thread->retval = retval;
 
-	for (clfn = thread->cleanup_fns; clfn; ) {
-		struct rthread_cleanup_fn *oclfn = clfn;
-		clfn = clfn->next;
-		oclfn->fn(oclfn->arg);
-		free(oclfn);
-	}
+	while ((clfn = thread->cleanup_fns) != NULL)
+		_thread_cleanup_pop(1, clfn);
 	_rthread_tls_destructors(thread);
 
 	if (_thread_cb.tc_thread_release != NULL)
diff --git a/lib/libc/thread/rthread_cleanup.c b/lib/libc/thread/rthread_cleanup.c
new file mode 100644
index 00000000000..4accd64494d
--- /dev/null
+++ b/lib/libc/thread/rthread_cleanup.c
@@ -0,0 +1,84 @@
+/*	$OpenBSD: rthread_cleanup.c,v 1.1 2017/10/28 21:23:14 guenther Exp $ */
+/*
+ * Copyright (c) 2017 Philip Guenther <guenther@openbsd.org>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <syslog.h>
+
+#include "thread_private.h"
+
+/* If this is static then clang will optimize it away */
+uintptr_t cleanup_cookie[4] __dso_hidden
+    __attribute__((section(".openbsd.randomdata")));
+
+void
+_thread_cleanup_push(void (*fn)(void *), void *arg,
+    struct __thread_cleanup *tc)
+{
+	pthread_t self = pthread_self();
+
+	tc->__tc_magic	= cleanup_cookie[0];
+	tc->__tc_next	= cleanup_cookie[1] ^ (uintptr_t)self->cleanup_fns;
+	tc->__tc_fn	= cleanup_cookie[2] ^ (uintptr_t)fn;
+	tc->__tc_arg	= cleanup_cookie[3] ^ (uintptr_t)arg;
+	self->cleanup_fns = tc;
+}
+DEF_STRONG(_thread_cleanup_push);
+
+static const char bad_magic[] = ": corrupted buffer in pthread_cleanup_pop";
+static const char bad_pop[] = ": non-LIFO pthread_cleanup_pop";
+
+void
+_thread_cleanup_pop(int execute, struct __thread_cleanup *tc)
+{
+	pthread_t self = pthread_self();
+
+	if (__predict_false(tc->__tc_magic != cleanup_cookie[0] ||
+	    tc != self->cleanup_fns)) {
+		char buf[1024];
+		const char *msg;
+		size_t msgsiz;
+
+		/* <10> is LOG_CRIT */
+		strlcpy(buf, "<10>", sizeof buf);
+
+		/* Make sure progname does not fill the whole buffer */
+		if (tc->__tc_magic != cleanup_cookie[0]) {
+			msgsiz = sizeof bad_magic;
+			msg = bad_magic;
+		} else {
+			msgsiz = sizeof bad_pop;
+			msg = bad_pop;
+		}
+		strlcat(buf, __progname, sizeof(buf) - msgsiz);
+		strlcat(buf, msg, sizeof buf);
+
+		sendsyslog(buf, strlen(buf), LOG_CONS);
+		abort();
+	}
+
+	self->cleanup_fns = (void *)(cleanup_cookie[1] ^ tc->__tc_next);
+	if (execute) {
+		void (*fn)(void *), *arg;
+
+		fn  = (void *)(cleanup_cookie[2] ^ tc->__tc_fn);
+		arg = (void *)(cleanup_cookie[3] ^ tc->__tc_arg);
+		fn(arg);
+	}
+}
+DEF_STRONG(_thread_cleanup_pop);