Add hw.ncpuonline to count the number of online CPUs.

The introduction of hw.smt means that logical CPUs can be disabled
after boot and prior to suspend/resume.  If hw.smt=0 (the default),
there needs to be a way to count the number of hardware threads
available on the system at any given time.

So, import HW_NCPUONLINE/hw.ncpuonline from NetBSD and document it.
hw.ncpu becomes equal to the number of CPUs given to sched_init_cpu()
during boot, while hw.ncpuonline is equal to the number of CPUs available
to the scheduler in the cpuset "sched_all_cpus". Set_SC_NPROCESSORS_ONLN
equal to this new sysctl and keep _SC_NPROCESSORS_CONF equal to hw.ncpu.

This is preferable to adding a new sysctl to count the number of
configured CPUs and keeping hw.ncpu equal to the number of online
CPUs because such a change would break software in the ecosystem
that relies on HW_NCPU/hw.ncpu to measure CPU usage and the like.
Such software in base includes top(1), systat(1), and snmpd(8),
and perhaps others.

We don't need additional locking to count the cardinality of a cpuset
in this case because the only interfaces that can modify said cardinality
are sysctl(2) and ioctl(2), both of which are under the KERNEL_LOCK.

Software using HW_NCPU/hw.ncpu to determine optimal parallism will need
to be updated to use HW_NCPUONLINE/hw.ncpuonline.  Until then, such software
may perform suboptimally.  However, most changes will be similar to the
change included here for libcxx's std::thread:hardware_concurrency():
using HW_NCPUONLINE in lieu of HW_NCPU should be sufficient for determining
optimal parallelism for most software if the change to _SC_NPROCESSORS_ONLN
is insufficient.

Prompted by deraadt. Discussed at length with kettenis, deraadt, and sthen.
Lots of patch tweaks from kettenis.

ok kettenis, "proceed" deraadt

9 files changed, 48 insertions, 18 deletions

diff --git a/lib/libc/gen/sysconf.c b/lib/libc/gen/sysconf.c
index e73bc437dab..1bcc2a7bc90 100644
--- a/lib/libc/gen/sysconf.c
+++ b/lib/libc/gen/sysconf.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sysconf.c,v 1.25 2017/09/10 18:20:00 guenther Exp $ */
+/*	$OpenBSD: sysconf.c,v 1.26 2018/07/12 01:23:38 cheloha Exp $ */
 /*-
  * Copyright (c) 1993
  *	The Regents of the University of California.  All rights reserved.
@@ -461,7 +461,7 @@ sysconf(int name)
 		break;
 	case _SC_NPROCESSORS_ONLN:
 		mib[0] = CTL_HW;
-		mib[1] = HW_NCPU;
+		mib[1] = HW_NCPUONLINE;
 		break;
 
 	default:
diff --git a/lib/libc/sys/sysctl.2 b/lib/libc/sys/sysctl.2
index 6a1de7b0231..b600074fea1 100644
--- a/lib/libc/sys/sysctl.2
+++ b/lib/libc/sys/sysctl.2
@@ -1,4 +1,4 @@
-.\"	$OpenBSD: sysctl.2,v 1.9 2018/06/21 20:28:36 jmc Exp $
+.\"	$OpenBSD: sysctl.2,v 1.10 2018/07/12 01:23:38 cheloha Exp $
 .\"
 .\" Copyright (c) 1993
 .\"	The Regents of the University of California.  All rights reserved.
@@ -27,7 +27,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd $Mdocdate: June 21 2018 $
+.Dd $Mdocdate: July 12 2018 $
 .Dt SYSCTL 2
 .Os
 .Sh NAME
@@ -286,6 +286,7 @@ privileges may change the value.
 .It Dv HW_MODEL Ta "string" Ta "no"
 .It Dv HW_NCPU Ta "integer" Ta "no"
 .It Dv HW_NCPUFOUND Ta "integer" Ta "no"
+.It Dv HW_NCPUONLINE Ta "integer" Ta "no"
 .It Dv HW_PAGESIZE Ta "integer" Ta "no"
 .It Dv HW_PERFPOLICY Ta "string" Ta "yes"
 .It Dv HW_PHYSMEM Ta "integer" Ta "no"
@@ -328,9 +329,11 @@ The machine class.
 .It Dv HW_MODEL Pq Va hw.model
 The machine model.
 .It Dv HW_NCPU Pq Va hw.ncpu
-The number of CPUs being used.
+The number of CPUs configured.
 .It Dv HW_NCPUFOUND Pq Va hw.ncpufound
 The number of CPUs found.
+.It Dv HW_NCPUONLINE Pq Va hw.ncpuonline
+The number of CPUs online.
 .It Dv HW_PAGESIZE Pq Va hw.pagesize
 The software page size.
 .It Dv HW_PERFPOLICY Pq Va hw.perfpolicy
diff --git a/lib/libcxx/src/thread.cpp b/lib/libcxx/src/thread.cpp
index 467402b6b42..d4977cd9a73 100644
--- a/lib/libcxx/src/thread.cpp
+++ b/lib/libcxx/src/thread.cpp
@@ -78,9 +78,9 @@ thread::detach()
 unsigned
 thread::hardware_concurrency() _NOEXCEPT
 {
-#if defined(CTL_HW) && defined(HW_NCPU)
+#if defined(CTL_HW) && defined(HW_NCPUONLINE)
     unsigned n;
-    int mib[2] = {CTL_HW, HW_NCPU};
+    int mib[2] = {CTL_HW, HW_NCPUONLINE};
     std::size_t s = sizeof(n);
     sysctl(mib, 2, &n, &s, 0, 0);
     return n;
diff --git a/sys/kern/kern_pledge.c b/sys/kern/kern_pledge.c
index bcba39b5c69..c5f223e8b1b 100644
--- a/sys/kern/kern_pledge.c
+++ b/sys/kern/kern_pledge.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_pledge.c,v 1.234 2018/06/25 22:29:16 kettenis Exp $	*/
+/*	$OpenBSD: kern_pledge.c,v 1.235 2018/07/12 01:23:38 cheloha Exp $	*/
 
 /*
  * Copyright (c) 2015 Nicholas Marriott <nicm@openbsd.org>
@@ -917,8 +917,8 @@ pledge_sysctl(struct proc *p, int miblen, int *mib, void *new)
 	if (miblen == 2 &&		/* setproctitle() */
 	    mib[0] == CTL_VM && mib[1] == VM_PSSTRINGS)
 		return (0);
-	if (miblen == 2 &&		/* hw.ncpu */
-	    mib[0] == CTL_HW && mib[1] == HW_NCPU)
+	if (miblen == 2 &&		/* hw.ncpu / hw.ncpuonline */
+	    mib[0] == CTL_HW && (mib[1] == HW_NCPU || mib[1] == HW_NCPUONLINE))
 		return (0);
 	if (miblen == 2 &&		/* vm.loadavg / getloadavg(3) */
 	    mib[0] == CTL_VM && mib[1] == VM_LOADAVG)
diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index 4309255ca9d..ce9e80066a3 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sched.c,v 1.50 2018/07/07 15:19:25 visa Exp $	*/
+/*	$OpenBSD: kern_sched.c,v 1.51 2018/07/12 01:23:38 cheloha Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
  *
@@ -812,6 +812,26 @@ cpuset_complement(struct cpuset *to, struct cpuset *a, struct cpuset *b)
 		to->cs_set[i] = b->cs_set[i] & ~a->cs_set[i];
 }
 
+int
+cpuset_cardinality(struct cpuset *cs)
+{
+	int cardinality, i, n;
+
+	cardinality = 0;
+
+	for (i = 0; i < CPUSET_ASIZE(ncpus); i++)
+		for (n = cs->cs_set[i]; n != 0; n &= n - 1)
+			cardinality++;
+
+	return (cardinality);
+}
+
+int
+sysctl_hwncpuonline(void)
+{
+	return cpuset_cardinality(&sched_all_cpus);
+}
+
 #ifdef __HAVE_CPU_TOPOLOGY
 
 #include <sys/sysctl.h>
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index 50e6e6ac6ae..7a565b2de88 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sysctl.c,v 1.345 2018/07/02 14:36:33 visa Exp $	*/
+/*	$OpenBSD: kern_sysctl.c,v 1.346 2018/07/12 01:23:38 cheloha Exp $	*/
 /*	$NetBSD: kern_sysctl.c,v 1.17 1996/05/20 17:49:05 mrg Exp $	*/
 
 /*-
@@ -701,6 +701,9 @@ hw_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp,
 		return (sysctl_rdint(oldp, oldlenp, newp, ncpus));
 	case HW_NCPUFOUND:
 		return (sysctl_rdint(oldp, oldlenp, newp, ncpusfound));
+	case HW_NCPUONLINE:
+		return (sysctl_rdint(oldp, oldlenp, newp,
+		    sysctl_hwncpuonline()));
 	case HW_BYTEORDER:
 		return (sysctl_rdint(oldp, oldlenp, newp, BYTE_ORDER));
 	case HW_PHYSMEM:
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index cb95743ba69..f05d70ee7d5 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: proc.h,v 1.250 2018/07/10 04:19:59 guenther Exp $	*/
+/*	$OpenBSD: proc.h,v 1.251 2018/07/12 01:23:38 cheloha Exp $	*/
 /*	$NetBSD: proc.h,v 1.44 1996/04/22 01:23:21 christos Exp $	*/
 
 /*-
@@ -594,6 +594,7 @@ void cpuset_copy(struct cpuset *, struct cpuset *);
 void cpuset_union(struct cpuset *, struct cpuset *, struct cpuset *);
 void cpuset_intersection(struct cpuset *t, struct cpuset *, struct cpuset *);
 void cpuset_complement(struct cpuset *, struct cpuset *, struct cpuset *);
+int cpuset_cardinality(struct cpuset *);
 struct cpu_info *cpuset_first(struct cpuset *);
 
 #endif	/* _KERNEL */
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 0ce5332db07..8f80d233ff7 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.46 2018/06/19 19:29:52 kettenis Exp $	*/
+/*	$OpenBSD: sched.h,v 1.47 2018/07/12 01:23:38 cheloha Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -153,6 +153,7 @@ void sched_barrier(struct cpu_info *ci);
 int sysctl_hwsetperf(void *, size_t *, void *, size_t);
 int sysctl_hwperfpolicy(void *, size_t *, void *, size_t);
 int sysctl_hwsmt(void *, size_t *, void *, size_t);
+int sysctl_hwncpuonline(void);
 
 #ifdef MULTIPROCESSOR
 void sched_start_secondary_cpus(void);
diff --git a/sys/sys/sysctl.h b/sys/sys/sysctl.h
index 25c35da5cac..7544d630e92 100644
--- a/sys/sys/sysctl.h
+++ b/sys/sys/sysctl.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sysctl.h,v 1.178 2018/06/19 19:29:52 kettenis Exp $	*/
+/*	$OpenBSD: sysctl.h,v 1.179 2018/07/12 01:23:38 cheloha Exp $	*/
 /*	$NetBSD: sysctl.h,v 1.16 1996/04/09 20:55:36 cgd Exp $	*/
 
 /*
@@ -842,7 +842,7 @@ struct kinfo_file {
  */
 #define	HW_MACHINE		 1	/* string: machine class */
 #define	HW_MODEL		 2	/* string: specific machine model */
-#define	HW_NCPU			 3	/* int: number of cpus being used */
+#define	HW_NCPU			 3	/* int: number of configured cpus */
 #define	HW_BYTEORDER		 4	/* int: machine byte order */
 #define	HW_PHYSMEM		 5	/* int: total memory */
 #define	HW_USERMEM		 6	/* int: non-kernel memory */
@@ -860,11 +860,12 @@ struct kinfo_file {
 #define	HW_UUID			18	/* string: universal unique id */
 #define	HW_PHYSMEM64		19	/* quad: total memory */
 #define	HW_USERMEM64		20	/* quad: non-kernel memory */
-#define	HW_NCPUFOUND		21	/* int: number of cpus found*/
+#define	HW_NCPUFOUND		21	/* int: number of cpus found */
 #define	HW_ALLOWPOWERDOWN	22	/* allow power button shutdown */
 #define	HW_PERFPOLICY		23	/* set performance policy */
 #define	HW_SMT			24	/* int: enable SMT/HT/CMT */
-#define	HW_MAXID		25	/* number of valid hw ids */
+#define	HW_NCPUONLINE		25	/* int: number of cpus being used */
+#define	HW_MAXID		26	/* number of valid hw ids */
 
 #define	CTL_HW_NAMES { \
 	{ 0, 0 }, \
@@ -892,6 +893,7 @@ struct kinfo_file {
 	{ "allowpowerdown", CTLTYPE_INT }, \
 	{ "perfpolicy", CTLTYPE_STRING }, \
 	{ "smt", CTLTYPE_INT }, \
+	{ "ncpuonline", CTLTYPE_INT }, \
 }
 
 /*