KERN_CPTIME2: set ENODEV if the CPU is offline.

This lets userspace distinguish between idle CPUs and those that are
not schedulable because hw.smt=0.

A subsequent commit probably needs to add documentation for this
to sysctl.2 (and perhaps elsewhere) after the dust settles.

Also included here are changes to systat(1) and top(1) that account
for the ENODEV case and adjust behavior accordingly:

 - systat(1)'s cpu view prints placeholder marks ('-') instead of
   percentages for each state if the given CPU is offline.

 - systat(1)'s vmstat view checks for offline CPUs when computing the
   machine state total and excludes them, so the CPU usage graph
   only represents the states for online CPUs.

 - top(1) does not draw CPU rows for offline CPUs when the view is
   redrawn.  If CPUs "go offline", percentages for each state are
   replaced by placeholder marks ('-'); the view will need to be
   redrawn to remove these rows.  If CPUs "go online" the view will
   need to be redrawn to show these new CPUs.  In "combined CPU" mode,
   the count and the state totals only represent online CPUs.

Ports using KERN_CPTIME2 will need to be updated.  The changes
described above to make systat(1) and top(1) aware of the ENODEV
case *and* gracefully handle a changing HW_NCPUONLINE while the
application is running are not necessarily appropriate for each
and every port.

The changes described above are so extensive in part to demonstrate
one way a program *might* be made robust to changing CPU availability.
In particular, changing hw.smt after boot is an extremely rare event,
and this needs to be weighed when updating ports.

The logic needed to account for the KERN_CPTIME2 ENODEV case is
very roughly:

	if (sysctl(...) == -1) {
		if (errno != ENODEV) {
			/* Actual error occurred. */
		} else {
			/* CPU is offline. */
		}
	} else {
		/* CPU is online and CPU states were set by sysctl(2). */
	}

Prompted by deraadt@.  Basic idea for ENODEV from kettenis@.  Discussed at
length with kettenis@.  Additional testing by tb@.

No complaints from hackers@ after a week.

ok kettenis@, "I think you should commit [now]" deraadt@

10 files changed, 139 insertions, 52 deletions

diff --git a/sys/kern/kern_sched.c b/sys/kern/kern_sched.c
index ce9e80066a3..20ae05d0001 100644
--- a/sys/kern/kern_sched.c
+++ b/sys/kern/kern_sched.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sched.c,v 1.51 2018/07/12 01:23:38 cheloha Exp $	*/
+/*	$OpenBSD: kern_sched.c,v 1.52 2018/09/26 17:23:13 cheloha Exp $	*/
 /*
  * Copyright (c) 2007, 2008 Artur Grabowski <art@openbsd.org>
  *
@@ -832,6 +832,12 @@ sysctl_hwncpuonline(void)
 	return cpuset_cardinality(&sched_all_cpus);
 }
 
+int
+cpu_is_online(struct cpu_info *ci)
+{
+	return cpuset_isset(&sched_all_cpus, ci);
+}
+
 #ifdef __HAVE_CPU_TOPOLOGY
 
 #include <sys/sysctl.h>
diff --git a/sys/kern/kern_sysctl.c b/sys/kern/kern_sysctl.c
index f83cb70a9a6..8b0da7c6475 100644
--- a/sys/kern/kern_sysctl.c
+++ b/sys/kern/kern_sysctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: kern_sysctl.c,v 1.347 2018/09/20 18:59:10 bluhm Exp $	*/
+/*	$OpenBSD: kern_sysctl.c,v 1.348 2018/09/26 17:23:13 cheloha Exp $	*/
 /*	$NetBSD: kern_sysctl.c,v 1.17 1996/05/20 17:49:05 mrg Exp $	*/
 
 /*-
@@ -2397,6 +2397,8 @@ sysctl_cptime2(int *name, u_int namelen, void *oldp, size_t *oldlenp,
 	}
 	if (!found)
 		return (ENOENT);
+	if (!cpu_is_online(ci))
+		return (ENODEV);
 
 	return (sysctl_rdstruct(oldp, oldlenp, newp,
 	    &ci->ci_schedstate.spc_cp_time,
diff --git a/sys/sys/sched.h b/sys/sys/sched.h
index 8f80d233ff7..f8f6d7b7af4 100644
--- a/sys/sys/sched.h
+++ b/sys/sys/sched.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sched.h,v 1.47 2018/07/12 01:23:38 cheloha Exp $	*/
+/*	$OpenBSD: sched.h,v 1.48 2018/09/26 17:23:13 cheloha Exp $	*/
 /* $NetBSD: sched.h,v 1.2 1999/02/28 18:14:58 ross Exp $ */
 
 /*-
@@ -155,6 +155,8 @@ int sysctl_hwperfpolicy(void *, size_t *, void *, size_t);
 int sysctl_hwsmt(void *, size_t *, void *, size_t);
 int sysctl_hwncpuonline(void);
 
+int cpu_is_online(struct cpu_info *);
+
 #ifdef MULTIPROCESSOR
 void sched_start_secondary_cpus(void);
 void sched_stop_secondary_cpus(void);
diff --git a/usr.bin/systat/cpu.c b/usr.bin/systat/cpu.c
index ac7e2ba000e..22f525fd626 100644
--- a/usr.bin/systat/cpu.c
+++ b/usr.bin/systat/cpu.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: cpu.c,v 1.6 2018/05/14 12:31:21 mpi Exp $	*/
+/*	$OpenBSD: cpu.c,v 1.7 2018/09/26 17:23:13 cheloha Exp $	*/
 
 /*
  * Copyright (c) 2013 Reyk Floeter <reyk@openbsd.org>
@@ -50,6 +50,7 @@
 #include <sys/sched.h>
 #include <sys/sysctl.h>
 
+#include <errno.h>
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>
@@ -99,6 +100,7 @@ field_view views_cpu[] = {
 };
 
 int	  cpu_count;
+int	 *cpu_online;
 int64_t	 *cpu_states;
 int64_t	**cpu_tm;
 int64_t	**cpu_old;
@@ -156,8 +158,13 @@ cpu_info(void)
 	for (i = 0; i < cpu_count; i++) {
 		cpu_time_mib[2] = i;
 		tmpstate = cpu_states + (CPUSTATES * i);
-		if (sysctl(cpu_time_mib, 3, cpu_tm[i], &size, NULL, 0) < 0)
-			error("sysctl KERN_CPTIME2");
+		if (sysctl(cpu_time_mib, 3, cpu_tm[i], &size, NULL, 0) < 0) {
+			if (errno != ENODEV)
+				error("sysctl KERN_CPTIME2");
+			cpu_online[i] = 0;
+			continue;
+		}
+		cpu_online[i] = 1;
 		percentages(CPUSTATES, tmpstate, cpu_tm[i],
 		    cpu_old[i], cpu_diff[i]);
 	}
@@ -200,6 +207,8 @@ initcpu(void)
 	mib[1] = HW_NCPU;
 	if (sysctl(mib, 2, &cpu_count, &size, NULL, 0) == -1)
 		return (-1);
+	if ((cpu_online = calloc(cpu_count, sizeof(*cpu_online))) == NULL)
+		return (-1);
 	if ((cpu_states = calloc(cpu_count,
 	    CPUSTATES * sizeof(int64_t))) == NULL)
 		return (-1);
@@ -246,6 +255,21 @@ initcpu(void)
 			return;						\
 	} while (0)
 
+#define ADD_OFFLINE_CPU(v) do {						\
+	if (cur >= dispstart && cur < end) { 				\
+		print_fld_size(FLD_CPU_CPU, (v));			\
+		print_fld_str(FLD_CPU_USR, "-");			\
+		print_fld_str(FLD_CPU_NIC, "-");			\
+		print_fld_str(FLD_CPU_SYS, "-");			\
+		print_fld_str(FLD_CPU_SPIN, "-");			\
+		print_fld_str(FLD_CPU_INT, "-");			\
+		print_fld_str(FLD_CPU_IDLE, "-");			\
+		end_line();						\
+	}								\
+	if (++cur >= end)						\
+		return;							\
+} while (0)
+
 void
 print_cpu(void)
 {
@@ -258,6 +282,10 @@ print_cpu(void)
 		end = num_disp;
 
 	for (c = 0; c < cpu_count; c++) {
+		if (!cpu_online[c]) {
+			ADD_OFFLINE_CPU(c);
+			continue;
+		}
 		states = cpu_states + (CPUSTATES * c);
 
 		for (i = 0; i < CPUSTATES; i++)
diff --git a/usr.bin/systat/vmstat.c b/usr.bin/systat/vmstat.c
index 53922d90196..b59e43941fc 100644
--- a/usr.bin/systat/vmstat.c
+++ b/usr.bin/systat/vmstat.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: vmstat.c,v 1.86 2018/06/22 14:22:06 krw Exp $	*/
+/*	$OpenBSD: vmstat.c,v 1.87 2018/09/26 17:23:13 cheloha Exp $	*/
 /*	$NetBSD: vmstat.c,v 1.5 1996/05/10 23:16:40 thorpej Exp $	*/
 
 /*-
@@ -98,6 +98,7 @@ static	time_t t;
 static	double etime;
 static	float hertz;
 static	int nintr;
+static	int ncpu;
 static	long *intrloc;
 static	char **intrname;
 static	int ipktsrow;
@@ -200,6 +201,12 @@ initvmstat(void)
 			return (-1);
 	}
 
+	mib[0] = CTL_HW;
+	mib[1] = HW_NCPU;
+	size = sizeof(ncpu);
+	if (sysctl(mib, 2, &ncpu, &size, NULL, 0) < 0)
+		return (-1);
+
 	allocinfo(&s);
 	allocinfo(&s1);
 	allocinfo(&s2);
@@ -592,11 +599,12 @@ putfloat(double f, int l, int c, int w, int d, int nz)
 static void
 getinfo(struct Info *si)
 {
-	static int cp_time_mib[] = { CTL_KERN, KERN_CPTIME };
+	static int cp_time2_mib[3] = { CTL_KERN, KERN_CPTIME2, 0 };
 	static int nchstats_mib[2] = { CTL_KERN, KERN_NCHSTATS };
 	static int uvmexp_mib[2] = { CTL_VM, VM_UVMEXP };
 	static int vmtotal_mib[2] = { CTL_VM, VM_METER };
-	int mib[4], i;
+	int mib[4], cpu, i;
+	long cpu_time[CPUSTATES];
 	size_t size;
 
 	dkreadstats();
@@ -612,10 +620,21 @@ getinfo(struct Info *si)
 		}
 	}
 
-	size = sizeof(si->time);
-	if (sysctl(cp_time_mib, 2, &si->time, &size, NULL, 0) < 0) {
-		error("Can't get KERN_CPTIME: %s\n", strerror(errno));
-		memset(&si->time, 0, sizeof(si->time));
+	memset(&si->time, 0, sizeof(si->time));
+	for (cpu = 0; cpu < ncpu; cpu++) {
+		cp_time2_mib[2] = cpu;
+		size = sizeof(cpu_time);
+		if (sysctl(cp_time2_mib, 3, &cpu_time, &size, NULL, 0) < 0) {
+			if (errno != ENODEV) {
+				error("Can't get KERN_CPTIME2: %s\n",
+				    strerror(errno));
+				memset(&si->time, 0, sizeof(si->time));
+				break;
+			}
+			continue;	/* ignore offline CPUs */
+		}
+		for (i = 0; i < nitems(si->time); i++)
+			si->time[i] += cpu_time[i];
 	}
 
 	size = sizeof(si->nchstats);
diff --git a/usr.bin/top/display.c b/usr.bin/top/display.c
index c9bd7064a75..02e348f2e7b 100644
--- a/usr.bin/top/display.c
+++ b/usr.bin/top/display.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: display.c,v 1.54 2018/01/04 17:44:20 deraadt Exp $	 */
+/* $OpenBSD: display.c,v 1.55 2018/09/26 17:23:13 cheloha Exp $	 */
 
 /*
  *  Top users/processes display for Unix
@@ -125,8 +125,10 @@ static int (*standendp)(void);
 int
 display_resize(void)
 {
-	int display_lines;
-	int cpu_lines = (combine_cpus ? 1 : ncpu);
+	int cpu_lines, display_lines, ncpuonline;
+
+	ncpuonline = getncpuonline();
+	cpu_lines = (combine_cpus ? 1 : ncpuonline);
 
 	y_mem = 2 + cpu_lines;
 	y_header = 4 + cpu_lines;
@@ -136,7 +138,7 @@ display_resize(void)
 	/* if operating in "dumb" mode, we only need one line */
 	display_lines = smart_terminal ? screen_length - y_procs : 1;
 
-	y_idlecursor = y_message = 3 + (combine_cpus ? 1 : ncpu);
+	y_idlecursor = y_message = 3 + cpu_lines;
 	if (screen_length <= y_message)
 		y_idlecursor = y_message = screen_length - 1;
 
@@ -377,13 +379,19 @@ cpustates_tag(int cpu)
 }
 
 void
-i_cpustates(int64_t *ostates)
+i_cpustates(int64_t *ostates, int *online)
 {
-	int i, first, cpu;
+	int i, first, cpu, ncpuonline;
 	double value;
 	int64_t *states;
 	char **names, *thisname;
 
+	ncpuonline = 0;
+	for (i = 0; i < ncpu; i++) {
+		if (online[i])
+			ncpuonline++;
+	}
+
 	if (combine_cpus) {
 		static double *values;
 		if (!values) {
@@ -393,6 +401,8 @@ i_cpustates(int64_t *ostates)
 		}
 		memset(values, 0, num_cpustates * sizeof(*values));
 		for (cpu = 0; cpu < ncpu; cpu++) {
+			if (!online[cpu])
+				continue;
 			names = cpustate_names;
 			states = ostates + (CPUSTATES * cpu);
 			i = 0;
@@ -409,11 +419,11 @@ i_cpustates(int64_t *ostates)
 			first = 0;
 			move(2, 0);
 			clrtoeol();
-			printwp("%-3d CPUs: ", ncpu);
+			printwp("%-3d CPUs: ", ncpuonline);
 
 			while ((thisname = *names++) != NULL) {
 				if (*thisname != '\0') {
-					value = values[i++] / ncpu;
+					value = values[i++] / ncpuonline;
 					/* if percentage is >= 1000, print it as 100% */
 					printwp((value >= 1000 ? "%s%4.0f%% %s" :
 					    "%s%4.1f%% %s"), first++ == 0 ? "" : ", ",
@@ -430,13 +440,20 @@ i_cpustates(int64_t *ostates)
 		first = 0;
 		states = ostates + (CPUSTATES * cpu);
 
-		if (screen_length > 2 + cpu || !smart_terminal) {
+		if ((screen_length > 2 + cpu && 2 + cpu < y_mem) ||
+		    !smart_terminal) {
 			move(2 + cpu, 0);
 			clrtoeol();
 			addstrp(cpustates_tag(cpu));
 
 			while ((thisname = *names++) != NULL) {
 				if (*thisname != '\0') {
+					if (!online[cpu]) {
+						printwp("%s%5s %s",
+						    first++ == 0 ? "" : ", ",
+						    "-", thisname);
+						continue;
+					}
 					/* retrieve the value and remember it */
 					value = *states++;
 
diff --git a/usr.bin/top/display.h b/usr.bin/top/display.h
index bd4711c5bd2..5e170ef350a 100644
--- a/usr.bin/top/display.h
+++ b/usr.bin/top/display.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: display.h,v 1.12 2013/01/14 21:33:59 guenther Exp $	*/
+/*	$OpenBSD: display.h,v 1.13 2018/09/26 17:23:13 cheloha Exp $	*/
 
 /*
  *  Top users/processes display for Unix
@@ -40,7 +40,7 @@ void u_loadave(int, double *);
 void i_timeofday(time_t *);
 void i_procstates(int, int *, int);
 void u_procstates(int, int *);
-void i_cpustates(int64_t *);
+void i_cpustates(int64_t *, int *);
 void u_cpustates(int64_t *);
 void i_memory(int *);
 void u_memory(int *);
diff --git a/usr.bin/top/machine.c b/usr.bin/top/machine.c
index 10cf0b4a2bb..04c1d1b0443 100644
--- a/usr.bin/top/machine.c
+++ b/usr.bin/top/machine.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: machine.c,v 1.92 2018/09/22 16:50:35 millert Exp $	 */
+/* $OpenBSD: machine.c,v 1.93 2018/09/26 17:23:13 cheloha Exp $	 */
 
 /*-
  * Copyright (c) 1994 Thorsten Lockert <tholo@sigmasoft.com>
@@ -111,6 +111,9 @@ char *cpustatenames[] = {
 	"user", "nice", "sys", "spin", "intr", "idle", NULL
 };
 
+/* this tracks which cpus are online */
+int *cpu_online;
+
 /* these are for detailing the memory statistics */
 int memory_stats[10];
 char *memorynames[] = {
@@ -170,6 +173,20 @@ getncpu(void)
 }
 
 int
+getncpuonline(void)
+{
+	int mib[] = { CTL_HW, HW_NCPUONLINE };
+	int numcpu;
+	size_t size = sizeof(numcpu);
+
+	if (sysctl(mib, sizeof(mib) / sizeof(mib[0]),
+	    &numcpu, &size, NULL, 0) == -1)
+		return (-1);
+
+	return (numcpu);
+}
+
+int
 machine_init(struct statics *statics)
 {
 	int pagesize, cpu;
@@ -182,6 +199,9 @@ machine_init(struct statics *statics)
 	cpu_states = calloc(ncpu, CPUSTATES * sizeof(int64_t));
 	if (cpu_states == NULL)
 		err(1, NULL);
+	cpu_online = calloc(ncpu, sizeof(*cpu_online));
+	if (cpu_online == NULL)
+		err(1, NULL);
 	cp_time = calloc(ncpu, sizeof(int64_t *));
 	cp_old  = calloc(ncpu, sizeof(int64_t *));
 	cp_diff = calloc(ncpu, sizeof(int64_t *));
@@ -243,6 +263,7 @@ format_header(char *second_field, int show_threads)
 void
 get_system_info(struct system_info *si)
 {
+	static int cp_time_mib[] = {CTL_KERN, KERN_CPTIME2, /*fillme*/0};
 	static int sysload_mib[] = {CTL_VM, VM_LOADAVG};
 	static int uvmexp_mib[] = {CTL_VM, VM_UVMEXP};
 	static int bcstats_mib[] = {CTL_VFS, VFS_GENERIC, VFS_BCACHESTAT};
@@ -254,31 +275,20 @@ get_system_info(struct system_info *si)
 	int i;
 	int64_t *tmpstate;
 
-	if (ncpu > 1) {
-		int cp_time_mib[] = {CTL_KERN, KERN_CPTIME2, /*fillme*/0};
-
-		size = CPUSTATES * sizeof(int64_t);
-		for (i = 0; i < ncpu; i++) {
-			cp_time_mib[2] = i;
-			tmpstate = cpu_states + (CPUSTATES * i);
-			if (sysctl(cp_time_mib, 3, cp_time[i], &size, NULL, 0) < 0)
+	size = CPUSTATES * sizeof(int64_t);
+	for (i = 0; i < ncpu; i++) {
+		cp_time_mib[2] = i;
+		tmpstate = cpu_states + (CPUSTATES * i);
+		if (sysctl(cp_time_mib, 3, cp_time[i], &size, NULL, 0) < 0) {
+			if (errno != ENODEV)
 				warn("sysctl kern.cp_time2 failed");
-			/* convert cp_time2 counts to percentages */
-			(void) percentages(CPUSTATES, tmpstate, cp_time[i],
-			    cp_old[i], cp_diff[i]);
+			cpu_online[i] = 0;
+			continue;
 		}
-	} else {
-		int cp_time_mib[] = {CTL_KERN, KERN_CPTIME};
-		long cp_time_tmp[CPUSTATES];
-
-		size = sizeof(cp_time_tmp);
-		if (sysctl(cp_time_mib, 2, cp_time_tmp, &size, NULL, 0) < 0)
-			warn("sysctl kern.cp_time failed");
-		for (i = 0; i < CPUSTATES; i++)
-			cp_time[0][i] = cp_time_tmp[i];
-		/* convert cp_time counts to percentages */
-		(void) percentages(CPUSTATES, cpu_states, cp_time[0],
-		    cp_old[0], cp_diff[0]);
+		cpu_online[i] = 1;
+		/* convert cp_time2 counts to percentages */
+		(void) percentages(CPUSTATES, tmpstate, cp_time[i],
+		    cp_old[i], cp_diff[i]);
 	}
 
 	size = sizeof(sysload);
@@ -317,6 +327,7 @@ get_system_info(struct system_info *si)
 
 	/* set arrays and strings */
 	si->cpustates = cpu_states;
+	si->cpuonline = cpu_online;
 	si->memory = memory_stats;
 	si->last_pid = -1;
 }
diff --git a/usr.bin/top/machine.h b/usr.bin/top/machine.h
index 55a6b8572c2..f4ce8c88581 100644
--- a/usr.bin/top/machine.h
+++ b/usr.bin/top/machine.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: machine.h,v 1.22 2018/09/22 16:50:35 millert Exp $	 */
+/* $OpenBSD: machine.h,v 1.23 2018/09/26 17:23:13 cheloha Exp $	 */
 
 /*
  *  Top users/processes display for Unix
@@ -56,6 +56,7 @@ struct system_info {
 					 * "active" */
 	int            *procstates;
 	int64_t        *cpustates;
+	int            *cpuonline;
 	int            *memory;
 };
 
@@ -95,4 +96,5 @@ extern uid_t    proc_owner(pid_t);
 extern struct kinfo_proc	*getprocs(int, int, int *);
 
 int		getncpu(void);
+int		getncpuonline(void);
 int		getfscale(void);
diff --git a/usr.bin/top/top.c b/usr.bin/top/top.c
index 3a45cdb44b6..2014f13ea01 100644
--- a/usr.bin/top/top.c
+++ b/usr.bin/top/top.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: top.c,v 1.92 2018/09/22 16:50:35 millert Exp $	*/
+/*	$OpenBSD: top.c,v 1.93 2018/09/26 17:23:13 cheloha Exp $	*/
 
 /*
  *  Top users/processes display for Unix
@@ -251,7 +251,7 @@ parseargs(int ac, char **av)
 		}
 	}
 
-	i = getncpu();
+	i = getncpuonline();
 	if (i == -1)
 		err(1, NULL);
 
@@ -467,7 +467,7 @@ restart:
 		    ps.threads);
 
 		/* display the cpu state percentage breakdown */
-		i_cpustates(system_info.cpustates);
+		i_cpustates(system_info.cpustates, system_info.cpuonline);
 
 		/* display memory stats */
 		i_memory(system_info.memory);