KERN_CPTIME2: set ENODEV if the CPU is offline.

This lets userspace distinguish between idle CPUs and those that are
not schedulable because hw.smt=0.

A subsequent commit probably needs to add documentation for this
to sysctl.2 (and perhaps elsewhere) after the dust settles.

Also included here are changes to systat(1) and top(1) that account
for the ENODEV case and adjust behavior accordingly:

 - systat(1)'s cpu view prints placeholder marks ('-') instead of
   percentages for each state if the given CPU is offline.

 - systat(1)'s vmstat view checks for offline CPUs when computing the
   machine state total and excludes them, so the CPU usage graph
   only represents the states for online CPUs.

 - top(1) does not draw CPU rows for offline CPUs when the view is
   redrawn.  If CPUs "go offline", percentages for each state are
   replaced by placeholder marks ('-'); the view will need to be
   redrawn to remove these rows.  If CPUs "go online" the view will
   need to be redrawn to show these new CPUs.  In "combined CPU" mode,
   the count and the state totals only represent online CPUs.

Ports using KERN_CPTIME2 will need to be updated.  The changes
described above to make systat(1) and top(1) aware of the ENODEV
case *and* gracefully handle a changing HW_NCPUONLINE while the
application is running are not necessarily appropriate for each
and every port.

The changes described above are so extensive in part to demonstrate
one way a program *might* be made robust to changing CPU availability.
In particular, changing hw.smt after boot is an extremely rare event,
and this needs to be weighed when updating ports.

The logic needed to account for the KERN_CPTIME2 ENODEV case is
very roughly:

	if (sysctl(...) == -1) {
		if (errno != ENODEV) {
			/* Actual error occurred. */
		} else {
			/* CPU is offline. */
		}
	} else {
		/* CPU is online and CPU states were set by sysctl(2). */
	}

Prompted by deraadt@.  Basic idea for ENODEV from kettenis@.  Discussed at
length with kettenis@.  Additional testing by tb@.

No complaints from hackers@ after a week.

ok kettenis@, "I think you should commit [now]" deraadt@

1 files changed, 35 insertions, 24 deletions

diff --git a/usr.bin/top/machine.c b/usr.bin/top/machine.c
index 10cf0b4a2bb..04c1d1b0443 100644
--- a/usr.bin/top/machine.c
+++ b/usr.bin/top/machine.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: machine.c,v 1.92 2018/09/22 16:50:35 millert Exp $	 */
+/* $OpenBSD: machine.c,v 1.93 2018/09/26 17:23:13 cheloha Exp $	 */
 
 /*-
  * Copyright (c) 1994 Thorsten Lockert <tholo@sigmasoft.com>
@@ -111,6 +111,9 @@ char *cpustatenames[] = {
 	"user", "nice", "sys", "spin", "intr", "idle", NULL
 };
 
+/* this tracks which cpus are online */
+int *cpu_online;
+
 /* these are for detailing the memory statistics */
 int memory_stats[10];
 char *memorynames[] = {
@@ -170,6 +173,20 @@ getncpu(void)
 }
 
 int
+getncpuonline(void)
+{
+	int mib[] = { CTL_HW, HW_NCPUONLINE };
+	int numcpu;
+	size_t size = sizeof(numcpu);
+
+	if (sysctl(mib, sizeof(mib) / sizeof(mib[0]),
+	    &numcpu, &size, NULL, 0) == -1)
+		return (-1);
+
+	return (numcpu);
+}
+
+int
 machine_init(struct statics *statics)
 {
 	int pagesize, cpu;
@@ -182,6 +199,9 @@ machine_init(struct statics *statics)
 	cpu_states = calloc(ncpu, CPUSTATES * sizeof(int64_t));
 	if (cpu_states == NULL)
 		err(1, NULL);
+	cpu_online = calloc(ncpu, sizeof(*cpu_online));
+	if (cpu_online == NULL)
+		err(1, NULL);
 	cp_time = calloc(ncpu, sizeof(int64_t *));
 	cp_old  = calloc(ncpu, sizeof(int64_t *));
 	cp_diff = calloc(ncpu, sizeof(int64_t *));
@@ -243,6 +263,7 @@ format_header(char *second_field, int show_threads)
 void
 get_system_info(struct system_info *si)
 {
+	static int cp_time_mib[] = {CTL_KERN, KERN_CPTIME2, /*fillme*/0};
 	static int sysload_mib[] = {CTL_VM, VM_LOADAVG};
 	static int uvmexp_mib[] = {CTL_VM, VM_UVMEXP};
 	static int bcstats_mib[] = {CTL_VFS, VFS_GENERIC, VFS_BCACHESTAT};
@@ -254,31 +275,20 @@ get_system_info(struct system_info *si)
 	int i;
 	int64_t *tmpstate;
 
-	if (ncpu > 1) {
-		int cp_time_mib[] = {CTL_KERN, KERN_CPTIME2, /*fillme*/0};
-
-		size = CPUSTATES * sizeof(int64_t);
-		for (i = 0; i < ncpu; i++) {
-			cp_time_mib[2] = i;
-			tmpstate = cpu_states + (CPUSTATES * i);
-			if (sysctl(cp_time_mib, 3, cp_time[i], &size, NULL, 0) < 0)
+	size = CPUSTATES * sizeof(int64_t);
+	for (i = 0; i < ncpu; i++) {
+		cp_time_mib[2] = i;
+		tmpstate = cpu_states + (CPUSTATES * i);
+		if (sysctl(cp_time_mib, 3, cp_time[i], &size, NULL, 0) < 0) {
+			if (errno != ENODEV)
 				warn("sysctl kern.cp_time2 failed");
-			/* convert cp_time2 counts to percentages */
-			(void) percentages(CPUSTATES, tmpstate, cp_time[i],
-			    cp_old[i], cp_diff[i]);
+			cpu_online[i] = 0;
+			continue;
 		}
-	} else {
-		int cp_time_mib[] = {CTL_KERN, KERN_CPTIME};
-		long cp_time_tmp[CPUSTATES];
-
-		size = sizeof(cp_time_tmp);
-		if (sysctl(cp_time_mib, 2, cp_time_tmp, &size, NULL, 0) < 0)
-			warn("sysctl kern.cp_time failed");
-		for (i = 0; i < CPUSTATES; i++)
-			cp_time[0][i] = cp_time_tmp[i];
-		/* convert cp_time counts to percentages */
-		(void) percentages(CPUSTATES, cpu_states, cp_time[0],
-		    cp_old[0], cp_diff[0]);
+		cpu_online[i] = 1;
+		/* convert cp_time2 counts to percentages */
+		(void) percentages(CPUSTATES, tmpstate, cp_time[i],
+		    cp_old[i], cp_diff[i]);
 	}
 
 	size = sizeof(sysload);
@@ -317,6 +327,7 @@ get_system_info(struct system_info *si)
 
 	/* set arrays and strings */
 	si->cpustates = cpu_states;
+	si->cpuonline = cpu_online;
 	si->memory = memory_stats;
 	si->last_pid = -1;
 }