Make sasyncd fail back correctly with carp preemption enabled.

Hold the carp demotion when booting, to prevent carp from preempting until
we've sync'd with our peers.  This adds a new CTL_ENDSNAP message to the
exchange between the sasync daemons to indicate when the complete snapshot
has been sent. Undemote after 60 seconds, or when recieve a CTL_ENDSNAP from
all our peers.

Syntax is slightly changed, removing the 'carp' keyword (so do
"interface carp0" rather than "carp interface carp0". Adds 'group <ifgroup>',
defaults to the 'carp' group.

ok moritz@

10 files changed, 186 insertions, 25 deletions

diff --git a/usr.sbin/sasyncd/carp.c b/usr.sbin/sasyncd/carp.c
index 918328dc51e..7c67dc1aed0 100644
--- a/usr.sbin/sasyncd/carp.c
+++ b/usr.sbin/sasyncd/carp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: carp.c,v 1.3 2006/06/01 22:43:12 mcbride Exp $	*/
+/*	$OpenBSD: carp.c,v 1.4 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -43,6 +43,8 @@
 
 #include "sasyncd.h"
 
+int carp_demoted = 0;
+
 static enum RUNSTATE
 carp_map_state(u_char link_state)
 {
@@ -94,6 +96,54 @@ carp_get_state(char *ifname)
 	return carp_map_state(ifrdat.ifi_link_state);
 }
 
+void
+carp_demote(int demote, int force)
+{
+	struct ifgroupreq        ifgr;
+	int s;
+
+	if (carp_demoted + demote < 0) {
+		log_msg(1, "carp_demote: mismatched promotion");
+		return;
+	}
+
+	s = socket(AF_INET, SOCK_DGRAM, 0);
+	if (s < 0) {
+		log_msg(1, "carp_demote: couldn't open socket");
+		return;
+	}
+		
+	bzero(&ifgr, sizeof(ifgr));
+	strlcpy(ifgr.ifgr_name, cfgstate.carp_ifgroup, sizeof(ifgr.ifgr_name));
+
+	/* Unless we force it, don't demote if we're not demoting already. */
+	if (!force) {
+		if (ioctl(s, SIOCGIFGATTR, (caddr_t)&ifgr) == -1) {
+			log_msg(1, "carp_demote: unable to get "
+			    "the demote state of group '%s'", 
+			    cfgstate.carp_ifgroup);		
+			    goto done;
+		}
+	
+		if (ifgr.ifgr_attrib.ifg_carp_demoted == 0)
+			goto done;
+	}
+
+	ifgr.ifgr_attrib.ifg_carp_demoted = demote;
+	if (ioctl(s, SIOCSIFGATTR, (caddr_t)&ifgr) == -1)
+		log_msg(1, "carp_demote: unable to %s the demote state "
+		    "of group '%s'", (demote > 0) ?
+		    "increment" : "decrement", cfgstate.carp_ifgroup);
+	else {
+		carp_demoted += demote;
+		log_msg(1, "carp_demote: %sed the demote state "
+		    "of group '%s'", (demote > 0) ?
+		    "increment" : "decrement", cfgstate.carp_ifgroup);
+	}
+done:
+	close(s);
+}
+
 const char*
 carp_state_name(enum RUNSTATE state)
 {
diff --git a/usr.sbin/sasyncd/conf.y b/usr.sbin/sasyncd/conf.y
index 51108fd5293..b8d6de3a418 100644
--- a/usr.sbin/sasyncd/conf.y
+++ b/usr.sbin/sasyncd/conf.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: conf.y,v 1.9 2006/06/01 22:43:12 mcbride Exp $	*/
+/*	$OpenBSD: conf.y,v 1.10 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -57,9 +57,9 @@ void	yyerror(const char *);
 	int	 val;
 }
 
-%token MODE CARP INTERFACE INTERVAL LISTEN ON PORT PEER SHAREDKEY
+%token MODE INTERFACE INTERVAL LISTEN ON PORT PEER SHAREDKEY
 %token Y_SLAVE Y_MASTER INET INET6 FLUSHMODE STARTUP NEVER SYNC
-%token SKIPSLAVE
+%token GROUP SKIPSLAVE 
 %token <string> STRING
 %token <val>	VALUE
 %type  <val>	af port mode flushmode
@@ -102,12 +102,21 @@ flushmode	: STARTUP		{ $$ = FM_STARTUP; }
 		| SYNC			{ $$ = FM_SYNC; }
 		;
 
-setting		: CARP INTERFACE STRING
+setting		: INTERFACE STRING
 		{
 			if (cfgstate.carp_ifname)
 				free(cfgstate.carp_ifname);
-			cfgstate.carp_ifname = $3;
-			log_msg(2, "config: carp interface %s", $3);
+			cfgstate.carp_ifname = $2;
+			log_msg(2, "config: interface %s",
+			    cfgstate.carp_ifname);
+		}
+		| GROUP STRING
+		{
+			if (cfgstate.carp_ifgroup)
+				free(cfgstate.carp_ifgroup);
+			cfgstate.carp_ifgroup = $2;
+			log_msg(2, "config: group %s",
+			    cfgstate.carp_ifgroup);
 		}
 		| FLUSHMODE flushmode
 		{
@@ -140,6 +149,7 @@ setting		: CARP INTERFACE STRING
 				peer->name = $2;
 			}
 			LIST_INSERT_HEAD(&cfgstate.peerlist, peer, link);
+			cfgstate.peercnt++;
 			log_msg(2, "config: add peer %s", peer->name);
 		}
 		| LISTEN ON STRING af port
@@ -193,8 +203,8 @@ match(char *token)
 {
 	/* Sorted */
 	static const struct keyword keywords[] = {
-		{ "carp", CARP },
 		{ "flushmode", FLUSHMODE },
+		{ "group", GROUP },
 		{ "inet", INET },
 		{ "inet6", INET6 },
 		{ "interface", INTERFACE },
@@ -322,6 +332,9 @@ conf_parse_file(char *cfgfile)
 	r = yyparse();
 	free(buf);
 
+	if (!cfgstate.carp_ifgroup)
+		cfgstate.carp_ifgroup = strdup("carp");
+
 	return r;
 
   bad:
diff --git a/usr.sbin/sasyncd/monitor.c b/usr.sbin/sasyncd/monitor.c
index 2f403990d0d..94c9a0b337e 100644
--- a/usr.sbin/sasyncd/monitor.c
+++ b/usr.sbin/sasyncd/monitor.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: monitor.c,v 1.7 2006/01/26 09:53:46 moritz Exp $	*/
+/*	$OpenBSD: monitor.c,v 1.8 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -157,22 +157,60 @@ monitor_loop(void)
 				break;
 		}
 
-		/* Wait for next snapshot task. Disregard read data. */
+		/* Wait for next task */
 		if ((r = m_read(m_state.s, &v, sizeof v)) < 1) {
 			if (r == -1)
 				log_err(0, "monitor_loop: read() ");
 			break;
 		}
 
-		/* Get the data. */
-		m_priv_pfkey_snap(m_state.s);
+		switch (v) {
+		case MONITOR_GETSNAP:
+			/* Get the data. */
+			m_priv_pfkey_snap(m_state.s);
+			break;
+
+		case MONITOR_CARPINC:
+			carp_demote(CARP_INC, 1);
+			break;
+
+		case MONITOR_CARPDEC:
+			carp_demote(CARP_DEC, 1);
+			break;
+		}
 	}
 
+	monitor_carpundemote(NULL);
+
 	if (!sigchld)
 		log_msg(0, "monitor_loop: priv process exiting abnormally");
 	exit(0);
 }
 
+void
+monitor_carpundemote(void *v)
+{
+	u_int32_t mtype = MONITOR_CARPDEC;
+	if (!carp_demoted)
+		return;
+	if (m_write(m_state.s, &mtype, sizeof mtype) < 1)
+		log_msg(1, "monitor_carpundemote: unable to write to monitor");
+	else
+		carp_demoted = 0;
+}
+
+void
+monitor_carpdemote(void *v)
+{
+	u_int32_t mtype = MONITOR_CARPINC;
+	if (carp_demoted)
+		return;
+	if (m_write(m_state.s, &mtype, sizeof mtype) < 1)
+		log_msg(1, "monitor_carpdemote: unable to write to monitor");
+	else
+		carp_demoted = 1;
+}
+
 int
 monitor_get_pfkey_snap(u_int8_t **sadb, u_int32_t *sadbsize, u_int8_t **spd,
     u_int32_t *spdsize)
@@ -180,8 +218,7 @@ monitor_get_pfkey_snap(u_int8_t **sadb, u_int32_t *sadbsize, u_int8_t **spd,
 	u_int32_t	v;
 	ssize_t		rbytes;
 
-	/* We write a (any) value to the monitor socket to start a snapshot. */
-	v = 0;
+	v = MONITOR_GETSNAP;
 	if (m_write(m_state.s, &v, sizeof v) < 1)
 		return -1;
 
diff --git a/usr.sbin/sasyncd/net.c b/usr.sbin/sasyncd/net.c
index 21c9287ca45..c131a1be1d3 100644
--- a/usr.sbin/sasyncd/net.c
+++ b/usr.sbin/sasyncd/net.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: net.c,v 1.12 2006/04/16 19:28:36 moritz Exp $	*/
+/*	$OpenBSD: net.c,v 1.13 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -653,6 +653,7 @@ net_shutdown(void)
 		if (p->name)
 			free(p->name);
 		LIST_REMOVE(p, link);
+		cfgstate.peercnt--;
 		free(p);
 	}
 
diff --git a/usr.sbin/sasyncd/net.h b/usr.sbin/sasyncd/net.h
index 2e8b7baa00d..fa9acffef61 100644
--- a/usr.sbin/sasyncd/net.h
+++ b/usr.sbin/sasyncd/net.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: net.h,v 1.4 2005/05/27 20:47:11 ho Exp $	*/
+/*	$OpenBSD: net.h,v 1.5 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -43,9 +43,11 @@ struct syncpeer {
 };
 
 /* Control message types. */
-enum CTLTYPE { RESERVED = 0, CTL_STATE, CTL_ERROR, CTL_ACK, CTL_UNKNOWN };
+enum CTLTYPE { RESERVED = 0, CTL_STATE, CTL_ERROR, CTL_ACK,
+    CTL_ENDSNAP, CTL_UNKNOWN };
 #define CTLTYPES { \
-	"RESERVED", "CTL_STATE", "CTL_ERROR", "CTL_ACK", "CTL_UNKNOWN" \
+	"RESERVED", "CTL_STATE", "CTL_ERROR", "CTL_ACK", \
+	"CTL_ENDSNAP", "CTL_UNKNOWN" \
 };
 
 /* net.c */
@@ -56,5 +58,6 @@ void	net_disconnect_peer(struct syncpeer *);
 void	net_ctl_handle_msg(struct syncpeer *, u_int8_t *, u_int32_t);
 int	net_ctl_send_ack(struct syncpeer *, enum CTLTYPE, u_int32_t);
 int	net_ctl_send_error(struct syncpeer *, enum CTLTYPE);
+int	net_ctl_send_endsnap(struct syncpeer *);
 int	net_ctl_send_state(struct syncpeer *);
 
diff --git a/usr.sbin/sasyncd/net_ctl.c b/usr.sbin/sasyncd/net_ctl.c
index 811ed648ba5..fcd346a0ad5 100644
--- a/usr.sbin/sasyncd/net_ctl.c
+++ b/usr.sbin/sasyncd/net_ctl.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: net_ctl.c,v 1.7 2006/06/01 22:43:12 mcbride Exp $	*/
+/*	$OpenBSD: net_ctl.c,v 1.8 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -50,6 +50,8 @@ struct ctlmsg {
 	u_int32_t	data2;
 };
 
+int snapcount = 0;
+
 static int
 net_ctl_check_state(struct syncpeer *p, enum RUNSTATE nstate)
 {
@@ -89,6 +91,18 @@ net_ctl_handle_msg(struct syncpeer *p, u_int8_t *msg, u_int32_t msglen)
 	}
 
 	switch (ntohl(ctl->type)) {
+	case CTL_ENDSNAP:
+		log_msg(4, "net_ctl: got CTL_ENDSNAP from peer \"%s\"",
+		    p->name);
+
+		/* XXX More sophistication required to handle multiple peers. */
+		if (carp_demoted) {
+			snapcount++;
+			if (snapcount >= cfgstate.peercnt)
+				monitor_carpundemote(NULL);
+		}
+		break;
+
 	case CTL_STATE:
 		log_msg(4, "net_ctl: got CTL_STATE from peer \"%s\"", p->name);
 		nstate = (enum RUNSTATE)ntohl(ctl->data);
@@ -174,6 +188,12 @@ net_ctl_send_error(struct syncpeer *p, enum CTLTYPE prevtype)
 	return net_ctl_send(p, CTL_ERROR, (u_int32_t)prevtype, 0);
 }
 
+int
+net_ctl_send_endsnap(struct syncpeer *p)
+{
+	return net_ctl_send(p, CTL_ENDSNAP, 0, 0);
+}
+
 /* After a CARP tracker state change, send an state ctl msg to all peers. */
 void
 net_ctl_update_state(void)
diff --git a/usr.sbin/sasyncd/pfkey.c b/usr.sbin/sasyncd/pfkey.c
index 9f342e140ce..ed735f7f42f 100644
--- a/usr.sbin/sasyncd/pfkey.c
+++ b/usr.sbin/sasyncd/pfkey.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: pfkey.c,v 1.13 2006/02/15 05:09:29 david Exp $	*/
+/*	$OpenBSD: pfkey.c,v 1.14 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -396,6 +396,7 @@ pfkey_send_message(fd_set *fds)
 	SIMPLEQ_REMOVE_HEAD(&pfkey_msglist, next);
 	free(pmsg->buf);
 	free(pmsg);
+
 	return;
 }
 
@@ -524,5 +525,7 @@ pfkey_snapshot(void *v)
 		memset(spd, 0, spdsz);
 		free(spd);
 	}
+
+	net_ctl_send_endsnap(p);
 	return;
 }
diff --git a/usr.sbin/sasyncd/sasyncd.c b/usr.sbin/sasyncd/sasyncd.c
index d9277382b93..f07904bfab4 100644
--- a/usr.sbin/sasyncd/sasyncd.c
+++ b/usr.sbin/sasyncd/sasyncd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sasyncd.c,v 1.10 2006/06/01 22:43:12 mcbride Exp $	*/
+/*	$OpenBSD: sasyncd.c,v 1.11 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -79,6 +79,9 @@ sasyncd_run(pid_t ppid)
 	signal(SIGTERM, sasyncd_stop);
 	signal(SIGHUP, sasyncd_stop);
 
+	timer_add("carp_undemote", CARP_DEMOTE_MAXTIME,
+	    monitor_carpundemote, NULL);
+
 	while (!daemon_shutdown) {
 		memset(rfds, 0, fdsetsize);
 		memset(wfds, 0, fdsetsize);
@@ -98,7 +101,7 @@ sasyncd_run(pid_t ppid)
 
 		timeout = &tv;
 		timer_next_event(&tv);
-
+ 
 		n = select(maxfd, rfds, wfds, 0, timeout);
 		if (n == -1) {
 			if (errno != EINTR) {
@@ -159,6 +162,8 @@ main(int argc, char **argv)
 	if (r)
 		return 1;
 
+	carp_demote(CARP_INC, 0);
+
 	if (carp_init())
 		return 1;
 	if (pfkey_init(0))
diff --git a/usr.sbin/sasyncd/sasyncd.conf.5 b/usr.sbin/sasyncd/sasyncd.conf.5
index 05981e463cc..874ef7f49c9 100644
--- a/usr.sbin/sasyncd/sasyncd.conf.5
+++ b/usr.sbin/sasyncd/sasyncd.conf.5
@@ -1,4 +1,4 @@
-.\" $OpenBSD: sasyncd.conf.5,v 1.11 2006/06/01 22:43:12 mcbride Exp $
+.\" $OpenBSD: sasyncd.conf.5,v 1.12 2006/06/02 20:09:43 mcbride Exp $
 .\"
 .\" Copyright (c) 2005 Håkan Olsson.  All rights reserved.
 .\"
@@ -59,7 +59,7 @@ pseudo user and be readable only by this owner, e.g.\&
 The following configuration settings are understood:
 .Bl -tag -width Ds
 .It Xo
-.Ic carp interface
+.Ic interface
 .Ar interface
 .Xc
 Specify which
@@ -67,6 +67,16 @@ Specify which
 interface
 .Nm sasyncd
 should track master/slave state on.
+.It Xo
+.Ic group
+.Ar group
+.Xc
+Specify which interface group
+.Nm sasyncd
+should use to suppress carp preemption while the system boots.
+The default is to use the 
+.Dq carp
+group.
 .It Ic flushmode sync | startup | never
 Controls how the
 .Xr sasyncd 8
diff --git a/usr.sbin/sasyncd/sasyncd.h b/usr.sbin/sasyncd/sasyncd.h
index f5ecd723831..5e285c9e699 100644
--- a/usr.sbin/sasyncd/sasyncd.h
+++ b/usr.sbin/sasyncd/sasyncd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: sasyncd.h,v 1.9 2006/06/01 22:43:12 mcbride Exp $	*/
+/*	$OpenBSD: sasyncd.h,v 1.10 2006/06/02 20:09:43 mcbride Exp $	*/
 
 /*
  * Copyright (c) 2005 Håkan Olsson.  All rights reserved.
@@ -46,6 +46,7 @@ struct cfgstate {
 	u_int32_t	 flags;
 
 	char		*carp_ifname;
+	char		*carp_ifgroup;
 	int		 carp_ifindex;
 
 	char		*sharedkey;
@@ -58,6 +59,7 @@ struct cfgstate {
 	in_port_t	 listen_port;
 	sa_family_t	 listen_family;
 
+	int		 peercnt;
 	LIST_HEAD(, syncpeer) peerlist;
 };
 
@@ -71,6 +73,7 @@ struct cfgstate {
 #define SKIP_LOCAL_SAS	0x0004
 
 extern struct cfgstate	cfgstate;
+extern int		carp_demoted;
 
 #define SASYNCD_USER	"_isakmpd"
 #define SASYNCD_CFGFILE	"/etc/sasyncd.conf"
@@ -92,12 +95,26 @@ extern struct cfgstate	cfgstate;
 #define MSG_PFKEYDATA	1
 #define MSG_MAXTYPE	1	/* Increase when new types are added. */
 
+
+
+enum {
+	MONITOR_GETSNAP,
+	MONITOR_CARPINC,
+	MONITOR_CARPDEC
+};
+
+#define CARP_DEC	-1
+#define CARP_INC	1
+
+#define CARP_DEMOTE_MAXTIME	60
+
 /* conf.c */
 int	conf_init(int, char **);
 
 /* carp.c */
 int		carp_init(void);
 void		carp_check_state(void);
+void		carp_demote(int, int);
 void		carp_update_state(enum RUNSTATE);
 void		carp_set_rfd(fd_set *);
 void		carp_read_message(fd_set *);
@@ -121,6 +138,8 @@ void	log_err(const char *, ...);
 /* monitor.c */
 pid_t	monitor_init(void);
 void	monitor_loop(void);
+void	monitor_carpdemote(void *);
+void	monitor_carpundemote(void *);
 int	monitor_get_pfkey_snap(u_int8_t **, u_int32_t *, u_int8_t **,
     u_int32_t *);