implement carp demotion control for bgpd.

sessions can be configured to modify the carp demotion counter for a
given interface group (usually, "carp", which has all carp interfaces)
when the session is not established. once the session is established for
60 seconds, the demotion is cleared.
this, used correctly, can prevent a bgpd-box which lost all sessions (and
thus has no routes) to be carp master, while the backup has sessions.
thought through and partially hacked on a drive from calgary to vancouver
with ryan, ok claudio

8 files changed, 151 insertions, 14 deletions

diff --git a/usr.sbin/bgpd/Makefile b/usr.sbin/bgpd/Makefile
index c74c6e999be..4a1572bb86a 100644
--- a/usr.sbin/bgpd/Makefile
+++ b/usr.sbin/bgpd/Makefile
@@ -1,4 +1,4 @@
-#	$OpenBSD: Makefile,v 1.24 2006/01/03 22:19:59 claudio Exp $
+#	$OpenBSD: Makefile,v 1.25 2006/06/17 14:06:09 henning Exp $
 
 .PATH:		${.CURDIR}/..
 
@@ -6,7 +6,7 @@ PROG=	bgpd
 SRCS=	bgpd.c buffer.c session.c log.c parse.y config.c imsg.c \
 	rde.c rde_rib.c rde_decide.c rde_prefix.c mrt.c kroute.c \
 	control.c pfkey.c rde_update.c rde_attr.c printconf.c \
-	rde_filter.c pftable.c name2id.c util.c
+	rde_filter.c pftable.c name2id.c util.c carp.c
 CFLAGS+= -Wall -I${.CURDIR}
 CFLAGS+= -Wstrict-prototypes -Wmissing-prototypes
 CFLAGS+= -Wmissing-declarations
diff --git a/usr.sbin/bgpd/bgpd.8 b/usr.sbin/bgpd/bgpd.8
index 1dea4542e7e..e4e08d5f582 100644
--- a/usr.sbin/bgpd/bgpd.8
+++ b/usr.sbin/bgpd/bgpd.8
@@ -1,4 +1,4 @@
-.\" $OpenBSD: bgpd.8,v 1.18 2006/03/23 20:20:45 claudio Exp $
+.\" $OpenBSD: bgpd.8,v 1.19 2006/06/17 14:06:09 henning Exp $
 .\"
 .\" Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
 .\"
@@ -72,6 +72,28 @@ utility.
 .Pp
 The options are as follows:
 .Bl -tag -width "-f fileXXX"
+.It Fl c
+Force
+.Nm
+to do
+.Xr carp 4
+demotion when the
+.Em demote
+functionality is used.
+Normally,
+.Nm
+will only do demotion when the demotion counter for the group in question
+is already greater than 0.
+At system startup,
+.Xr rc 8
+has the demotion counter for the group
+.Em carp
+increased until after
+.Nm
+is started, so this option should
+.Em not
+be used in
+.Xr rc.conf 8 .
 .It Fl D Ar macro Ns = Ns Ar value
 Define
 .Ar macro
diff --git a/usr.sbin/bgpd/bgpd.c b/usr.sbin/bgpd/bgpd.c
index 3da06935170..06abb9d0eae 100644
--- a/usr.sbin/bgpd/bgpd.c
+++ b/usr.sbin/bgpd/bgpd.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bgpd.c,v 1.137 2006/05/27 21:24:36 claudio Exp $ */
+/*	$OpenBSD: bgpd.c,v 1.138 2006/06/17 14:06:09 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -132,8 +132,11 @@ main(int argc, char *argv[])
 	peer_l = NULL;
 	conf.csock = SOCKET_NAME;
 
-	while ((ch = getopt(argc, argv, "dD:f:nr:s:v")) != -1) {
+	while ((ch = getopt(argc, argv, "cdD:f:nr:s:v")) != -1) {
 		switch (ch) {
+		case 'c':
+			conf.opts |= BGPD_OPT_FORCE_DEMOTE;
+			break;
 		case 'd':
 			debug = 1;
 			break;
@@ -376,6 +379,7 @@ main(int argc, char *argv[])
 	free(rules_l);
 	control_cleanup(conf.csock);
 	control_cleanup(conf.rcsock);
+	carp_demote_shutdown();
 	kr_shutdown();
 	pftable_clear_all();
 	free(conf.listen_addrs);
@@ -646,6 +650,19 @@ dispatch_imsg(struct imsgbuf *ibuf, int idx)
 			else
 				kr_ifinfo(imsg.data);
 			break;
+		case IMSG_DEMOTE:
+			if (idx != PFD_PIPE_SESSION)
+				log_warnx("demote request not from SE");
+			else if (imsg.hdr.len != IMSG_HEADER_SIZE +
+			    sizeof(struct demote_msg))
+				log_warnx("DEMOTE request with wrong len");
+			else {
+				struct demote_msg	*msg;
+
+				msg = imsg.data;
+				carp_demote_set(msg->demote_group, msg->level);
+			}
+			break;
 		default:
 			break;
 		}
diff --git a/usr.sbin/bgpd/bgpd.conf.5 b/usr.sbin/bgpd/bgpd.conf.5
index 0dafcfd45d1..75036cf9771 100644
--- a/usr.sbin/bgpd/bgpd.conf.5
+++ b/usr.sbin/bgpd/bgpd.conf.5
@@ -1,4 +1,4 @@
-.\" $OpenBSD: bgpd.conf.5,v 1.70 2006/05/26 04:02:59 deraadt Exp $
+.\" $OpenBSD: bgpd.conf.5,v 1.71 2006/06/17 14:06:09 henning Exp $
 .\"
 .\" Copyright (c) 2004 Claudio Jeker <claudio@openbsd.org>
 .\" Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -425,6 +425,21 @@ are supported) are announced during the capabilities negotiation.
 Only routes for that address family and subsequent address family will be
 announced and processed.
 .Pp
+.It Ic demote Ar group
+Increase the
+.Xr carp
+demotion counter on the given group, usually
+.Ar carp ,
+when the session is not in state
+.Em ESTABLISHED .
+The demotion counter will be increased as soon as bgpd starts and decreased
+60 seconds after the session went to state
+.Em ESTABLISHED.
+For neighbors added at runtime, the demotion counter is only increased after
+the session has been
+.Em ESTABLISHED
+at least once before dropping.
+.Pp
 .It Ic depend on Ar interface
 The neighbor session will be kept in state
 .Em IDLE
diff --git a/usr.sbin/bgpd/bgpd.h b/usr.sbin/bgpd/bgpd.h
index f961150e395..fddadae0156 100644
--- a/usr.sbin/bgpd/bgpd.h
+++ b/usr.sbin/bgpd/bgpd.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bgpd.h,v 1.203 2006/06/15 10:04:40 claudio Exp $ */
+/*	$OpenBSD: bgpd.h,v 1.204 2006/06/17 14:06:09 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -49,6 +49,7 @@
 #define	BGPD_OPT_VERBOSE		0x0001
 #define	BGPD_OPT_VERBOSE2		0x0002
 #define	BGPD_OPT_NOACTION		0x0004
+#define	BGPD_OPT_FORCE_DEMOTE		0x0008
 
 #define	BGPD_FLAG_NO_FIB_UPDATE		0x0001
 #define	BGPD_FLAG_NO_EVALUATE		0x0002
@@ -220,6 +221,7 @@ struct peer_config {
 	char			 group[PEER_DESCR_LEN];
 	char			 descr[PEER_DESCR_LEN];
 	char			 if_depend[IFNAMSIZ];
+	char			 demote_group[IFNAMSIZ];
 	u_int32_t		 id;
 	u_int32_t		 groupid;
 	u_int32_t		 max_prefix;
@@ -327,7 +329,8 @@ enum imsg_type {
 	IMSG_CTL_SHOW_RIB_MEM,
 	IMSG_CTL_SHOW_TERSE,
 	IMSG_REFRESH,
-	IMSG_IFINFO
+	IMSG_IFINFO,
+	IMSG_DEMOTE
 };
 
 struct imsg_hdr {
@@ -342,6 +345,11 @@ struct imsg {
 	void		*data;
 };
 
+struct demote_msg {
+	char		 demote_group[IFNAMSIZ];
+	int		 level;
+};
+
 enum ctl_results {
 	CTL_RES_OK,
 	CTL_RES_NOSUCHPEER,
diff --git a/usr.sbin/bgpd/parse.y b/usr.sbin/bgpd/parse.y
index 23981247e33..f163bce7cbd 100644
--- a/usr.sbin/bgpd/parse.y
+++ b/usr.sbin/bgpd/parse.y
@@ -1,4 +1,4 @@
-/*	$OpenBSD: parse.y,v 1.190 2006/05/31 02:16:25 pat Exp $ */
+/*	$OpenBSD: parse.y,v 1.191 2006/06/17 14:06:09 henning Exp $ */
 
 /*
  * Copyright (c) 2002, 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -149,7 +149,7 @@ typedef struct {
 %token	AS ROUTERID HOLDTIME YMIN LISTEN ON FIBUPDATE
 %token	RDE EVALUATE IGNORE COMPARE
 %token	GROUP NEIGHBOR NETWORK
-%token	REMOTEAS DESCR LOCALADDR MULTIHOP PASSIVE MAXPREFIX ANNOUNCE
+%token	REMOTEAS DESCR LOCALADDR MULTIHOP PASSIVE MAXPREFIX ANNOUNCE DEMOTE
 %token	ENFORCE NEIGHBORAS CAPABILITIES REFLECTOR DEPEND DOWN SOFTRECONFIG
 %token	DUMP IN OUT
 %token	LOG ROUTECOLL TRANSPARENT
@@ -909,6 +909,24 @@ peeropts	: REMOTEAS asnumber	{
 			}
 			free($3);
 		}
+		| DEMOTE STRING		{
+			if (strlcpy(curpeer->conf.demote_group, $2,
+			    sizeof(curpeer->conf.demote_group)) >=
+			    sizeof(curpeer->conf.demote_group)) {
+				yyerror("demote group name \"%s\" too long: "
+				    "max %u", $2,
+				    sizeof(curpeer->conf.demote_group) - 1);
+				free($2);
+				YYERROR;
+			}
+			free($2);
+			if (carp_demote_init(curpeer->conf.demote_group,
+			    conf->opts & BGPD_OPT_FORCE_DEMOTE) == -1) {
+				yyerror("error initializing group \"%s\"",
+				    curpeer->conf.demote_group);
+				YYERROR;
+			}
+		}
 		| SOFTRECONFIG inout yesno {
 			if ($2)
 				curpeer->conf.softreconfig_in = $3;
@@ -1558,6 +1576,7 @@ lookup(char *s)
 		{ "compare",		COMPARE},
 		{ "connected",		CONNECTED},
 		{ "delete",		DELETE},
+		{ "demote",		DEMOTE},
 		{ "deny",		DENY},
 		{ "depend",		DEPEND},
 		{ "descr",		DESCR},
diff --git a/usr.sbin/bgpd/session.c b/usr.sbin/bgpd/session.c
index e322bd19a22..f1e1c321e12 100644
--- a/usr.sbin/bgpd/session.c
+++ b/usr.sbin/bgpd/session.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: session.c,v 1.251 2006/06/12 12:45:14 henning Exp $ */
+/*	$OpenBSD: session.c,v 1.252 2006/06/17 14:06:09 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004, 2005 Henning Brauer <henning@openbsd.org>
@@ -81,6 +81,7 @@ int	parse_capabilities(struct peer *, u_char *, u_int16_t);
 void	session_dispatch_imsg(struct imsgbuf *, int, u_int *);
 void	session_up(struct peer *);
 void	session_down(struct peer *);
+void	session_demote(struct peer *, int);
 
 int			 la_cmp(struct listen_addr *, struct listen_addr *);
 struct peer		*getpeerbyip(struct sockaddr *);
@@ -288,6 +289,9 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 
 				/* deletion due? */
 				if (p->conf.reconf_action == RECONF_DELETE) {
+					if (p->demoted)
+						session_demote(p, -1);
+					p->conf.demote_group[0] = 0;
 					bgp_fsm(p, EVNT_STOP);
 					log_peer_warnx(&p->conf, "removed");
 					if (last != NULL)
@@ -408,6 +412,17 @@ session_main(struct bgpd_config *config, struct peer *cpeers,
 			    p->IdleHoldResetTimer < nextaction)
 				nextaction = p->IdleHoldResetTimer;
 
+			/* carp demotion */
+			if (p->demoted && p->state == STATE_ESTABLISHED) {
+				if (time(NULL) - p->stats.last_updown >=
+				    INTERVAL_HOLD_DEMOTED)
+					session_demote(p, -1);
+				if (p->stats.last_updown +
+				    INTERVAL_HOLD_DEMOTED < nextaction)
+					nextaction = p->stats.last_updown +
+					    INTERVAL_HOLD_DEMOTED;
+			}
+
 			/* are we waiting for a write? */
 			events = POLLIN;
 			if (p->wbuf.queued > 0 || p->state == STATE_CONNECT)
@@ -560,6 +575,14 @@ init_peer(struct peer *p)
 		p->IdleHoldTimer = 0;		/* no autostart */
 	else
 		p->IdleHoldTimer = time(NULL);	/* start ASAP */
+
+	/*
+	 * on startup, demote if requested.
+	 * do not handle new peers. they must reach ESTABLISHED beforehands.
+	 * peers added at runtime have reconf_action set to RECONF_REINIT.
+	 */
+	if (p->conf.reconf_action != RECONF_REINIT && p->conf.demote_group[0])
+		session_demote(p, +1);
 }
 
 void
@@ -841,6 +864,11 @@ change_state(struct peer *peer, enum session_state state,
 
 	switch (state) {
 	case STATE_IDLE:
+		/* carp demotion first. new peers handled in peer_init */
+		if (peer->state == STATE_ESTABLISHED &&
+		    peer->conf.demote_group[0] && !peer->demoted)
+			session_demote(peer, +1);
+
 		/*
 		 * try to write out what's buffered (maybe a notification),
 		 * don't bother if it fails
@@ -2272,11 +2300,16 @@ session_dispatch_imsg(struct imsgbuf *ibuf, int idx, u_int *listener_cnt)
 				p->next = peers;
 				peers = p;
 			}
-			/* find ones to be deleted */
-			for (p = peers; p != NULL; p = p->next)
+			/* find ones that need attention */
+			for (p = peers; p != NULL; p = p->next) {
+				/* needs to be deleted? */
 				if (p->conf.reconf_action == RECONF_NONE &&
 				    !p->conf.cloned)
 					p->conf.reconf_action = RECONF_DELETE;
+				/* had demotion, is demoted, demote removed? */
+				if (p->demoted && !p->conf.demote_group[0])
+						session_demote(p, -1);
+			}
 
 			/* delete old listeners */
 			for (la = TAILQ_FIRST(conf->listen_addrs); la != NULL;
@@ -2736,3 +2769,18 @@ addr2sa(struct bgpd_addr *addr, u_int16_t port)
 
 	return ((struct sockaddr *)&ss);
 }
+
+void
+session_demote(struct peer *p, int level)
+{
+	struct demote_msg	msg;
+
+	strlcpy(msg.demote_group, p->conf.demote_group,
+	    sizeof(msg.demote_group));
+	msg.level = level;
+	if (imsg_compose(ibuf_main, IMSG_DEMOTE, p->conf.id, 0, -1,
+	    &msg, sizeof(msg)) == -1)
+		fatalx("imsg_compose error");
+
+	p->demoted += level;
+}
diff --git a/usr.sbin/bgpd/session.h b/usr.sbin/bgpd/session.h
index 38fc2f5584e..bbd20d26d7f 100644
--- a/usr.sbin/bgpd/session.h
+++ b/usr.sbin/bgpd/session.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: session.h,v 1.83 2006/05/27 15:43:13 claudio Exp $ */
+/*	$OpenBSD: session.h,v 1.84 2006/06/17 14:06:09 henning Exp $ */
 
 /*
  * Copyright (c) 2003, 2004 Henning Brauer <henning@openbsd.org>
@@ -26,6 +26,7 @@
 #define	INTERVAL_HOLD			90
 #define	INTERVAL_IDLE_HOLD_INITIAL	30
 #define	INTERVAL_HOLD_CLONED		3600
+#define	INTERVAL_HOLD_DEMOTED		60
 #define	MAX_IDLE_HOLD			3600
 #define	MSGSIZE_HEADER			19
 #define	MSGSIZE_HEADER_MARKER		16
@@ -178,6 +179,7 @@ struct peer {
 	u_int16_t		 holdtime;
 	u_int8_t		 auth_established;
 	u_int8_t		 depend_ok;
+	u_int8_t		 demoted;
 	u_int8_t		 passive;
 };
 
@@ -230,3 +232,9 @@ int	pfkey_init(struct bgpd_sysdep *);
 /* printconf.c */
 void	print_config(struct bgpd_config *, struct network_head *, struct peer *,
 	    struct filter_head *, struct mrt_head *);
+
+/* carp.c */
+int	 carp_demote_init(char *, int);
+void	 carp_demote_shutdown(void);
+int	 carp_demote_get(char *);
+int	 carp_demote_set(char *, int);