Rework the logout code and use this to do a proper logout when

exiting.  This works well for idle sessions but still has some
issues on busy session. It seems more task scheduler changes are
needed to make this work.  This also includes some mem-leak fixes
in error pathes found by Igor Zinovik.
go for it dlg@

5 files changed, 255 insertions, 69 deletions

diff --git a/usr.sbin/iscsid/connection.c b/usr.sbin/iscsid/connection.c
index cb76c158090..6102ddd3110 100644
--- a/usr.sbin/iscsid/connection.c
+++ b/usr.sbin/iscsid/connection.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: connection.c,v 1.11 2011/04/28 18:32:01 claudio Exp $ */
+/*	$OpenBSD: connection.c,v 1.12 2011/05/02 06:32:56 claudio Exp $ */
 
 /*
  * Copyright (c) 2009 Claudio Jeker <claudio@openbsd.org>
@@ -17,7 +17,6 @@
  */
 
 #include <sys/types.h>
-#include <sys/ioctl.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
@@ -26,8 +25,6 @@
 #include <netinet/tcp.h>
 
 #include <scsi/iscsi.h>
-#include <scsi/scsi_all.h>
-#include <dev/vscsivar.h>
 
 #include <errno.h>
 #include <event.h>
@@ -181,31 +178,16 @@ conn_write_dispatch(int fd, short event, void *arg)
 }
 
 void
-conn_logout(struct connection *c)
-{
-	conn_fsm(c, CONN_EV_LOGOUT);
-}
-
-void
 conn_fail(struct connection *c)
 {
 	log_debug("conn_fail");
 	conn_fsm(c, CONN_EV_FAIL);
 }
 
-void
-conn_loggedin(struct connection *c)
-{
-	if (c->session->config.SessionType == SESSION_TYPE_DISCOVERY)
-		conn_fsm(c, CONN_EV_DISCOVERY);
-	else
-		conn_fsm(c, CONN_EV_LOGGED_IN);
-}
-
 int
 conn_task_ready(struct connection *c)
 {
-	if ((c->state & CONN_LOGGED_IN) && TAILQ_EMPTY(&c->tasks))
+	if ((c->state & CONN_RUNNING) && TAILQ_EMPTY(&c->tasks))
 		return 1;
 	return 0;
 }
@@ -291,12 +273,12 @@ struct {
 	{ CONN_FREE, CONN_EV_CONNECT, c_do_connect },		/* T1 */
 	{ CONN_XPT_WAIT, CONN_EV_CONNECTED, c_do_login },	/* T4 */
 	{ CONN_IN_LOGIN, CONN_EV_LOGGED_IN, c_do_loggedin },	/* T5 */
-	{ CONN_IN_LOGIN, CONN_EV_DISCOVERY, c_do_loggedin },	/* T5 */
 	{ CONN_LOGGED_IN, CONN_EV_LOGOUT, c_do_logout },	/* T9 */
 	{ CONN_LOGOUT_REQ, CONN_EV_LOGOUT, c_do_logout },	/* T10 */
 	{ CONN_IN_LOGOUT, CONN_EV_LOGGED_OUT, c_do_loggedout },	/* T13 */
 	{ CONN_ANYSTATE, CONN_EV_CLOSED, c_do_fail },
 	{ CONN_ANYSTATE, CONN_EV_FAIL, c_do_fail },
+	{ CONN_ANYSTATE, CONN_EV_FREE, c_do_fail },
 	{ 0, 0, NULL }
 };
 
@@ -363,18 +345,15 @@ c_do_login(struct connection *c, enum c_event ev)
 int
 c_do_loggedin(struct connection *c, enum c_event ev)
 {
-	if (ev == CONN_EV_LOGGED_IN)
-		vscsi_event(VSCSI_REQPROBE, c->session->target, -1);
-	else
-		initiator_discovery(c->session);
+	session_fsm(c->session, SESS_EV_CONN_LOGGED_IN, c);
+
 	return (CONN_LOGGED_IN);
 }
 
 int
 c_do_logout(struct connection *c, enum c_event ev)
 {
-	/* do full logout */
-	initiator_logout(c, ISCSI_LOGOUT_CLOSE_SESS, 1);
+	/* logout is in progress ... */
 	return (CONN_IN_LOGOUT);
 }
 
@@ -386,8 +365,7 @@ c_do_loggedout(struct connection *c, enum c_event ev)
 	event_del(&c->wev);
 	close(c->fd);
 
-	session_fsm(c->session, SESS_EV_CONN_CLOSED, c);
-
+	/* session is informed by the logout handler */
 	return (CONN_FREE);
 }
 
@@ -401,12 +379,11 @@ c_do_fail(struct connection *c, enum c_event ev)
 
 	session_fsm(c->session, SESS_EV_CONN_FAIL, c);
 
-	if (c->state & CONN_NOT_LOGGED_IN)
+	if (ev == CONN_EV_FREE || c->state & CONN_NEVER_LOGGED_IN)
 		return (CONN_FREE);
 	return (CONN_CLEANUP_WAIT);
 }
 
-
 const char *
 conn_state(int s)
 {
@@ -452,14 +429,14 @@ conn_event(enum c_event e)
 		return "connected";
 	case CONN_EV_LOGGED_IN:
 		return "logged in";
-	case CONN_EV_DISCOVERY:
-		return "discovery";
 	case CONN_EV_LOGOUT:
 		return "logout";
 	case CONN_EV_LOGGED_OUT:
 		return "logged out";
 	case CONN_EV_CLOSED:
 		return "closed";
+	case CONN_EV_FREE:
+		return "forced free";
 	}
 
 	snprintf(buf, sizeof(buf), "UKNWN %d", e);
diff --git a/usr.sbin/iscsid/initiator.c b/usr.sbin/iscsid/initiator.c
index 7d7975e8e85..7510baffed4 100644
--- a/usr.sbin/iscsid/initiator.c
+++ b/usr.sbin/iscsid/initiator.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: initiator.c,v 1.7 2011/04/27 19:02:07 claudio Exp $ */
+/*	$OpenBSD: initiator.c,v 1.8 2011/05/02 06:32:56 claudio Exp $ */
 
 /*
  * Copyright (c) 2009 Claudio Jeker <claudio@openbsd.org>
@@ -47,7 +47,7 @@ initiator_init(void)
 	    arc4random_uniform(0xffffff) | ISCSI_ISID_RAND;
 	initiator->config.isid_qual = arc4random_uniform(0xffff);
 	TAILQ_INIT(&initiator->sessions);
-	return (initiator);
+	return initiator;
 }
 
 void
@@ -62,6 +62,30 @@ initiator_cleanup(struct initiator *i)
 	free(initiator);
 }
 
+void
+initiator_shutdown(struct initiator *i)
+{
+	struct session *s;
+
+	log_debug("initiator_shutdown: going down");
+
+	TAILQ_FOREACH(s, &initiator->sessions, entry)
+		session_shutdown(s);	
+}
+
+int
+initiator_isdown(struct initiator *i)
+{
+	struct session *s;
+	int inprogres = 0;
+
+	TAILQ_FOREACH(s, &initiator->sessions, entry) {
+		if ((s->state & SESS_RUNNING) && !(s->state & SESS_FREE))
+			inprogres = 1;
+	}
+	return !inprogres;
+}
+
 struct session *
 initiator_t2s(u_int target)
 {
@@ -205,7 +229,7 @@ initiator_login_cb(struct connection *c, void *arg, struct pdu *p)
 	}
 
 	conn_task_cleanup(c, &tl->task);
-	conn_loggedin(c);
+	conn_fsm(c, CONN_EV_LOGGED_IN);
 	free(tl);
 	pdu_free(p);
 }
@@ -254,7 +278,7 @@ initiator_discovery_cb(struct connection *c, void *arg, struct pdu *p)
 		    lresp->datalen[2];
 		if (size == 0) {
 			/* empty response */
-			conn_logout(c);
+			session_shutdown(c->session);
 			break;
 		}
 		buf = pdu_getbuf(p, &n, PDU_DATA);
@@ -268,7 +292,7 @@ initiator_discovery_cb(struct connection *c, void *arg, struct pdu *p)
 			log_debug("%s\t=>\t%s", k->key, k->value);
 		}
 		free(kvp);
-		conn_logout(c);
+		session_shutdown(c->session);
 		break;
 	default:
 		log_debug("initiator_discovery_cb: unexpected message type %x",
@@ -282,7 +306,7 @@ fail:
 }
 
 void
-initiator_logout(struct connection *c, u_int8_t reason, int onconn)
+initiator_logout(struct session *s, struct connection *c, u_int8_t reason)
 {
 	struct task_logout *tl;
 	struct pdu *p;
@@ -290,7 +314,7 @@ initiator_logout(struct connection *c, u_int8_t reason, int onconn)
 
 	if (!(tl = calloc(1, sizeof(*tl)))) {
 		log_warn("initiator_logout");
-		conn_fail(c);
+		/* XXX sess_fail */
 		return;
 	}
 	tl->c = c;
@@ -298,26 +322,29 @@ initiator_logout(struct connection *c, u_int8_t reason, int onconn)
 
 	if (!(p = pdu_new())) {
 		log_warn("initiator_logout");
-		conn_fail(c);
+		/* XXX sess_fail */
+		free(tl);
 		return;
 	}
 	if (!(loreq = pdu_gethdr(p))) {
 		log_warn("initiator_logout");
-		conn_fail(c);
+		/* XXX sess_fail */
+		pdu_free(p);
+		free(tl);
 		return;
 	}
 
 	loreq->opcode = ISCSI_OP_LOGOUT_REQUEST;
 	loreq->flags = ISCSI_LOGOUT_F | reason;
-	if (reason != 0)
+	if (reason != ISCSI_LOGOUT_CLOSE_SESS)
 		loreq->cid = c->cid;
 
-	task_init(&tl->task, c->session, 0, tl, initiator_logout_cb, NULL);
+	task_init(&tl->task, s, 0, tl, initiator_logout_cb, NULL);
 	task_pdu_add(&tl->task, p);
-	if (onconn)
+	if (c && (c->state & CONN_RUNNING))
 		conn_task_issue(c, &tl->task);
 	else
-		session_task_issue(c->session, &tl->task);
+		session_logout_issue(s, &tl->task);
 }
 
 void
@@ -326,21 +353,27 @@ initiator_logout_cb(struct connection *c, void *arg, struct pdu *p)
 	struct task_logout *tl = arg;
 	struct iscsi_pdu_logout_response *loresp;
 
-	c = tl->c;
 	loresp = pdu_getbuf(p, NULL, PDU_HEADER);
 	log_debug("initiator_logout_cb: "
-	    "reason %d, Time2Wait %d, Time2Retain %d",
+	    "response %d, Time2Wait %d, Time2Retain %d",
 	    loresp->response, loresp->time2wait, loresp->time2retain);
 
 	switch (loresp->response) {
 	case ISCSI_LOGOUT_RESP_SUCCESS:
-		conn_fsm(tl->c, CONN_EV_LOGGED_OUT);
+		if (tl->reason == ISCSI_LOGOUT_CLOSE_SESS) {
+			conn_fsm(c, CONN_EV_LOGGED_OUT);
+			session_fsm(c->session, SESS_EV_CLOSED, NULL);
+		} else {
+			conn_fsm(tl->c, CONN_EV_LOGGED_OUT);
+			session_fsm(c->session, SESS_EV_CONN_CLOSED, tl->c);
+		}
 		break;
 	case ISCSI_LOGOUT_RESP_UNKN_CID:
 		/* connection ID not found, retry will not help */
 		log_warnx("%s: logout failed, cid %d unknown, giving up\n",
 		    tl->c->session->config.SessionName,
 		    tl->c->cid);
+		conn_fsm(tl->c, CONN_EV_FREE);
 		break;
 	case ISCSI_LOGOUT_RESP_NO_SUPPORT:
 	case ISCSI_LOGOUT_RESP_ERROR:
diff --git a/usr.sbin/iscsid/iscsid.c b/usr.sbin/iscsid/iscsid.c
index fe34cf72886..1989c5775ea 100644
--- a/usr.sbin/iscsid/iscsid.c
+++ b/usr.sbin/iscsid/iscsid.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iscsid.c,v 1.5 2011/04/27 19:16:15 claudio Exp $ */
+/*	$OpenBSD: iscsid.c,v 1.6 2011/05/02 06:32:56 claudio Exp $ */
 
 /*
  * Copyright (c) 2009 Claudio Jeker <claudio@openbsd.org>
@@ -36,8 +36,12 @@
 
 void		main_sig_handler(int, short, void *);
 __dead void	usage(void);
+void		shutdown_cb(int, short, void *);
 
 struct initiator *initiator;
+struct event exit_ev;
+int exit_rounds;
+#define ISCSI_EXIT_WAIT 5
 
 int
 main(int argc, char *argv[])
@@ -127,12 +131,18 @@ main(int argc, char *argv[])
 void
 main_sig_handler(int sig, short event, void *arg)
 {
+	struct timeval tv;
+
 	/* signal handler rules don't apply, libevent decouples for us */
 	switch (sig) {
 	case SIGTERM:
 	case SIGINT:
 	case SIGHUP:
-		event_loopexit(NULL);
+		initiator_shutdown(initiator);
+		evtimer_set(&exit_ev, shutdown_cb, NULL);
+		timerclear(&tv);
+		if (evtimer_add(&exit_ev, &tv) == -1)
+			fatal("main_sig_handler");
 		break;
 	default:
 		fatalx("unexpected signal");
@@ -227,3 +237,18 @@ iscsid_ctrl_dispatch(void *ch, struct pdu *pdu)
 done:
 	pdu_free(pdu);
 }
+
+void
+shutdown_cb(int fd, short event, void *arg)
+{
+	struct timeval tv;
+
+	if (exit_rounds++ >= ISCSI_EXIT_WAIT || initiator_isdown(initiator))
+		event_loopexit(NULL);
+
+	timerclear(&tv);
+	tv.tv_sec = 1;
+
+	if (evtimer_add(&exit_ev, &tv) == -1)
+		fatal("shutdown_cb");
+}
diff --git a/usr.sbin/iscsid/iscsid.h b/usr.sbin/iscsid/iscsid.h
index 4eaf77598bd..42ec7c5237c 100644
--- a/usr.sbin/iscsid/iscsid.h
+++ b/usr.sbin/iscsid/iscsid.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: iscsid.h,v 1.7 2011/04/27 19:02:07 claudio Exp $ */
+/*	$OpenBSD: iscsid.h,v 1.8 2011/05/02 06:32:56 claudio Exp $ */
 
 /*
  * Copyright (c) 2009 Claudio Jeker <claudio@openbsd.org>
@@ -67,11 +67,12 @@ TAILQ_HEAD(taskq, task);
 
 #define SESS_INIT		0x0001
 #define SESS_FREE		0x0002
-#define SESS_LOGGED_IN		0x0003
-#define SESS_FAILED		0x0004
+#define SESS_LOGGED_IN		0x0004
+#define SESS_FAILED		0x0008
+#define SESS_DOWN		0x0010
 #define SESS_ANYSTATE		0xffff
+#define SESS_RUNNING		(SESS_FREE | SESS_LOGGED_IN | SESS_FAILED)
 
-#define CONN_DONE		0x0000	/* no real state just return value */
 #define	CONN_FREE		0x0001
 #define	CONN_XPT_WAIT		0x0002
 #define	CONN_XPT_UP		0x0004
@@ -82,14 +83,16 @@ TAILQ_HEAD(taskq, task);
 #define	CONN_CLEANUP_WAIT	0x0080
 #define	CONN_IN_CLEANUP		0x0100
 #define CONN_ANYSTATE		0xffff
-#define CONN_NOT_LOGGED_IN	(CONN_FREE | CONN_XPT_WAIT | CONN_XPT_UP)
+#define CONN_RUNNING		(CONN_LOGGED_IN | CONN_LOGOUT_REQ)
+#define CONN_FAILED		(CONN_CLEANUP_WAIT | CONN_IN_CLEANUP)
+#define CONN_NEVER_LOGGED_IN	(CONN_FREE | CONN_XPT_WAIT | CONN_XPT_UP)
 
 enum c_event {
 	CONN_EV_FAIL,
+	CONN_EV_FREE,
 	CONN_EV_CONNECT,
 	CONN_EV_CONNECTED,
 	CONN_EV_LOGGED_IN,
-	CONN_EV_DISCOVERY,
 	CONN_EV_LOGOUT,
 	CONN_EV_LOGGED_OUT,
 	CONN_EV_CLOSED
@@ -97,11 +100,16 @@ enum c_event {
 
 enum s_event {
 	SESS_EV_START,
+	SESS_EV_CONN_LOGGED_IN,
 	SESS_EV_CONN_FAIL,
 	SESS_EV_CONN_CLOSED,
+	SESS_EV_CLOSED,
 	SESS_EV_FAIL
 };
 
+#define SESS_ACT_UP		0
+#define SESS_ACT_DOWN		1
+
 struct pdu {
 	TAILQ_ENTRY(pdu)	 entry;
 	struct iovec		 iov[PDU_MAXIOV];
@@ -220,6 +228,7 @@ struct session {
 	u_int16_t		 tsih;		/* target session id handle */
 	u_int			 target;
 	int			 state;
+	int			 action;
 };
 
 struct connection {
@@ -264,10 +273,12 @@ void	iscsid_ctrl_dispatch(void *, struct pdu *);
 
 struct initiator *initiator_init(void);
 void	initiator_cleanup(struct initiator *);
+void	initiator_shutdown(struct initiator *);
+int	initiator_isdown(struct initiator *);
 struct session *initiator_t2s(u_int);
 void	initiator_login(struct connection *);
 void	initiator_discovery(struct session *);
-void	initiator_logout(struct connection *, u_int8_t, int);
+void	initiator_logout(struct session *, struct connection *, u_int8_t);
 void	initiator_nop_in_imm(struct connection *, struct pdu *);
 char	*default_initiator_name(void);
 
@@ -280,8 +291,10 @@ int	control_compose(void *, u_int16_t, void *, size_t);
 struct session	*session_find(struct initiator *, char *);
 struct session	*session_new(struct initiator *, u_int8_t);
 void	session_cleanup(struct session *);
+int	session_shutdown(struct session *);
 void	session_config(struct session *, struct session_config *);
 void	session_task_issue(struct session *, struct task *);
+void	session_logout_issue(struct session *, struct task *);
 void	session_schedule(struct session *);
 void	session_task_login(struct connection *);
 void	session_fsm(struct session *, enum s_event, struct connection *);
@@ -293,9 +306,7 @@ void	conn_task_issue(struct connection *, struct task *);
 void	conn_task_schedule(struct connection *);
 void	conn_task_cleanup(struct connection *c, struct task *);
 void	conn_pdu_write(struct connection *, struct pdu *);
-void	conn_logout(struct connection *);
 void	conn_fail(struct connection *);
-void	conn_loggedin(struct connection *);
 void	conn_fsm(struct connection *, enum c_event);
 
 void	*pdu_gethdr(struct pdu *);
diff --git a/usr.sbin/iscsid/session.c b/usr.sbin/iscsid/session.c
index ad570f47503..5ce56b1fd36 100644
--- a/usr.sbin/iscsid/session.c
+++ b/usr.sbin/iscsid/session.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: session.c,v 1.2 2011/04/27 07:25:26 claudio Exp $ */
+/*	$OpenBSD: session.c,v 1.3 2011/05/02 06:32:56 claudio Exp $ */
 
 /*
  * Copyright (c) 2011 Claudio Jeker <claudio@openbsd.org>
@@ -16,13 +16,15 @@
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
-#include <sys/param.h>
 #include <sys/types.h>
+#include <sys/ioctl.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/uio.h>
 
 #include <scsi/iscsi.h>
+#include <scsi/scsi_all.h>
+#include <dev/vscsivar.h>
 
 #include <event.h>
 #include <stdio.h>
@@ -35,7 +37,10 @@
 
 void	session_fsm_callback(int, short, void *);
 int	sess_do_start(struct session *, struct sessev *);
-int	sess_do_fail(struct session *, struct sessev *);
+int	sess_do_conn_loggedin(struct session *, struct sessev *);
+int	sess_do_conn_fail(struct session *, struct sessev *);
+int	sess_do_conn_closed(struct session *, struct sessev *);
+int	sess_do_down(struct session *, struct sessev *);
 
 const char *sess_state(int);
 const char *sess_event(enum s_event);
@@ -87,6 +92,8 @@ session_cleanup(struct session *s)
 {
 	struct connection *c;
 
+	taskq_cleanup(&s->tasks);
+
 	while ((c = TAILQ_FIRST(&s->connections)) != NULL)
 		conn_free(c);
 
@@ -95,6 +102,26 @@ session_cleanup(struct session *s)
 	free(s);
 }
 
+int
+session_shutdown(struct session *s)
+{
+	log_debug("session[%s] going down", s->config.SessionName);
+
+	s->action = SESS_ACT_DOWN;
+	if (s->state & (SESS_INIT | SESS_FREE | SESS_DOWN)) {
+		struct connection *c;
+		while ((c = TAILQ_FIRST(&s->connections)) != NULL)
+			conn_free(c);
+		return 0;
+	}
+
+	/* cleanup task queue and issue a logout */
+	taskq_cleanup(&s->tasks);
+	initiator_logout(s, NULL, ISCSI_LOGOUT_CLOSE_SESS);
+
+	return 1;
+}
+
 void
 session_config(struct session *s, struct session_config *sc)
 {
@@ -128,6 +155,32 @@ session_task_issue(struct session *s, struct task *t)
 }
 
 void
+session_logout_issue(struct session *s, struct task *t)
+{
+	struct connection *c, *rc = NULL;
+
+	/* find first free session or first available session */
+	TAILQ_FOREACH(c, &s->connections, entry) {
+		if (conn_task_ready(c)) {
+			conn_fsm(c, CONN_EV_LOGOUT);
+			conn_task_issue(c, t);
+			return;
+		}
+		if (c->state & CONN_RUNNING)
+			rc = c;
+	}
+
+	if (rc) {
+		conn_fsm(rc, CONN_EV_LOGOUT);
+		conn_task_issue(rc, t);
+		return;
+	}
+
+	/* XXX must open new connection, gulp */
+	fatalx("session_logout_issue needs more work");
+}
+
+void
 session_schedule(struct session *s)
 {
 	struct task *t = TAILQ_FIRST(&s->tasks);
@@ -154,7 +207,7 @@ session_schedule(struct session *s)
 void
 session_fsm(struct session *s, enum s_event ev, struct connection *c)
 {
-	struct timeval  tv;
+	struct timeval tv;
 	struct sessev *sev;
 
 	if ((sev = malloc(sizeof(*sev))) == NULL)
@@ -174,8 +227,11 @@ struct {
 	int		(*action)(struct session *, struct sessev *);
 } s_fsm[] = {
 	{ SESS_INIT, SESS_EV_START, sess_do_start },
-	{ SESS_FREE, SESS_EV_CONN_FAIL, sess_do_fail },
-	{ SESS_FREE, SESS_EV_CONN_CLOSED, sess_do_fail },
+	{ SESS_FREE, SESS_EV_CONN_LOGGED_IN, sess_do_conn_loggedin },
+	{ SESS_LOGGED_IN, SESS_EV_CONN_LOGGED_IN, sess_do_conn_loggedin },
+	{ SESS_RUNNING, SESS_EV_CONN_FAIL, sess_do_conn_fail },
+	{ SESS_RUNNING, SESS_EV_CONN_CLOSED, sess_do_conn_closed },
+	{ SESS_RUNNING, SESS_EV_CLOSED, sess_do_down },
 	{ 0, 0, NULL }
 };
 
@@ -223,12 +279,34 @@ sess_do_start(struct session *s, struct sessev *sev)
 	    log_sockaddr(&s->config.connection.TargetAddr));
 	conn_new(s, &s->config.connection);
 
-	return (SESS_FREE);
+	return SESS_FREE;
+}
+
+int
+sess_do_conn_loggedin(struct session *s, struct sessev *sev)
+{
+	if (s->state & SESS_LOGGED_IN)
+		return SESS_LOGGED_IN;
+
+	if (s->config.SessionType == SESSION_TYPE_DISCOVERY)
+		initiator_discovery(s);
+	else
+		vscsi_event(VSCSI_REQPROBE, s->target, -1);
+
+	return SESS_LOGGED_IN;
 }
 
 int
-sess_do_fail(struct session *s, struct sessev *sev)
+sess_do_conn_fail(struct session *s, struct sessev *sev)
 {
+	struct connection *c = sev->conn;
+	int state = SESS_FREE;
+
+	if (sev->conn == NULL) {
+		log_warnx("Just what do you think you're doing, Dave?");
+		return -1;
+	}
+
 	/*
 	 * cleanup connections:
 	 * Connections in state FREE can be removed.
@@ -236,7 +314,63 @@ sess_do_fail(struct session *s, struct sessev *sev)
 	 * the FAILED state. If no sessions are left and the session was
 	 * not already FREE then explicit recovery needs to be done.
 	 */
-	return (SESS_FREE);
+
+	switch (c->state) {
+	case CONN_FREE:
+		conn_free(c);
+		break;
+	case CONN_CLEANUP_WAIT:
+		break;
+	default:
+		log_warnx("It can only be attributable to human error.");
+		return -1;
+	}
+
+	TAILQ_FOREACH(c, &s->connections, entry) {
+		if (c->state & CONN_FAILED) {
+			state = SESS_FAILED;
+			break;
+		} else if (c->state & CONN_RUNNING)
+			state = SESS_LOGGED_IN;
+	}
+
+	return state;
+}
+
+int
+sess_do_conn_closed(struct session *s, struct sessev *sev)
+{
+	struct connection *c = sev->conn;
+	int state = SESS_FREE;
+
+	if (c == NULL || c->state != CONN_FREE) {
+		log_warnx("Just what do you think you're doing, Dave?");
+		return -1;
+	}
+	conn_free(c);
+
+	TAILQ_FOREACH(c, &s->connections, entry) {
+		if (c->state & CONN_FAILED) {
+			state = SESS_FAILED;
+			break;
+		} else if (c->state & CONN_RUNNING)
+			state = SESS_LOGGED_IN;
+	}
+
+	return state;
+}
+
+int
+sess_do_down(struct session *s, struct sessev *sev)
+{
+	struct connection *c;
+
+	while ((c = TAILQ_FIRST(&s->connections)) != NULL)
+		conn_free(c);
+
+	/* XXX anything else to reset to initial state? */
+
+	return SESS_DOWN;
 }
 
 const char *
@@ -253,6 +387,8 @@ sess_state(int s)
 		return "LOGGED_IN";
 	case SESS_FAILED:
 		return "FAILED";
+	case SESS_DOWN:
+		return "DOWN";
 	default:
 		snprintf(buf, sizeof(buf), "UKNWN %x", s);
 		return buf;
@@ -268,10 +404,14 @@ sess_event(enum s_event e)
 	switch (e) {
 	case SESS_EV_START:
 		return "start";
+	case SESS_EV_CONN_LOGGED_IN:
+		return "connection logged in";
 	case SESS_EV_CONN_FAIL:
 		return "connection fail";
 	case SESS_EV_CONN_CLOSED:
 		return "connection closed";
+	case SESS_EV_CLOSED:
+		return "session closed";
 	case SESS_EV_FAIL:
 		return "fail";
 	}