Up to now, tmux sees \033\033[OA as M-Up and since we turned on

xterm-keys by default, generates \033[1;3A instead of
\033\033[OA. Unfortunately this confuses vi, which doesn't understand
xterm keys and now sees Escape+Up pressed within escape-time as Escape
followed by A.

The issue doesn't happen in xterm itself because it gets the keys from X
and can distinguish between a genuine M-Up and Escape+Up.

Because xterm can, tmux can too: xterm will give us \033[1;3A (that is,
kUP3) for a real M-Up and \033\033OA for Escape+Up - in fact, we can be
sure any \033 preceding an xterm key is a real Escape key press because
Meta would be part of the xterm key instead of a separate \033.

So change tmux to recognise both sequences as M-Up for its own purposes,
but generate the xterm version of M-Up only if it originally received
the xterm version from the terminal.

This means we will return to sending \033\033OA instead of the xterm key
for terminals that do not support xterm keys themselves, but there is no
practical way around this because they do not allow us to distinguish
between Escape+Up and M-Up. xterm style escape sequences are now the de
facto standard for these keys in any case.

Problem reported by jsing@ and subsequently by Cecile Tonglet in GitHub
issue 907.

6 files changed, 106 insertions, 80 deletions

diff --git a/usr.bin/tmux/input-keys.c b/usr.bin/tmux/input-keys.c
index 91a2af9efe8..284fddbeb25 100644
--- a/usr.bin/tmux/input-keys.c
+++ b/usr.bin/tmux/input-keys.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: input-keys.c,v 1.59 2017/02/01 09:55:07 nicm Exp $ */
+/* $OpenBSD: input-keys.c,v 1.60 2017/05/07 21:25:59 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -201,6 +201,7 @@ input_key(struct window_pane *wp, key_code key, struct mouse_event *m)
 			return;
 		}
 	}
+	key &= ~KEYC_XTERM;
 
 	/* Otherwise look the key up in the table. */
 	for (i = 0; i < nitems(input_keys); i++) {
diff --git a/usr.bin/tmux/key-bindings.c b/usr.bin/tmux/key-bindings.c
index f0e0e1a950d..a54cbd11b4a 100644
--- a/usr.bin/tmux/key-bindings.c
+++ b/usr.bin/tmux/key-bindings.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: key-bindings.c,v 1.76 2017/04/21 19:33:07 nicm Exp $ */
+/* $OpenBSD: key-bindings.c,v 1.77 2017/05/07 21:25:59 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -92,7 +92,7 @@ key_bindings_add(const char *name, key_code key, int repeat,
 
 	table = key_bindings_get_table(name, 1);
 
-	bd_find.key = key;
+	bd_find.key = (key & ~KEYC_XTERM);
 	bd = RB_FIND(key_bindings, &table->key_bindings, &bd_find);
 	if (bd != NULL) {
 		RB_REMOVE(key_bindings, &table->key_bindings, bd);
@@ -119,7 +119,7 @@ key_bindings_remove(const char *name, key_code key)
 	if (table == NULL)
 		return;
 
-	bd_find.key = key;
+	bd_find.key = (key & ~KEYC_XTERM);
 	bd = RB_FIND(key_bindings, &table->key_bindings, &bd_find);
 	if (bd == NULL)
 		return;
diff --git a/usr.bin/tmux/server-client.c b/usr.bin/tmux/server-client.c
index ce9956db60d..e33b70d920a 100644
--- a/usr.bin/tmux/server-client.c
+++ b/usr.bin/tmux/server-client.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: server-client.c,v 1.228 2017/05/01 12:20:55 nicm Exp $ */
+/* $OpenBSD: server-client.c,v 1.229 2017/05/07 21:25:59 nicm Exp $ */
 
 /*
  * Copyright (c) 2009 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -904,7 +904,7 @@ retry:
 		log_debug("currently repeating");
 
 	/* Try to see if there is a key binding in the current table. */
-	bd_find.key = key;
+	bd_find.key = (key & ~KEYC_XTERM);
 	bd = RB_FIND(key_bindings, &table->key_bindings, &bd_find);
 	if (bd != NULL) {
 		/*
diff --git a/usr.bin/tmux/tmux.h b/usr.bin/tmux/tmux.h
index 13c8dbc79ae..71309d6364a 100644
--- a/usr.bin/tmux/tmux.h
+++ b/usr.bin/tmux/tmux.h
@@ -1,4 +1,4 @@
-/* $OpenBSD: tmux.h,v 1.758 2017/05/04 07:16:43 nicm Exp $ */
+/* $OpenBSD: tmux.h,v 1.759 2017/05/07 21:25:59 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -91,9 +91,10 @@ struct tmuxproc;
 #define KEYC_ESCAPE 0x200000000000ULL
 #define KEYC_CTRL   0x400000000000ULL
 #define KEYC_SHIFT  0x800000000000ULL
+#define KEYC_XTERM 0x1000000000000ULL
 
 /* Mask to obtain key w/o modifiers. */
-#define KEYC_MASK_MOD (KEYC_ESCAPE|KEYC_CTRL|KEYC_SHIFT)
+#define KEYC_MASK_MOD (KEYC_ESCAPE|KEYC_CTRL|KEYC_SHIFT|KEYC_XTERM)
 #define KEYC_MASK_KEY (~KEYC_MASK_MOD)
 
 /* Is this a mouse key? */
diff --git a/usr.bin/tmux/tty-keys.c b/usr.bin/tmux/tty-keys.c
index 0571bb60f97..cd5a196b1ff 100644
--- a/usr.bin/tmux/tty-keys.c
+++ b/usr.bin/tmux/tty-keys.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: tty-keys.c,v 1.96 2017/04/18 21:41:42 nicm Exp $ */
+/* $OpenBSD: tty-keys.c,v 1.97 2017/05/07 21:25:59 nicm Exp $ */
 
 /*
  * Copyright (c) 2007 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -257,67 +257,70 @@ static const struct tty_default_key_code tty_default_code_keys[] = {
 	{ TTYC_KCUB1, KEYC_LEFT },
 	{ TTYC_KCUF1, KEYC_RIGHT },
 
-	/* Key and modifier capabilities. */
-	{ TTYC_KDC2, KEYC_DC|KEYC_SHIFT },
-	{ TTYC_KDC3, KEYC_DC|KEYC_ESCAPE },
-	{ TTYC_KDC4, KEYC_DC|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KDC5, KEYC_DC|KEYC_CTRL },
-	{ TTYC_KDC6, KEYC_DC|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KDC7, KEYC_DC|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KDN2, KEYC_DOWN|KEYC_SHIFT },
-	{ TTYC_KDN3, KEYC_DOWN|KEYC_ESCAPE },
-	{ TTYC_KDN4, KEYC_DOWN|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KDN5, KEYC_DOWN|KEYC_CTRL },
-	{ TTYC_KDN6, KEYC_DOWN|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KDN7, KEYC_DOWN|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KEND2, KEYC_END|KEYC_SHIFT },
-	{ TTYC_KEND3, KEYC_END|KEYC_ESCAPE },
-	{ TTYC_KEND4, KEYC_END|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KEND5, KEYC_END|KEYC_CTRL },
-	{ TTYC_KEND6, KEYC_END|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KEND7, KEYC_END|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KHOM2, KEYC_HOME|KEYC_SHIFT },
-	{ TTYC_KHOM3, KEYC_HOME|KEYC_ESCAPE },
-	{ TTYC_KHOM4, KEYC_HOME|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KHOM5, KEYC_HOME|KEYC_CTRL },
-	{ TTYC_KHOM6, KEYC_HOME|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KHOM7, KEYC_HOME|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KIC2, KEYC_IC|KEYC_SHIFT },
-	{ TTYC_KIC3, KEYC_IC|KEYC_ESCAPE },
-	{ TTYC_KIC4, KEYC_IC|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KIC5, KEYC_IC|KEYC_CTRL },
-	{ TTYC_KIC6, KEYC_IC|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KIC7, KEYC_IC|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KLFT2, KEYC_LEFT|KEYC_SHIFT },
-	{ TTYC_KLFT3, KEYC_LEFT|KEYC_ESCAPE },
-	{ TTYC_KLFT4, KEYC_LEFT|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KLFT5, KEYC_LEFT|KEYC_CTRL },
-	{ TTYC_KLFT6, KEYC_LEFT|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KLFT7, KEYC_LEFT|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KNXT2, KEYC_NPAGE|KEYC_SHIFT },
-	{ TTYC_KNXT3, KEYC_NPAGE|KEYC_ESCAPE },
-	{ TTYC_KNXT4, KEYC_NPAGE|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KNXT5, KEYC_NPAGE|KEYC_CTRL },
-	{ TTYC_KNXT6, KEYC_NPAGE|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KNXT7, KEYC_NPAGE|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KPRV2, KEYC_PPAGE|KEYC_SHIFT },
-	{ TTYC_KPRV3, KEYC_PPAGE|KEYC_ESCAPE },
-	{ TTYC_KPRV4, KEYC_PPAGE|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KPRV5, KEYC_PPAGE|KEYC_CTRL },
-	{ TTYC_KPRV6, KEYC_PPAGE|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KPRV7, KEYC_PPAGE|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KRIT2, KEYC_RIGHT|KEYC_SHIFT },
-	{ TTYC_KRIT3, KEYC_RIGHT|KEYC_ESCAPE },
-	{ TTYC_KRIT4, KEYC_RIGHT|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KRIT5, KEYC_RIGHT|KEYC_CTRL },
-	{ TTYC_KRIT6, KEYC_RIGHT|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KRIT7, KEYC_RIGHT|KEYC_ESCAPE|KEYC_CTRL },
-	{ TTYC_KUP2, KEYC_UP|KEYC_SHIFT },
-	{ TTYC_KUP3, KEYC_UP|KEYC_ESCAPE },
-	{ TTYC_KUP4, KEYC_UP|KEYC_SHIFT|KEYC_ESCAPE },
-	{ TTYC_KUP5, KEYC_UP|KEYC_CTRL },
-	{ TTYC_KUP6, KEYC_UP|KEYC_SHIFT|KEYC_CTRL },
-	{ TTYC_KUP7, KEYC_UP|KEYC_ESCAPE|KEYC_CTRL },
+	/*
+	 * Key and modifier capabilities. We set the xterm flag to mark that
+	 * any leading escape means an escape key press and not the modifier.
+	 */
+	{ TTYC_KDC2, KEYC_DC|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KDC3, KEYC_DC|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDC4, KEYC_DC|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDC5, KEYC_DC|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDC6, KEYC_DC|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDC7, KEYC_DC|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDN2, KEYC_DOWN|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KDN3, KEYC_DOWN|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDN4, KEYC_DOWN|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KDN5, KEYC_DOWN|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDN6, KEYC_DOWN|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KDN7, KEYC_DOWN|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KEND2, KEYC_END|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KEND3, KEYC_END|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KEND4, KEYC_END|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KEND5, KEYC_END|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KEND6, KEYC_END|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KEND7, KEYC_END|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KHOM2, KEYC_HOME|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KHOM3, KEYC_HOME|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KHOM4, KEYC_HOME|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KHOM5, KEYC_HOME|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KHOM6, KEYC_HOME|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KHOM7, KEYC_HOME|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KIC2, KEYC_IC|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KIC3, KEYC_IC|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KIC4, KEYC_IC|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KIC5, KEYC_IC|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KIC6, KEYC_IC|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KIC7, KEYC_IC|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KLFT2, KEYC_LEFT|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KLFT3, KEYC_LEFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KLFT4, KEYC_LEFT|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KLFT5, KEYC_LEFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KLFT6, KEYC_LEFT|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KLFT7, KEYC_LEFT|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KNXT2, KEYC_NPAGE|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KNXT3, KEYC_NPAGE|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KNXT4, KEYC_NPAGE|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KNXT5, KEYC_NPAGE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KNXT6, KEYC_NPAGE|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KNXT7, KEYC_NPAGE|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KPRV2, KEYC_PPAGE|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KPRV3, KEYC_PPAGE|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KPRV4, KEYC_PPAGE|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KPRV5, KEYC_PPAGE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KPRV6, KEYC_PPAGE|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KPRV7, KEYC_PPAGE|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KRIT2, KEYC_RIGHT|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KRIT3, KEYC_RIGHT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KRIT4, KEYC_RIGHT|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KRIT5, KEYC_RIGHT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KRIT6, KEYC_RIGHT|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KRIT7, KEYC_RIGHT|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KUP2, KEYC_UP|KEYC_SHIFT|KEYC_XTERM },
+	{ TTYC_KUP3, KEYC_UP|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KUP4, KEYC_UP|KEYC_SHIFT|KEYC_ESCAPE|KEYC_XTERM },
+	{ TTYC_KUP5, KEYC_UP|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KUP6, KEYC_UP|KEYC_SHIFT|KEYC_CTRL|KEYC_XTERM },
+	{ TTYC_KUP7, KEYC_UP|KEYC_ESCAPE|KEYC_CTRL|KEYC_XTERM },
 };
 
 /* Add key to tree. */
@@ -476,6 +479,7 @@ tty_keys_next1(struct tty *tty, const char *buf, size_t len, key_code *key,
 	enum utf8_state		 more;
 	u_int			 i;
 	wchar_t			 wc;
+	int			 n;
 
 	log_debug("%s: next key is %zu (%.*s) (expired=%d)", c->name, len,
 	    (int)len, buf, expired);
@@ -493,6 +497,13 @@ tty_keys_next1(struct tty *tty, const char *buf, size_t len, key_code *key,
 		return (0);
 	}
 
+	/* Is this an an xterm(1) key? */
+	n = xterm_keys_find(buf, len, size, key);
+	if (n == 0)
+		return (0);
+	if (n == 1 && !expired)
+		return (1);
+
 	/* Is this valid UTF-8? */
 	more = utf8_open(&ud, (u_char)*buf);
 	if (more == UTF8_MORE) {
@@ -573,6 +584,18 @@ first_key:
 		/* Look for a key without the escape. */
 		n = tty_keys_next1(tty, buf + 1, len - 1, &key, &size, expired);
 		if (n == 0) {	/* found */
+			if (key & KEYC_XTERM) {
+				/*
+				 * We want the escape key as well as the xterm
+				 * key, because the xterm sequence implicitly
+				 * includes the escape (so if we see
+				 * \033\033[1;3D we know it is an Escape
+				 * followed by M-Left, not just M-Left).
+				 */
+				key = '\033';
+				size = 1;
+				goto complete_key;
+			}
 			key |= KEYC_ESCAPE;
 			size++;
 			goto complete_key;
@@ -588,13 +611,6 @@ first_key:
 	if (n == 1)
 		goto partial_key;
 
-	/* Is this an an xterm(1) key? */
-	n = xterm_keys_find(buf, len, &size, &key);
-	if (n == 0)
-		goto complete_key;
-	if (n == 1 && !expired)
-		goto partial_key;
-
 	/*
 	 * At this point, we know the key is not partial (with or without
 	 * escape). So pass it through even if the timer has not expired.
diff --git a/usr.bin/tmux/xterm-keys.c b/usr.bin/tmux/xterm-keys.c
index 34962ecec58..36e7542ba46 100644
--- a/usr.bin/tmux/xterm-keys.c
+++ b/usr.bin/tmux/xterm-keys.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: xterm-keys.c,v 1.21 2017/01/25 14:36:08 nicm Exp $ */
+/* $OpenBSD: xterm-keys.c,v 1.22 2017/05/07 21:25:59 nicm Exp $ */
 
 /*
  * Copyright (c) 2009 Nicholas Marriott <nicholas.marriott@gmail.com>
@@ -197,7 +197,7 @@ xterm_keys_find(const char *buf, size_t len, size_t *size, key_code *key)
 		if (matched == -1)
 			continue;
 		if (matched == 0)
-			*key = entry->key | modifiers;
+			*key = (entry->key|modifiers|KEYC_XTERM);
 		return (matched);
 	}
 	return (-1);
@@ -227,8 +227,16 @@ xterm_keys_lookup(key_code key)
 	if (modifiers == 1)
 		return (NULL);
 
+	/*
+	 * If this has the escape modifier, but was not originally an xterm
+	 * key, it may be a genuine escape + key. So don't pass it through as
+	 * an xterm key or programs like vi may be confused.
+	 */
+	if ((key & (KEYC_ESCAPE|KEYC_XTERM)) == KEYC_ESCAPE)
+		return (NULL);
+
 	/* Otherwise, find the key in the table. */
-	key &= ~(KEYC_SHIFT|KEYC_ESCAPE|KEYC_CTRL);
+	key &= KEYC_MASK_KEY;
 	for (i = 0; i < nitems(xterm_keys_table); i++) {
 		entry = &xterm_keys_table[i];
 		if (key == entry->key)