Fix printing long doubles on architectures with hm and lm bits.

Issue reported with initial patch by enh@google.com.

ok deraadt@

2 files changed, 22 insertions, 12 deletions

diff --git a/lib/libc/gdtoa/hdtoa.c b/lib/libc/gdtoa/hdtoa.c
index 43e24474d64..4a7f798e204 100644
--- a/lib/libc/gdtoa/hdtoa.c
+++ b/lib/libc/gdtoa/hdtoa.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: hdtoa.c,v 1.4 2018/04/27 13:46:01 guenther Exp $	*/
+/*	$OpenBSD: hdtoa.c,v 1.5 2020/05/31 12:27:19 mortimer Exp $	*/
 /*-
  * Copyright (c) 2004, 2005 David Schultz <das@FreeBSD.ORG>
  * All rights reserved.
@@ -225,6 +225,7 @@ __hldtoa(long double e, const char *xdigs, int ndigits, int *decpt, int *sign,
 	struct ieee_ext *p = (struct ieee_ext *)&e;
 	char *s, *s0;
 	int bufsize;
+	int fbits = 0;
 
 	*sign = p->ext_sign;
 
@@ -273,23 +274,24 @@ __hldtoa(long double e, const char *xdigs, int ndigits, int *decpt, int *sign,
 	 */
 	for (s = s0 + bufsize - 1; s > s0 + sigfigs - 1; s--)
 		*s = 0;
-	for (; s > s0 + sigfigs - (EXT_FRACLBITS / 4) - 1 && s > s0; s--) {
+
+	for (fbits = EXT_FRACLBITS / 4; fbits > 0 && s > s0; s--, fbits--) {
 		*s = p->ext_fracl & 0xf;
 		p->ext_fracl >>= 4;
 	}
-#ifdef EXT_FRACHMBITS
-	for (; s > s0; s--) {
-		*s = p->ext_frachm & 0xf;
-		p->ext_frachm >>= 4;
-	}
-#endif
 #ifdef EXT_FRACLMBITS
-	for (; s > s0; s--) {
+	for (fbits = EXT_FRACLMBITS / 4; fbits > 0 && s > s0; s--, fbits--) {
 		*s = p->ext_fraclm & 0xf;
 		p->ext_fraclm >>= 4;
 	}
 #endif
-	for (; s > s0; s--) {
+#ifdef EXT_FRACHMBITS
+	for (fbits = EXT_FRACHMBITS / 4; fbits > 0 && s > s0; s--, fbits--) {
+		*s = p->ext_frachm & 0xf;
+		p->ext_frachm >>= 4;
+	}
+#endif
+	for (fbits = EXT_FRACHBITS / 4; fbits > 0 && s > s0; s--, fbits--) {
 		*s = p->ext_frach & 0xf;
 		p->ext_frach >>= 4;
 	}
@@ -300,7 +302,7 @@ __hldtoa(long double e, const char *xdigs, int ndigits, int *decpt, int *sign,
 	 * (partial) nibble, which is dealt with by the next
 	 * statement.  We also tack on the implicit normalization bit.
 	 */
-	*s = p->ext_frach | (1U << ((LDBL_MANT_DIG - 1) % 4));
+	*s = (p->ext_frach | (1U << ((LDBL_MANT_DIG - 1) % 4))) & 0xf;
 
 	/* If ndigits < 0, we are expected to auto-size the precision. */
 	if (ndigits < 0) {
diff --git a/regress/lib/libc/printf/fp.c b/regress/lib/libc/printf/fp.c
index 6ed52fdb494..299d05f55d4 100644
--- a/regress/lib/libc/printf/fp.c
+++ b/regress/lib/libc/printf/fp.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: fp.c,v 1.1 2008/09/07 20:36:10 martynas Exp $	*/
+/*	$OpenBSD: fp.c,v 1.2 2020/05/31 12:27:19 mortimer Exp $	*/
 /*-
  * Copyright (c) 2002, 2005 David Schultz <das@FreeBSD.org>
  * All rights reserved.
@@ -185,6 +185,14 @@ main(int argc, char *argv[])
 	testfmt("0x1p-1074", "%a", 0x1p-1074);
 	testfmt("0x1.2345p-1024", "%a", 0x1.2345p-1024);
 
+#if LDBL_MANT_DIG == 113
+	testfmt("-0x1.e7d7c7b7a7978777675747372717p-14344", "%La",
+			-0x1.e7d7c7b7a7978777675747372717p-14344L);
+#elif LDBL_MANT_DIG == 64
+	testfmt("-0x8.777675747372717p-16248", "%La",
+			-0x8.777675747372717p-16248L);
+#endif
+
 	return (0);
 }