Use mtrrmask instead of hardcoded values to accommodate newer CPUs

with a large physical address size (greater than 36 bit). Fixes a
major performance hit seen on newer servers where an incorrectly
programmed memory region length affects the PCI device mappings.

While here, make sure to invalidate the TLB after programming MSRs
and fix an incorrect behavior found by deraadt@ where MTRRdefType
was updated outside of the protected region.

The fix was partly obtained from FreeBSD, tested by many.
With and OK deraadt

2 files changed, 14 insertions, 14 deletions

diff --git a/sys/arch/amd64/amd64/amd64_mem.c b/sys/arch/amd64/amd64/amd64_mem.c
index 7cfe40a8711..94e7705a124 100644
--- a/sys/arch/amd64/amd64/amd64_mem.c
+++ b/sys/arch/amd64/amd64/amd64_mem.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: amd64_mem.c,v 1.5 2012/07/08 22:11:39 deraadt Exp $ */
+/* $OpenBSD: amd64_mem.c,v 1.6 2012/08/01 15:44:14 mikeb Exp $ */
 /*-
  * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
  * All rights reserved.
@@ -291,7 +291,7 @@ amd64_mrstoreone(struct mem_range_softc *sc)
 	if (cr4save & CR4_PGE)
 		lcr4(cr4save & ~CR4_PGE);
 	lcr0((rcr0() & ~CR0_NW) | CR0_CD); /* disable caches (CD = 1, NW = 0) */
-	wbinvd();		/* flush caches, TLBs */
+	wbinvd();		/* flush caches */
 	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~0x800);	/* disable MTRRs (E = 0) */
 	
 	/* Set fixed-range MTRRs */
@@ -340,7 +340,7 @@ amd64_mrstoreone(struct mem_range_softc *sc)
 		/* base/type register */
 		omsrv = rdmsr(msr);
 		if (mrd->mr_flags & MDF_ACTIVE) {
-			msrv = mrd->mr_base & 0xfffffffffffff000LL;
+			msrv = mrd->mr_base & mtrrmask;
 			msrv |= amd64_mrt2mtrr(mrd->mr_flags, omsrv);
 		} else {
 			msrv = 0;
@@ -349,14 +349,15 @@ amd64_mrstoreone(struct mem_range_softc *sc)
 		
 		/* mask/active register */
 		if (mrd->mr_flags & MDF_ACTIVE) {
-			msrv = 0x800 | (~(mrd->mr_len - 1) & 0x0000000ffffff000LL);
+			msrv = 0x800 | (~(mrd->mr_len - 1) & mtrrmask);
 		} else {
 			msrv = 0;
 		}
 		wrmsr(msr + 1, msrv);
 	}
-	wbinvd();							/* flush caches, TLBs */
-	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | 0x800);	/* restore MTRR state */
+	wbinvd();						/* flush caches */
+	tlbflushg();						/* flush TLB */
+	wrmsr(MSR_MTRRdefType, mtrrdef | 0x800);		/* set MTRR behaviour to match BSP and enable it */
 	lcr0(rcr0() & ~(CR0_CD | CR0_NW));  			/* enable caches CD = 0 and NW = 0 */
 	lcr4(cr4save);						/* restore cr4 */
 }
@@ -615,7 +616,6 @@ void
 amd64_mrinit_cpu(struct mem_range_softc *sc)
 {
 	amd64_mrstoreone(sc); /* set MTRRs to match BSP */
-	wrmsr(MSR_MTRRdefType, mtrrdef); /* set MTRR behaviour to match BSP */
 }
 
 void
diff --git a/sys/arch/i386/i386/i686_mem.c b/sys/arch/i386/i386/i686_mem.c
index 8997e4b35b7..3a38cf751e4 100644
--- a/sys/arch/i386/i386/i686_mem.c
+++ b/sys/arch/i386/i386/i686_mem.c
@@ -1,4 +1,4 @@
-/* $OpenBSD: i686_mem.c,v 1.13 2010/03/23 19:31:18 kettenis Exp $ */
+/* $OpenBSD: i686_mem.c,v 1.14 2012/08/01 15:44:14 mikeb Exp $ */
 /*-
  * Copyright (c) 1999 Michael Smith <msmith@freebsd.org>
  * All rights reserved.
@@ -291,7 +291,7 @@ i686_mrstoreone(struct mem_range_softc *sc)
 	if (cr4save & CR4_PGE)
 		lcr4(cr4save & ~CR4_PGE);
 	lcr0((rcr0() & ~CR0_NW) | CR0_CD); /* disable caches (CD = 1, NW = 0) */
-	wbinvd();		/* flush caches, TLBs */
+	wbinvd();		/* flush caches */
 	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) & ~0x800);	/* disable MTRRs (E = 0) */
 	
 	/* Set fixed-range MTRRs */
@@ -340,7 +340,7 @@ i686_mrstoreone(struct mem_range_softc *sc)
 		/* base/type register */
 		omsrv = rdmsr(msr);
 		if (mrd->mr_flags & MDF_ACTIVE) {
-			msrv = mrd->mr_base & 0x0000000ffffff000LL;
+			msrv = mrd->mr_base & mtrrmask;
 			msrv |= i686_mrt2mtrr(mrd->mr_flags, omsrv);
 		} else {
 			msrv = 0;
@@ -349,14 +349,15 @@ i686_mrstoreone(struct mem_range_softc *sc)
 		
 		/* mask/active register */
 		if (mrd->mr_flags & MDF_ACTIVE) {
-			msrv = 0x800 | (~(mrd->mr_len - 1) & 0x0000000ffffff000LL);
+			msrv = 0x800 | (~(mrd->mr_len - 1) & mtrrmask);
 		} else {
 			msrv = 0;
 		}
 		wrmsr(msr + 1, msrv);
 	}
-	wbinvd();							/* flush caches, TLBs */
-	wrmsr(MSR_MTRRdefType, rdmsr(MSR_MTRRdefType) | 0x800);	/* restore MTRR state */
+	wbinvd();					/* flush caches */
+	tlbflushg();					/* flush TLB */
+	wrmsr(MSR_MTRRdefType, mtrrdef | 0x800);	/* restore MTRR state */
 	lcr0(rcr0() & ~(CR0_CD | CR0_NW));  			/* enable caches CD = 0 and NW = 0 */
 	lcr4(cr4save);						/* restore cr4 */
 }
@@ -616,7 +617,6 @@ void
 i686_mrinit_cpu(struct mem_range_softc *sc)
 {
 	i686_mrstoreone(sc); /* set MTRRs to match BSP */
-	wrmsr(MSR_MTRRdefType, mtrrdef); /* set MTRR behaviour to match BSP */
 }
 
 void