twist the amd64 bus_space implementation a bit. instead of using the tag

as an identifier for IO or memory mappings that was checked inside the api,
turn it into a pointer to a structure of function pointers. the api then
generally becomes a set of macros that deref the function pointers on the
callers behalf. the idea is that following a pointer to very small
functions is cheap compared to doing compares continuously.

a notable exception to the macro wrappers is bus_space_barrier which is now
a static inline function. the only argument to it that gets used is the
flags, and thats usually a constant at compile time so it can be inlined
to a single fence instruction.

the kernel is smaller and the api is more cache friendly now.

ok deraadt@

2 files changed, 1124 insertions, 657 deletions

diff --git a/sys/arch/amd64/amd64/bus_space.c b/sys/arch/amd64/amd64/bus_space.c
index 9ef983558d8..db97076b29b 100644
--- a/sys/arch/amd64/amd64/bus_space.c
+++ b/sys/arch/amd64/amd64/bus_space.c
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus_space.c,v 1.21 2013/01/15 09:33:30 dlg Exp $	*/
+/*	$OpenBSD: bus_space.c,v 1.22 2013/01/18 01:54:40 dlg Exp $	*/
 /*	$NetBSD: bus_space.c,v 1.2 2003/03/14 18:47:53 christos Exp $	*/
 
 /*-
@@ -64,6 +64,249 @@ static	int ioport_malloc_safe;
 int	x86_mem_add_mapping(bus_addr_t, bus_size_t,
 	    int, bus_space_handle_t *);
 
+u_int8_t	x86_bus_space_io_read_1(bus_space_handle_t, bus_size_t);
+u_int16_t	x86_bus_space_io_read_2(bus_space_handle_t, bus_size_t);
+u_int32_t	x86_bus_space_io_read_4(bus_space_handle_t, bus_size_t);
+u_int64_t	x86_bus_space_io_read_8(bus_space_handle_t, bus_size_t);
+
+void		x86_bus_space_io_read_multi_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t *, bus_size_t);
+void		x86_bus_space_io_read_multi_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t *, bus_size_t);
+void		x86_bus_space_io_read_multi_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t *, bus_size_t);
+void		x86_bus_space_io_read_multi_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t *, bus_size_t);
+
+void		x86_bus_space_io_read_region_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t *, bus_size_t);
+void		x86_bus_space_io_read_region_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t *, bus_size_t);
+void		x86_bus_space_io_read_region_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t *, bus_size_t);
+void		x86_bus_space_io_read_region_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t *, bus_size_t);
+
+void		x86_bus_space_io_write_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t);
+void		x86_bus_space_io_write_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t);
+void		x86_bus_space_io_write_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t);
+void		x86_bus_space_io_write_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t);
+
+void		x86_bus_space_io_write_multi_1(bus_space_handle_t,
+		    bus_size_t, const u_int8_t *, bus_size_t);
+void		x86_bus_space_io_write_multi_2(bus_space_handle_t,
+		    bus_size_t, const u_int16_t *, bus_size_t);
+void		x86_bus_space_io_write_multi_4(bus_space_handle_t,
+		    bus_size_t, const u_int32_t *, bus_size_t);
+void		x86_bus_space_io_write_multi_8(bus_space_handle_t,
+		    bus_size_t, const u_int64_t *, bus_size_t);
+
+void		x86_bus_space_io_write_region_1(bus_space_handle_t,
+		    bus_size_t, const u_int8_t *, bus_size_t);
+void		x86_bus_space_io_write_region_2(bus_space_handle_t,
+		    bus_size_t, const u_int16_t *, bus_size_t);
+void		x86_bus_space_io_write_region_4(bus_space_handle_t,
+		    bus_size_t, const u_int32_t *, bus_size_t);
+void		x86_bus_space_io_write_region_8(bus_space_handle_t,
+		    bus_size_t, const u_int64_t *, bus_size_t);
+
+void		x86_bus_space_io_set_multi_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t, size_t);
+void		x86_bus_space_io_set_multi_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t, size_t);
+void		x86_bus_space_io_set_multi_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t, size_t);
+void		x86_bus_space_io_set_multi_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t, size_t);
+
+void		x86_bus_space_io_set_region_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t, size_t);
+void		x86_bus_space_io_set_region_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t, size_t);
+void		x86_bus_space_io_set_region_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t, size_t);
+void		x86_bus_space_io_set_region_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t, size_t);
+
+void		x86_bus_space_io_copy_1(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+void		x86_bus_space_io_copy_2(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+void		x86_bus_space_io_copy_4(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+void		x86_bus_space_io_copy_8(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+
+void *		x86_bus_space_io_vaddr(bus_space_handle_t);
+paddr_t		x86_bus_space_io_mmap(bus_addr_t, off_t, int, int);
+
+const struct x86_bus_space_ops x86_bus_space_io_ops = {
+	x86_bus_space_io_read_1,
+	x86_bus_space_io_read_2,
+	x86_bus_space_io_read_4,
+	x86_bus_space_io_read_8,
+	x86_bus_space_io_read_multi_1,
+	x86_bus_space_io_read_multi_2,
+	x86_bus_space_io_read_multi_4,
+	x86_bus_space_io_read_multi_8,
+	x86_bus_space_io_read_region_1,
+	x86_bus_space_io_read_region_2,
+	x86_bus_space_io_read_region_4,
+	x86_bus_space_io_read_region_8,
+	x86_bus_space_io_write_1,
+	x86_bus_space_io_write_2,
+	x86_bus_space_io_write_4,
+	x86_bus_space_io_write_8,
+	x86_bus_space_io_write_multi_1,
+	x86_bus_space_io_write_multi_2,
+	x86_bus_space_io_write_multi_4,
+	x86_bus_space_io_write_multi_8,
+	x86_bus_space_io_write_region_1,
+	x86_bus_space_io_write_region_2,
+	x86_bus_space_io_write_region_4,
+	x86_bus_space_io_write_region_8,
+	x86_bus_space_io_set_multi_1,
+	x86_bus_space_io_set_multi_2,
+	x86_bus_space_io_set_multi_4,
+	x86_bus_space_io_set_multi_8,
+	x86_bus_space_io_set_region_1,
+	x86_bus_space_io_set_region_2,
+	x86_bus_space_io_set_region_4,
+	x86_bus_space_io_set_region_8,
+	x86_bus_space_io_copy_1,
+	x86_bus_space_io_copy_2,
+	x86_bus_space_io_copy_4,
+	x86_bus_space_io_copy_8,
+	x86_bus_space_io_vaddr,
+	x86_bus_space_io_mmap
+};
+
+u_int8_t	x86_bus_space_mem_read_1(bus_space_handle_t, bus_size_t);
+u_int16_t	x86_bus_space_mem_read_2(bus_space_handle_t, bus_size_t);
+u_int32_t	x86_bus_space_mem_read_4(bus_space_handle_t, bus_size_t);
+u_int64_t	x86_bus_space_mem_read_8(bus_space_handle_t, bus_size_t);
+
+void		x86_bus_space_mem_read_multi_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t *, bus_size_t);
+void		x86_bus_space_mem_read_multi_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t *, bus_size_t);
+void		x86_bus_space_mem_read_multi_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t *, bus_size_t);
+void		x86_bus_space_mem_read_multi_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t *, bus_size_t);
+
+void		x86_bus_space_mem_read_region_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t *, bus_size_t);
+void		x86_bus_space_mem_read_region_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t *, bus_size_t);
+void		x86_bus_space_mem_read_region_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t *, bus_size_t);
+void		x86_bus_space_mem_read_region_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t *, bus_size_t);
+
+void		x86_bus_space_mem_write_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t);
+void		x86_bus_space_mem_write_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t);
+void		x86_bus_space_mem_write_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t);
+void		x86_bus_space_mem_write_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t);
+
+void		x86_bus_space_mem_write_multi_1(bus_space_handle_t,
+		    bus_size_t, const u_int8_t *, bus_size_t);
+void		x86_bus_space_mem_write_multi_2(bus_space_handle_t,
+		    bus_size_t, const u_int16_t *, bus_size_t);
+void		x86_bus_space_mem_write_multi_4(bus_space_handle_t,
+		    bus_size_t, const u_int32_t *, bus_size_t);
+void		x86_bus_space_mem_write_multi_8(bus_space_handle_t,
+		    bus_size_t, const u_int64_t *, bus_size_t);
+
+void		x86_bus_space_mem_write_region_1(bus_space_handle_t,
+		    bus_size_t, const u_int8_t *, bus_size_t);
+void		x86_bus_space_mem_write_region_2(bus_space_handle_t,
+		    bus_size_t, const u_int16_t *, bus_size_t);
+void		x86_bus_space_mem_write_region_4(bus_space_handle_t,
+		    bus_size_t, const u_int32_t *, bus_size_t);
+void		x86_bus_space_mem_write_region_8(bus_space_handle_t,
+		    bus_size_t, const u_int64_t *, bus_size_t);
+
+void		x86_bus_space_mem_set_multi_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t, size_t);
+void		x86_bus_space_mem_set_multi_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t, size_t);
+void		x86_bus_space_mem_set_multi_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t, size_t);
+void		x86_bus_space_mem_set_multi_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t, size_t);
+
+void		x86_bus_space_mem_set_region_1(bus_space_handle_t, bus_size_t,
+		    u_int8_t, size_t);
+void		x86_bus_space_mem_set_region_2(bus_space_handle_t, bus_size_t,
+		    u_int16_t, size_t);
+void		x86_bus_space_mem_set_region_4(bus_space_handle_t, bus_size_t,
+		    u_int32_t, size_t);
+void		x86_bus_space_mem_set_region_8(bus_space_handle_t, bus_size_t,
+		    u_int64_t, size_t);
+
+void		x86_bus_space_mem_copy_1(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+void		x86_bus_space_mem_copy_2(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+void		x86_bus_space_mem_copy_4(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+void		x86_bus_space_mem_copy_8(bus_space_handle_t, bus_size_t,
+		    bus_space_handle_t, bus_size_t, size_t);
+
+void *		x86_bus_space_mem_vaddr(bus_space_handle_t);
+
+paddr_t		x86_bus_space_mem_mmap(bus_addr_t, off_t, int, int);
+
+const struct x86_bus_space_ops x86_bus_space_mem_ops = {
+	x86_bus_space_mem_read_1,
+	x86_bus_space_mem_read_2,
+	x86_bus_space_mem_read_4,
+	x86_bus_space_mem_read_8,
+	x86_bus_space_mem_read_multi_1,
+	x86_bus_space_mem_read_multi_2,
+	x86_bus_space_mem_read_multi_4,
+	x86_bus_space_mem_read_multi_8,
+	x86_bus_space_mem_read_region_1,
+	x86_bus_space_mem_read_region_2,
+	x86_bus_space_mem_read_region_4,
+	x86_bus_space_mem_read_region_8,
+	x86_bus_space_mem_write_1,
+	x86_bus_space_mem_write_2,
+	x86_bus_space_mem_write_4,
+	x86_bus_space_mem_write_8,
+	x86_bus_space_mem_write_multi_1,
+	x86_bus_space_mem_write_multi_2,
+	x86_bus_space_mem_write_multi_4,
+	x86_bus_space_mem_write_multi_8,
+	x86_bus_space_mem_write_region_1,
+	x86_bus_space_mem_write_region_2,
+	x86_bus_space_mem_write_region_4,
+	x86_bus_space_mem_write_region_8,
+	x86_bus_space_mem_set_multi_1,
+	x86_bus_space_mem_set_multi_2,
+	x86_bus_space_mem_set_multi_4,
+	x86_bus_space_mem_set_multi_8,
+	x86_bus_space_mem_set_region_1,
+	x86_bus_space_mem_set_region_2,
+	x86_bus_space_mem_set_region_4,
+	x86_bus_space_mem_set_region_8,
+	x86_bus_space_mem_copy_1,
+	x86_bus_space_mem_copy_2,
+	x86_bus_space_mem_copy_4,
+	x86_bus_space_mem_copy_8,
+	x86_bus_space_mem_vaddr,
+	x86_bus_space_mem_mmap
+};
+
 void
 x86_bus_space_init(void)
 {
@@ -89,7 +332,6 @@ x86_bus_space_init(void)
 void
 x86_bus_space_mallocok(void)
 {
-
 	ioport_malloc_safe = 1;
 }
 
@@ -392,746 +634,874 @@ bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
 	return (0);
 }
 
-paddr_t
-bus_space_mmap(bus_space_tag_t t, bus_addr_t addr, off_t off, int prot, int flags)
-{
-
-	/* Can't mmap I/O space. */
-	if (t == X86_BUS_SPACE_IO)
-		return (-1);
-
-	/*
-	 * "addr" is the base address of the device we're mapping.
-	 * "off" is the offset into that device.
-	 *
-	 * Note we are called for each "page" in the device that
-	 * the upper layers want to map.
-	 */
-	return (addr + off);
-}
-
 u_int8_t
-bus_space_read_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+x86_bus_space_io_read_1(bus_space_handle_t h, bus_size_t o)
 {
-	return (t == X86_BUS_SPACE_IO ? (inb(h + o)) :
-	    (*(volatile u_int8_t *)(h + o)));
+	return (inb(h + o));
 }
 
 u_int16_t
-bus_space_read_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+x86_bus_space_io_read_2(bus_space_handle_t h, bus_size_t o)
 {
-	return (t == X86_BUS_SPACE_IO ? (inw(h + o)) :
-	    (*(volatile u_int16_t *)(h + o)));
+	return (inw(h + o));
 }
 
 u_int32_t
-bus_space_read_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+x86_bus_space_io_read_4(bus_space_handle_t h, bus_size_t o)
 {
-	return (t == X86_BUS_SPACE_IO ? (inl(h + o)) :
-	    (*(volatile u_int32_t *)(h + o)));
+	return (inl(h + o));
 }
 
 u_int64_t
-bus_space_read_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)
+x86_bus_space_io_read_8(bus_space_handle_t h, bus_size_t o)
 {
-	if (t == X86_BUS_SPACE_IO)
-		panic("bus_space_read_8: invalid I/O operation");
-
-	return (*(volatile u_int64_t *)(h + o));
+	panic("bus_space_read_8: invalid bus space tag");
 }
 
 void
-bus_space_read_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
-	    u_int8_t *ptr, bus_size_t cnt)
+x86_bus_space_io_read_multi_1(bus_space_handle_t h, bus_size_t o,
+    u_int8_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		insb(h + o, ptr, cnt);
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile(" cld					;"
-		"1:	movb (%2),%%al				;"
-		"	stosb					;"
-		"	loop 1b"				:
-		    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
-		    "memory");
-	}
+	insb(h + o, ptr, cnt);
 }
 
 void
-bus_space_read_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_io_read_multi_2(bus_space_handle_t h, bus_size_t o,
     u_int16_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		insw(h + o, ptr, cnt);
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile(" cld				;"
-		"1:	movw (%2),%%ax				;"
-		"	stosw					;"
-		"	loop 1b"				:
-		    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
-		    "memory");
-	}
+	insw(h + o, ptr, cnt);
 }
 
 void
-bus_space_read_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_io_read_multi_4(bus_space_handle_t h, bus_size_t o,
     u_int32_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		insl(h + o, ptr, cnt);
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile(" cld				;"
-		"1:	movl (%2),%%eax				;"
-		"	stosl					;"
-		"	loop 1b"				:
-		    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
-		    "memory");
-	}
+	insl(h + o, ptr, cnt);
 }
 
 void
-bus_space_read_multi_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_io_read_multi_8(bus_space_handle_t h, bus_size_t o,
     u_int64_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		panic("bus_space_multi_8: invalid I/O operation");
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile(" cld				;"
-		"1:	movq (%2),%%rax				;"
-		"	stosq					;"
-		"	loop 1b"				:
-		    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
-		    "memory");
-	}
+	panic("bus_space_multi_8: invalid bus space tag");
 }
 
 void
-bus_space_read_region_1(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_read_region_1(bus_space_handle_t h,
     bus_size_t o, u_int8_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		int __x;
-		u_int32_t port = h + o;
-		__asm __volatile(" cld					;"
-		"1:	inb %w1,%%al				;"
-		"	stosb					;"
-		"	incl %1					;"
-		"	loop 1b"				:
-		    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
-		    "=c" (dummy3)				:
-		    "1" (port), "2" (ptr), "3" (cnt)	:
-		    "memory");
-	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile(" cld					;"
-		"	repne					;"
-		"	movsb"					:
-		    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
-	}
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	int __x;
+	u_int32_t port = h + o;
+	__asm __volatile(" cld					;"
+	"1:	inb %w1,%%al				;"
+	"	stosb					;"
+	"	incl %1					;"
+	"	loop 1b"				:
+	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
+	    "=c" (dummy3)				:
+	    "1" (port), "2" (ptr), "3" (cnt)	:
+	    "memory");
 }
 
 void
-bus_space_read_region_2(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_read_region_2(bus_space_handle_t h,
     bus_size_t o, u_int16_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		int __x;
-		u_int32_t port = h + o;
-		__asm __volatile(" cld				;"
-		"1:	inw %w1,%%ax				;"
-		"	stosw					;"
-		"	addl $2,%1				;"
-		"	loop 1b"				:
-		    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
-		    "=c" (dummy3)				:
-		    "1" ((port)), "2" ((ptr)), "3" ((cnt))	:
-		    "memory");
-	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile(" cld				;"
-		"	repne					;"
-		"	movsw"					:
-		    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
-	}
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	int __x;
+	u_int32_t port = h + o;
+	__asm __volatile(" cld				;"
+	"1:	inw %w1,%%ax				;"
+	"	stosw					;"
+	"	addl $2,%1				;"
+	"	loop 1b"				:
+	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
+	    "=c" (dummy3)				:
+	    "1" ((port)), "2" ((ptr)), "3" ((cnt))	:
+	    "memory");
 }
 
 void
-bus_space_read_region_4(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_read_region_4(bus_space_handle_t h,
     bus_size_t o, u_int32_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		int __x;
-		u_int32_t port = h + o;
-		__asm __volatile("cld				;"
-		"1:	inl %w1,%%eax				;"
-		"	stosl					;"
-		"	addl $4,%1				;"
-		"	loop 1b"				:
-		    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
-		    "=c" (dummy3)				:
-		    "1" (port), "2" (ptr), "3" (cnt)	:
-		    "memory");
-	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile("cld				;"
-		"	repne					;"
-		"	movsl"					:
-		    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
-	}
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	int __x;
+	u_int32_t port = h + o;
+	__asm __volatile("cld				;"
+	"1:	inl %w1,%%eax				;"
+	"	stosl					;"
+	"	addl $4,%1				;"
+	"	loop 1b"				:
+	    "=&a" (__x), "=d" (dummy1), "=D" (dummy2),
+	    "=c" (dummy3)				:
+	    "1" (port), "2" (ptr), "3" (cnt)	:
+	    "memory");
 }
 
 void
-bus_space_read_region_8(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_read_region_8(bus_space_handle_t h,
     bus_size_t o, u_int64_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		panic("bus_space_read_region_8: invalid I/O operation");
-	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile("cld				;"
-		"	repne					;"
-		"	movsq"					:
-		    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
-	}
+	panic("bus_space_read_region_8: invalid bus space tag");
 }
 
-/*
- *	void bus_space_write_N(bus_space_tag_t tag,
- *	    bus_space_handle_t bsh, bus_size_t offset,
- *	    u_intN_t value);
- *
- * Write the 1, 2, 4, or 8 byte value `value' to bus space
- * described by tag/handle/offset.
- */
 void
-bus_space_write_1(bus_space_tag_t t, bus_space_handle_t h,
-    bus_size_t o, u_int8_t v)
+x86_bus_space_io_write_1(bus_space_handle_t h, bus_size_t o, u_int8_t v)
 {
-	if (t == X86_BUS_SPACE_IO)
-		outb(h + o, v);
-	else
-		((void)(*(volatile u_int8_t *)(h + o) = v));
+	outb(h + o, v);
 }
 
 void
-bus_space_write_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
-    u_int16_t v)
+x86_bus_space_io_write_2(bus_space_handle_t h, bus_size_t o, u_int16_t v)
 {
-	if ((t) == X86_BUS_SPACE_IO)
-		outw(h + o, v);
-	else
-		((void)(*(volatile u_int16_t *)(h + o) = v));
+	outw(h + o, v);
 }
 
 void
-bus_space_write_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
-    u_int32_t v)
+x86_bus_space_io_write_4(bus_space_handle_t h, bus_size_t o, u_int32_t v)
 {
-	if ((t) == X86_BUS_SPACE_IO)
-		outl(h + o, v);
-	else
-		((void)(*(volatile u_int32_t *)(h + o) = v));
+	outl(h + o, v);
 }
 
 void
-bus_space_write_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
-    u_int64_t v)
+x86_bus_space_io_write_8(bus_space_handle_t h, bus_size_t o, u_int64_t v)
 {
-	if ((t) == X86_BUS_SPACE_IO)
-		panic("bus_space_write_8: invalid I/O operation");
-	else
-		((void)(*(volatile u_int64_t *)(h + o) = v));
+	panic("bus_space_write_8: invalid bus space tag");
 }
 
 void
-bus_space_write_multi_1(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_write_multi_1(bus_space_handle_t h,
     bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		outsb(h + o, ptr, cnt);
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile("cld				;"
-		"1:	lodsb					;"
-		"	movb %%al,(%2)				;"
-		"	loop 1b"				:
-		    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" (ptr), "1" (cnt), "2" (h + o));
-	}
+	outsb(h + o, ptr, cnt);
 }
 
 void
-bus_space_write_multi_2(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_write_multi_2(bus_space_handle_t h,
     bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		outsw(h + o, ptr, cnt);
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile("cld					;"
-		"1:	lodsw					;"
-		"	movw %%ax,(%2)				;"
-		"	loop 1b"				:
-		    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" (ptr), "1" (cnt), "2" (h + o));
-	}
+	outsw(h + o, ptr, cnt);
 }
 
 void
-bus_space_write_multi_4(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_write_multi_4(bus_space_handle_t h,
     bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		outsl(h + o, ptr, cnt);
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile("cld				;"
-		"1:	lodsl					;"
-		"	movl %%eax,(%2)				;"
-		"	loop 1b"				:
-		    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" (ptr), "1" (cnt), "2" (h + o));
-	}
+	outsl(h + o, ptr, cnt);
 }
 
 void
-bus_space_write_multi_8(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_write_multi_8(bus_space_handle_t h,
     bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		panic("bus_space_write_multi_8: invalid I/O operation");
-	} else {
-		void *dummy1;
-		int dummy2;
-		void *dummy3;
-		int __x;
-		__asm __volatile("cld				;"
-		"1:	lodsq					;"
-		"	movq %%rax,(%2)				;"
-		"	loop 1b"				:
-		    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
-		    "0" (ptr), "1" (cnt), "2" (h + o));
-	}
+	panic("bus_space_write_multi_8: invalid bus space tag");
 }
 
 void
-bus_space_write_region_1(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_write_region_1(bus_space_handle_t h,
     bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		int __x;
-		u_int32_t port = h + o;
-		__asm __volatile("cld				;"
-		"1:	lodsb					;"
-		"	outb %%al,%w1				;"
-		"	incl %1					;"
-		"	loop 1b"				:
-		    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
-		    "=c" (dummy3)				:
-		    "1" (port), "2" (ptr), "3" (cnt)	:
-		    "memory");
-	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile("cld				;"
-		"	repne					;"
-		"	movsb"					:
-		    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
-	}
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	int __x;
+	u_int32_t port = h + o;
+	__asm __volatile("cld				;"
+	"1:	lodsb					;"
+	"	outb %%al,%w1				;"
+	"	incl %1					;"
+	"	loop 1b"				:
+	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
+	    "=c" (dummy3)				:
+	    "1" (port), "2" (ptr), "3" (cnt)	:
+	    "memory");
 }
 
 void
-bus_space_write_region_2(bus_space_tag_t t, bus_space_handle_t h,
+x86_bus_space_io_write_region_2(bus_space_handle_t h,
     bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		int __x;
-		u_int32_t port = h + o;
-		__asm __volatile("cld				;"
-		"1:	lodsw					;"
-		"	outw %%ax,%w1				;"
-		"	addl $2,%1				;"
-		"	loop 1b"				: 
-		    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
-		    "=c" (dummy3)				:
-		    "1" (port), "2" (ptr), "3" (cnt)	:
-		    "memory");
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	int __x;
+	u_int32_t port = h + o;
+	__asm __volatile("cld				;"
+	"1:	lodsw					;"
+	"	outw %%ax,%w1				;"
+	"	addl $2,%1				;"
+	"	loop 1b"				: 
+	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
+	    "=c" (dummy3)				:
+	    "1" (port), "2" (ptr), "3" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_io_write_region_4(bus_space_handle_t h,
+    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	int __x;
+	u_int32_t port = h + o;
+	__asm __volatile(" cld				;"
+	"1:	lodsl					;"
+	"	outl %%eax,%w1				;"
+	"	addl $4,%1				;"
+	"	loop 1b"				:
+	    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
+	    "=c" (dummy3)				:
+	    "1" (port), "2" (ptr), "3" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_io_write_region_8(bus_space_handle_t h,
+    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
+{
+	panic("bus_space_write_region_8: invalid bus space tag");
+}
+
+void
+x86_bus_space_io_set_multi_1(bus_space_handle_t h, bus_size_t o,
+    u_int8_t v, size_t c)
+{
+	bus_addr_t addr = h + o;
+
+	while (c--)
+		outb(addr, v);
+}
+
+void
+x86_bus_space_io_set_multi_2(bus_space_handle_t h, bus_size_t o,
+    u_int16_t v, size_t c)
+{
+	bus_addr_t addr = h + o;
+
+	while (c--)
+		outw(addr, v);
+}
+
+void
+x86_bus_space_io_set_multi_4(bus_space_handle_t h, bus_size_t o,
+    u_int32_t v, size_t c)
+{
+	bus_addr_t addr = h + o;
+
+	while (c--)
+		outl(addr, v);
+}
+
+void
+x86_bus_space_io_set_multi_8(bus_space_handle_t h, bus_size_t o,
+    u_int64_t v, size_t c)
+{
+	panic("bus_space_set_multi_8: invalid bus space tag");
+}
+
+void
+x86_bus_space_io_set_region_1(bus_space_handle_t h, bus_size_t o,
+    u_int8_t v, size_t c)
+{
+	bus_addr_t addr = h + o;
+
+	for (; c != 0; c--, addr++)
+		outb(addr, v);
+}
+
+void
+x86_bus_space_io_set_region_2(bus_space_handle_t h, bus_size_t o,
+    u_int16_t v, size_t c)
+{
+	bus_addr_t addr = h + o;
+
+	for (; c != 0; c--, addr += sizeof(v))
+		outw(addr, v);
+}
+
+void
+x86_bus_space_io_set_region_4(bus_space_handle_t h, bus_size_t o,
+    u_int32_t v, size_t c)
+{
+	bus_addr_t addr = h + o;
+
+	for (; c != 0; c--, addr += sizeof(v))
+		outl(addr, v);
+}
+
+void
+x86_bus_space_io_set_region_8(bus_space_handle_t h, bus_size_t o,
+    u_int64_t v, size_t c)
+{
+	panic("bus_space_set_region_8: invalid bus space tag");
+}
+
+void
+x86_bus_space_io_copy_1(bus_space_handle_t h1, bus_size_t o1,
+    bus_space_handle_t h2, bus_size_t o2, size_t c)
+{
+	bus_addr_t addr1 = h1 + o1;
+	bus_addr_t addr2 = h2 + o2;
+
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1++, addr2++)
+			outb(addr2, inb(addr1));
 	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile("cld				;"
-		"	repne					;"
-		"	movsw"					:
-		    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
+		/* dest after src: copy backwards */
+		for (addr1 += (c - 1), addr2 += (c - 1);
+		    c != 0; c--, addr1--, addr2--)
+			outb(addr2, inb(addr1));
 	}
 }
 
 void
-bus_space_write_region_4(bus_space_tag_t t, bus_space_handle_t h,
-    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
+x86_bus_space_io_copy_2(bus_space_handle_t h1, bus_size_t o1,
+    bus_space_handle_t h2, bus_size_t o2, size_t c)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		int __x;
-		u_int32_t port = h + o;
-		__asm __volatile(" cld				;"
-		"1:	lodsl					;"
-		"	outl %%eax,%w1				;"
-		"	addl $4,%1				;"
-		"	loop 1b"				:
-		    "=&a" (__x), "=d" (dummy1), "=S" (dummy2),
-		    "=c" (dummy3)				:
-		    "1" (port), "2" (ptr), "3" (cnt)	:
-		    "memory");
+	bus_addr_t addr1 = h1 + o1;
+	bus_addr_t addr2 = h2 + o2;
+
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1 += 2, addr2 += 2)
+			outw(addr2, inw(addr1));
 	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile("cld				;"
-		"	repne					;"
-		"	movsl"					:
-		    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
+		/* dest after src: copy backwards */
+		for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
+		    c != 0; c--, addr1 -= 2, addr2 -= 2)
+			outw(addr2, inw(addr1));
 	}
 }
 
 void
-bus_space_write_region_8(bus_space_tag_t t, bus_space_handle_t h,
-    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
+x86_bus_space_io_copy_4(bus_space_handle_t h1, bus_size_t o1,
+    bus_space_handle_t h2, bus_size_t o2, size_t c)
 {
-	if ((t) == X86_BUS_SPACE_IO) {
-		panic("bus_space_write_region_8: invalid I/O operation");
+	bus_addr_t addr1 = h1 + o1;
+	bus_addr_t addr2 = h2 + o2;
+
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1 += 4, addr2 += 4)
+			outl(addr2, inl(addr1));
 	} else {
-		int dummy1;
-		void *dummy2;
-		int dummy3;
-		__asm __volatile("cld				;"
-		"	repne					;"
-		"	movsq"					:
-		    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
-		    "0" (h + o), "1" (ptr), "2" (cnt)	:
-		    "memory");
+		/* dest after src: copy backwards */
+		for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
+		    c != 0; c--, addr1 -= 4, addr2 -= 4)
+			outl(addr2, inl(addr1));
 	}
 }
 
 void
-bus_space_set_multi_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_io_copy_8(bus_space_handle_t h1, bus_size_t o1,
+    bus_space_handle_t h2, bus_size_t o2, size_t c)
+{
+	panic("bus_space_set_region_8: invalid bus space tag");
+}
+
+void *
+x86_bus_space_io_vaddr(bus_space_handle_t h)
+{
+	return (NULL);
+}
+
+paddr_t
+x86_bus_space_io_mmap(bus_addr_t addr, off_t off, int prot, int flags)
+{
+	/* Can't mmap I/O space. */
+	return (-1);
+}
+
+void
+x86_bus_space_mem_write_1(bus_space_handle_t h, bus_size_t o, u_int8_t v)
+{
+	*(volatile u_int8_t *)(h + o) = v;
+}
+
+void
+x86_bus_space_mem_write_2(bus_space_handle_t h, bus_size_t o, u_int16_t v)
+{
+	*(volatile u_int16_t *)(h + o) = v;
+}
+
+u_int8_t
+x86_bus_space_mem_read_1(bus_space_handle_t h, bus_size_t o)
+{
+	return (*(volatile u_int8_t *)(h + o));
+}
+
+u_int16_t
+x86_bus_space_mem_read_2(bus_space_handle_t h, bus_size_t o)
+{
+	return (*(volatile u_int16_t *)(h + o));
+}
+
+u_int32_t
+x86_bus_space_mem_read_4(bus_space_handle_t h, bus_size_t o)
+{
+	return (*(volatile u_int32_t *)(h + o));
+}
+
+u_int64_t
+x86_bus_space_mem_read_8(bus_space_handle_t h, bus_size_t o)
+{
+	return (*(volatile u_int64_t *)(h + o));
+}
+
+void
+x86_bus_space_mem_read_multi_1(bus_space_handle_t h, bus_size_t o,
+    u_int8_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile(" cld				;"
+	"1:	movb (%2),%%al				;"
+	"	stosb					;"
+	"	loop 1b"				:
+	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_multi_2(bus_space_handle_t h, bus_size_t o,
+    u_int16_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile(" cld				;"
+	"1:	movw (%2),%%ax				;"
+	"	stosw					;"
+	"	loop 1b"				:
+	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_multi_4(bus_space_handle_t h, bus_size_t o,
+    u_int32_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile(" cld				;"
+	"1:	movl (%2),%%eax				;"
+	"	stosl					;"
+	"	loop 1b"				:
+	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_multi_8(bus_space_handle_t h, bus_size_t o,
+    u_int64_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile(" cld				;"
+	"1:	movq (%2),%%rax				;"
+	"	stosq					;"
+	"	loop 1b"				:
+	    "=D" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" ((ptr)), "1" ((cnt)), "2" (h + o)       :
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_region_1(bus_space_handle_t h,
+    bus_size_t o, u_int8_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile(" cld				;"
+	"	repne					;"
+	"	movsb"					:
+	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_region_2(bus_space_handle_t h,
+    bus_size_t o, u_int16_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile(" cld				;"
+	"	repne					;"
+	"	movsw"					:
+	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_region_4(bus_space_handle_t h,
+    bus_size_t o, u_int32_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile("cld				;"
+	"	repne					;"
+	"	movsl"					:
+	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_read_region_8(bus_space_handle_t h,
+    bus_size_t o, u_int64_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile("cld				;"
+	"	repne					;"
+	"	movsq"					:
+	    "=S" (dummy1), "=D" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_write_4(bus_space_handle_t h, bus_size_t o, u_int32_t v)
+{
+	*(volatile u_int32_t *)(h + o) = v;
+}
+
+void
+x86_bus_space_mem_write_8(bus_space_handle_t h, bus_size_t o, u_int64_t v)
+{
+	*(volatile u_int64_t *)(h + o) = v;
+}
+
+void
+x86_bus_space_mem_write_multi_1(bus_space_handle_t h,
+    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile("cld				;"
+	"1:	lodsb					;"
+	"	movb %%al,(%2)				;"
+	"	loop 1b"				:
+	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" (ptr), "1" (cnt), "2" (h + o));
+}
+
+void
+x86_bus_space_mem_write_multi_2(bus_space_handle_t h,
+    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile("cld					;"
+	"1:	lodsw					;"
+	"	movw %%ax,(%2)				;"
+	"	loop 1b"				:
+	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" (ptr), "1" (cnt), "2" (h + o));
+}
+
+void
+x86_bus_space_mem_write_multi_4(bus_space_handle_t h,
+    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile("cld				;"
+	"1:	lodsl					;"
+	"	movl %%eax,(%2)				;"
+	"	loop 1b"				:
+	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" (ptr), "1" (cnt), "2" (h + o));
+}
+
+void
+x86_bus_space_mem_write_multi_8(bus_space_handle_t h,
+    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
+{
+	void *dummy1;
+	int dummy2;
+	void *dummy3;
+	int __x;
+	__asm __volatile("cld				;"
+	"1:	lodsq					;"
+	"	movq %%rax,(%2)				;"
+	"	loop 1b"				:
+	    "=S" (dummy1), "=c" (dummy2), "=r" (dummy3), "=&a" (__x) :
+	    "0" (ptr), "1" (cnt), "2" (h + o));
+}
+
+void
+x86_bus_space_mem_write_region_1(bus_space_handle_t h,
+    bus_size_t o, const u_int8_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile("cld				;"
+	"	repne					;"
+	"	movsb"					:
+	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_write_region_2(bus_space_handle_t h,
+    bus_size_t o, const u_int16_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile("cld				;"
+	"	repne					;"
+	"	movsw"					:
+	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_write_region_4(bus_space_handle_t h,
+    bus_size_t o, const u_int32_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile("cld				;"
+	"	repne					;"
+	"	movsl"					:
+	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_write_region_8(bus_space_handle_t h,
+    bus_size_t o, const u_int64_t *ptr, bus_size_t cnt)
+{
+	int dummy1;
+	void *dummy2;
+	int dummy3;
+	__asm __volatile("cld				;"
+	"	repne					;"
+	"	movsq"					:
+	    "=D" (dummy1), "=S" (dummy2), "=c" (dummy3)	:
+	    "0" (h + o), "1" (ptr), "2" (cnt)	:
+	    "memory");
+}
+
+void
+x86_bus_space_mem_set_multi_1(bus_space_handle_t h, bus_size_t o,
     u_int8_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		while (c--)
-			outb(addr, v);
-	else
-		while (c--)
-			*(volatile u_int8_t *)(addr) = v;
+	while (c--)
+		*(volatile u_int8_t *)(addr) = v;
 }
 
 void
-bus_space_set_multi_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_multi_2(bus_space_handle_t h, bus_size_t o,
     u_int16_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		while (c--)
-			outw(addr, v);
-	else
-		while (c--)
-			*(volatile u_int16_t *)(addr) = v;
+	while (c--)
+		*(volatile u_int16_t *)(addr) = v;
 }
 
 void
-bus_space_set_multi_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_multi_4(bus_space_handle_t h, bus_size_t o,
     u_int32_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		while (c--)
-			outl(addr, v);
-	else
-		while (c--)
-			*(volatile u_int32_t *)(addr) = v;
+	while (c--)
+		*(volatile u_int32_t *)(addr) = v;
 }
 
 void
-bus_space_set_multi_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_multi_8(bus_space_handle_t h, bus_size_t o,
     u_int64_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		panic("bus_space_set_multi_8: invalid I/O operation");
-	else
-		while (c--)
-			*(volatile u_int64_t *)(addr) = v;
+	while (c--)
+		*(volatile u_int64_t *)(addr) = v;
 }
 
 void
-bus_space_set_region_1(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_region_1(bus_space_handle_t h, bus_size_t o,
     u_int8_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		for (; c != 0; c--, addr++)
-			outb(addr, v);
-	else
-		for (; c != 0; c--, addr++)
-			*(volatile u_int8_t *)(addr) = v;
+	for (; c != 0; c--, addr++)
+		*(volatile u_int8_t *)(addr) = v;
 }
 
 void
-bus_space_set_region_2(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_region_2(bus_space_handle_t h, bus_size_t o,
     u_int16_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		for (; c != 0; c--, addr += 2)
-			outw(addr, v);
-	else
-		for (; c != 0; c--, addr += 2)
-			*(volatile u_int16_t *)(addr) = v;
+	for (; c != 0; c--, addr += sizeof(v))
+		*(volatile u_int16_t *)(addr) = v;
 }
 
 void
-bus_space_set_region_4(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_region_4(bus_space_handle_t h, bus_size_t o,
     u_int32_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		for (; c != 0; c--, addr += 4)
-			outl(addr, v);
-	else
-		for (; c != 0; c--, addr += 4)
-			*(volatile u_int32_t *)(addr) = v;
+	for (; c != 0; c--, addr += sizeof(v))
+		*(volatile u_int32_t *)(addr) = v;
 }
 
 void
-bus_space_set_region_8(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o,
+x86_bus_space_mem_set_region_8(bus_space_handle_t h, bus_size_t o,
     u_int64_t v, size_t c)
 {
 	bus_addr_t addr = h + o;
 
-	if (t == X86_BUS_SPACE_IO)
-		panic("bus_space_set_region_8: invalid I/O operation");
-	else
-		for (; c != 0; c--, addr += sizeof(v))
-			*(volatile u_int64_t *)(addr) = v;
+	for (; c != 0; c--, addr += sizeof(v))
+		*(volatile u_int64_t *)(addr) = v;
 }
 
 void
-bus_space_copy_1(bus_space_tag_t t,
-    bus_space_handle_t h1, bus_size_t o1,
+x86_bus_space_mem_copy_1( bus_space_handle_t h1, bus_size_t o1,
     bus_space_handle_t h2, bus_size_t o2, size_t c)
 {
 	bus_addr_t addr1 = h1 + o1;
 	bus_addr_t addr2 = h2 + o2;
 
-	if (t == X86_BUS_SPACE_IO) {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1++, addr2++)
-				outb(addr2, inb(addr1));
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += (c - 1), addr2 += (c - 1);
-			    c != 0; c--, addr1--, addr2--)
-				outb(addr2, inb(addr1));
-		}
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1++, addr2++)
+			*(volatile u_int8_t *)(addr2) =
+			    *(volatile u_int8_t *)(addr1);
 	} else {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1++, addr2++)
-				*(volatile u_int8_t *)(addr2) =
-				    *(volatile u_int8_t *)(addr1);
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += (c - 1), addr2 += (c - 1);
-			    c != 0; c--, addr1--, addr2--)
-				*(volatile u_int8_t *)(addr2) =
-				    *(volatile u_int8_t *)(addr1);
-		}
+		/* dest after src: copy backwards */
+		for (addr1 += (c - 1), addr2 += (c - 1);
+		    c != 0; c--, addr1--, addr2--)
+			*(volatile u_int8_t *)(addr2) =
+			    *(volatile u_int8_t *)(addr1);
 	}
 }
 
 void
-bus_space_copy_2(bus_space_tag_t t,
-    bus_space_handle_t h1, bus_size_t o1,
+x86_bus_space_mem_copy_2(bus_space_handle_t h1, bus_size_t o1,
     bus_space_handle_t h2, bus_size_t o2, size_t c)
 {
 	bus_addr_t addr1 = h1 + o1;
 	bus_addr_t addr2 = h2 + o2;
 
-	if (t == X86_BUS_SPACE_IO) {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1 += 2, addr2 += 2)
-				outw(addr2, inw(addr1));
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
-			    c != 0; c--, addr1 -= 2, addr2 -= 2)
-				outw(addr2, inw(addr1));
-		}
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1 += 2, addr2 += 2)
+			*(volatile u_int16_t *)(addr2) =
+			    *(volatile u_int16_t *)(addr1);
 	} else {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1 += 2, addr2 += 2)
-				*(volatile u_int16_t *)(addr2) =
-				    *(volatile u_int16_t *)(addr1);
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
-			    c != 0; c--, addr1 -= 2, addr2 -= 2)
-				*(volatile u_int16_t *)(addr2) =
-				    *(volatile u_int16_t *)(addr1);
-		}
+		/* dest after src: copy backwards */
+		for (addr1 += 2 * (c - 1), addr2 += 2 * (c - 1);
+		    c != 0; c--, addr1 -= 2, addr2 -= 2)
+			*(volatile u_int16_t *)(addr2) =
+			    *(volatile u_int16_t *)(addr1);
 	}
 }
 
 void
-bus_space_copy_4(bus_space_tag_t t,
-    bus_space_handle_t h1, bus_size_t o1,
+x86_bus_space_mem_copy_4(bus_space_handle_t h1, bus_size_t o1,
     bus_space_handle_t h2, bus_size_t o2, size_t c)
 {
 	bus_addr_t addr1 = h1 + o1;
 	bus_addr_t addr2 = h2 + o2;
 
-	if (t == X86_BUS_SPACE_IO) {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1 += 4, addr2 += 4)
-				outl(addr2, inl(addr1));
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
-			    c != 0; c--, addr1 -= 4, addr2 -= 4)
-				outl(addr2, inl(addr1));
-		}
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1 += 4, addr2 += 4)
+			*(volatile u_int32_t *)(addr2) =
+			    *(volatile u_int32_t *)(addr1);
 	} else {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1 += 4, addr2 += 4)
-				*(volatile u_int32_t *)(addr2) =
-				    *(volatile u_int32_t *)(addr1);
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
-			    c != 0; c--, addr1 -= 4, addr2 -= 4)
-				*(volatile u_int32_t *)(addr2) =
-				    *(volatile u_int32_t *)(addr1);
-		}
+		/* dest after src: copy backwards */
+		for (addr1 += 4 * (c - 1), addr2 += 4 * (c - 1);
+		    c != 0; c--, addr1 -= 4, addr2 -= 4)
+			*(volatile u_int32_t *)(addr2) =
+			    *(volatile u_int32_t *)(addr1);
 	}
 }
 
 void
-bus_space_copy_8(bus_space_tag_t t,
-    bus_space_handle_t h1, bus_size_t o1,
+x86_bus_space_mem_copy_8(bus_space_handle_t h1, bus_size_t o1,
     bus_space_handle_t h2, bus_size_t o2, size_t c)
 {
 	bus_addr_t addr1 = h1 + o1;
 	bus_addr_t addr2 = h2 + o2;
 
-	if (t == X86_BUS_SPACE_IO) {
-		panic("bus_space_set_region_8: invalid I/O operation");
+	if (addr1 >= addr2) {
+		/* src after dest: copy forward */
+		for (; c != 0; c--, addr1 += 8, addr2 += 8)
+			*(volatile u_int64_t *)(addr2) =
+			    *(volatile u_int64_t *)(addr1);
 	} else {
-		if (addr1 >= addr2) {
-			/* src after dest: copy forward */
-			for (; c != 0; c--, addr1 += 8, addr2 += 8)
-				*(volatile u_int64_t *)(addr2) =
-				    *(volatile u_int64_t *)(addr1);
-		} else {
-			/* dest after src: copy backwards */
-			for (addr1 += 8 * (c - 1), addr2 += 8 * (c - 1);
-			    c != 0; c--, addr1 -= 8, addr2 -= 8)
-				*(volatile u_int64_t *)(addr2) =
-				    *(volatile u_int64_t *)(addr1);
-		}
+		/* dest after src: copy backwards */
+		for (addr1 += 8 * (c - 1), addr2 += 8 * (c - 1);
+		    c != 0; c--, addr1 -= 8, addr2 -= 8)
+			*(volatile u_int64_t *)(addr2) =
+			    *(volatile u_int64_t *)(addr1);
 	}
 }
 
-void
-bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t bsh,
-    bus_size_t offset, bus_size_t len, int flags)
+void *
+x86_bus_space_mem_vaddr(bus_space_handle_t h)
 {
-	if (flags == (BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE))
-		__asm __volatile("mfence");
-	else if (flags == BUS_SPACE_BARRIER_WRITE)
-		__asm __volatile("sfence");
-	else
-		__asm __volatile("lfence");
+	return ((void *)h);
 }
 
+paddr_t
+x86_bus_space_mem_mmap(bus_addr_t addr, off_t off, int prot, int flags)
+{
+	/*
+	 * "addr" is the base address of the device we're mapping.
+	 * "off" is the offset into that device.
+	 *
+	 * Note we are called for each "page" in the device that
+	 * the upper layers want to map.
+	 */
+	return (addr + off);
+}
diff --git a/sys/arch/amd64/include/bus.h b/sys/arch/amd64/include/bus.h
index 34849c9b81f..0ee92b63f07 100644
--- a/sys/arch/amd64/include/bus.h
+++ b/sys/arch/amd64/include/bus.h
@@ -1,4 +1,4 @@
-/*	$OpenBSD: bus.h,v 1.26 2013/01/15 09:33:30 dlg Exp $	*/
+/*	$OpenBSD: bus.h,v 1.27 2013/01/18 01:54:40 dlg Exp $	*/
 /*	$NetBSD: bus.h,v 1.6 1996/11/10 03:19:25 thorpej Exp $	*/
 
 /*-
@@ -72,12 +72,6 @@
 #include <machine/pio.h>
 
 /*
- * Values for the x86 bus space tag, not to be used directly by MI code.
- */
-#define	X86_BUS_SPACE_IO	0	/* space is i/o space */
-#define X86_BUS_SPACE_MEM	1	/* space is mem space */
-
-/*
  * Bus address and size types
  */
 typedef u_long bus_addr_t;
@@ -86,7 +80,8 @@ typedef u_long bus_size_t;
 /*
  * Access methods for bus resources and address space.
  */
-typedef	int bus_space_tag_t;
+struct x86_bus_space_ops;
+typedef	const struct x86_bus_space_ops *bus_space_tag_t;
 typedef	u_long bus_space_handle_t;
 
 int	bus_space_map(bus_space_tag_t t, bus_addr_t addr,
@@ -129,6 +124,8 @@ int	_bus_space_map(bus_space_tag_t t, bus_addr_t addr,
 int	bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
 	    bus_size_t offset, bus_size_t size, bus_space_handle_t *nbshp);
 
+struct x86_bus_space_ops {
+
 /*
  *	u_intN_t bus_space_read_N(bus_space_tag_t tag,
  *	    bus_space_handle_t bsh, bus_size_t offset);
@@ -136,17 +133,15 @@ int	bus_space_subregion(bus_space_tag_t t, bus_space_handle_t bsh,
  * Read a 1, 2, 4, or 8 byte quantity from bus space
  * described by tag/handle/offset.
  */
-u_int8_t	bus_space_read_1(bus_space_tag_t, bus_space_handle_t,
-		    bus_size_t);
-
-u_int16_t	bus_space_read_2(bus_space_tag_t, bus_space_handle_t,
-		    bus_size_t);
+	u_int8_t	(*read_1)(bus_space_handle_t, bus_size_t);
+	u_int16_t	(*read_2)(bus_space_handle_t, bus_size_t);
+	u_int32_t	(*read_4)(bus_space_handle_t, bus_size_t);
+	u_int64_t	(*read_8)(bus_space_handle_t, bus_size_t);
 
-u_int32_t	bus_space_read_4(bus_space_tag_t, bus_space_handle_t,
-		    bus_size_t);
-
-u_int64_t	bus_space_read_8(bus_space_tag_t, bus_space_handle_t,
-		    bus_size_t);
+#define bus_space_read_1(_t, _h, _o) ((_t)->read_1((_h), (_o)))
+#define bus_space_read_2(_t, _h, _o) ((_t)->read_2((_h), (_o)))
+#define bus_space_read_4(_t, _h, _o) ((_t)->read_4((_h), (_o)))
+#define bus_space_read_8(_t, _h, _o) ((_t)->read_8((_h), (_o)))
 
 /*
  *	void bus_space_read_multi_N(bus_space_tag_t tag,
@@ -157,17 +152,23 @@ u_int64_t	bus_space_read_8(bus_space_tag_t, bus_space_handle_t,
  * described by tag/handle/offset and copy into buffer provided.
  */
 
-void	bus_space_read_multi_1(bus_space_tag_t, bus_space_handle_t, bus_size_t,
-	    u_int8_t *, bus_size_t);
-
-void	bus_space_read_multi_2(bus_space_tag_t, bus_space_handle_t, bus_size_t,
-	    u_int16_t *, bus_size_t);
-
-void	bus_space_read_multi_4(bus_space_tag_t, bus_space_handle_t, bus_size_t,
-	    u_int32_t *, bus_size_t);
-
-void	bus_space_read_multi_8(bus_space_tag_t, bus_space_handle_t, bus_size_t,
-	    u_int64_t *, bus_size_t);
+	void		(*read_multi_1)(bus_space_handle_t, bus_size_t,
+			    u_int8_t *, bus_size_t);
+	void		(*read_multi_2)(bus_space_handle_t, bus_size_t,
+			    u_int16_t *, bus_size_t);
+	void		(*read_multi_4)(bus_space_handle_t, bus_size_t,
+			    u_int32_t *, bus_size_t);
+	void		(*read_multi_8)(bus_space_handle_t, bus_size_t,
+			    u_int64_t *, bus_size_t);
+
+#define bus_space_read_multi_1(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_1((_h), (_o), (_a), (_c)))
+#define bus_space_read_multi_2(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_2((_h), (_o), (_a), (_c)))
+#define bus_space_read_multi_4(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_4((_h), (_o), (_a), (_c)))
+#define bus_space_read_multi_8(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_8((_h), (_o), (_a), (_c)))
 
 /*
  *	void bus_space_read_raw_multi_N(bus_space_tag_t tag,
@@ -180,12 +181,12 @@ void	bus_space_read_multi_8(bus_space_tag_t, bus_space_handle_t, bus_size_t,
  * possible byte-swapping should be done by these functions.
  */
 
-#define	bus_space_read_raw_multi_2(t, h, o, a, c) \
-    bus_space_read_multi_2((t), (h), (o), (u_int16_t *)(a), (c) >> 1)
-#define	bus_space_read_raw_multi_4(t, h, o, a, c) \
-    bus_space_read_multi_4((t), (h), (o), (u_int32_t *)(a), (c) >> 2)
-#define	bus_space_read_raw_multi_8(t, h, o, a, c) \
-    bus_space_read_multi_8((t), (h), (o), (u_int32_t *)(a), (c) >> 3)
+#define bus_space_read_raw_multi_2(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_2((_h), (_o), (u_int16_t *)(_a), (_c) >> 1))
+#define bus_space_read_raw_multi_4(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_4((_h), (_o), (u_int32_t *)(_a), (_c) >> 2))
+#define bus_space_read_raw_multi_8(_t, _h, _o, _a, _c) \
+	((_t)->read_multi_8((_h), (_o), (u_int64_t *)(_a), (_c) >> 3))
 
 /*
  *	void bus_space_read_region_N(bus_space_tag_t tag,
@@ -197,14 +198,23 @@ void	bus_space_read_multi_8(bus_space_tag_t, bus_space_handle_t, bus_size_t,
  * buffer provided.
  */
 
-void	bus_space_read_region_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int8_t *, bus_size_t);
-void	bus_space_read_region_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int16_t *, bus_size_t);
-void	bus_space_read_region_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int32_t *, bus_size_t);
-void	bus_space_read_region_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*read_region_1)(bus_space_handle_t,
+			    bus_size_t, u_int8_t *, bus_size_t);
+	void		(*read_region_2)(bus_space_handle_t,
+			    bus_size_t, u_int16_t *, bus_size_t);
+	void		(*read_region_4)(bus_space_handle_t,
+			    bus_size_t, u_int32_t *, bus_size_t);
+	void		(*read_region_8)(bus_space_handle_t,
+			    bus_size_t, u_int64_t *, bus_size_t);
+
+#define bus_space_read_region_1(_t, _h, _o, _a, _c) \
+	((_t)->read_region_1((_h), (_o), (_a), (_c)))
+#define bus_space_read_region_2(_t, _h, _o, _a, _c) \
+	((_t)->read_region_2((_h), (_o), (_a), (_c)))
+#define bus_space_read_region_4(_t, _h, _o, _a, _c) \
+	((_t)->read_region_4((_h), (_o), (_a), (_c)))
+#define bus_space_read_region_8(_t, _h, _o, _a, _c) \
+	((_t)->read_region_8((_h), (_o), (_a), (_c)))
 
 /*
  *	void bus_space_read_raw_region_N(bus_space_tag_t tag,
@@ -218,12 +228,12 @@ void	bus_space_read_region_8(bus_space_tag_t, bus_space_handle_t,
  * these functions.
  */
 
-#define	bus_space_read_raw_region_2(t, h, o, a, c) \
-    bus_space_read_region_2((t), (h), (o), (u_int16_t *)(a), (c) >> 1)
-#define	bus_space_read_raw_region_4(t, h, o, a, c) \
-    bus_space_read_region_4((t), (h), (o), (u_int32_t *)(a), (c) >> 2)
-#define	bus_space_read_raw_region_8(t, h, o, a, c) \
-    bus_space_read_region_4((t), (h), (o), (u_int64_t *)(a), (c) >> 3)
+#define bus_space_read_raw_region_2(_t, _h, _o, _a, _c) \
+	((_t)->read_region_2((_h), (_o), (u_int16_t *)(_a), (_c) >> 1))
+#define bus_space_read_raw_region_4(_t, _h, _o, _a, _c) \
+	((_t)->read_region_4((_h), (_o), (u_int32_t *)(_a), (_c) >> 2))
+#define bus_space_read_raw_region_8(_t, _h, _o, _a, _c) \
+	((_t)->read_region_8((_h), (_o), (u_int64_t *)(_a), (_c) >> 3))
 
 /*
  *	void bus_space_write_N(bus_space_tag_t tag,
@@ -234,14 +244,19 @@ void	bus_space_read_region_8(bus_space_tag_t, bus_space_handle_t,
  * described by tag/handle/offset.
  */
 
-void	bus_space_write_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int8_t);
-void	bus_space_write_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int16_t);
-void	bus_space_write_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int32_t);
-void	bus_space_write_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int64_t);
+	void		(*write_1)(bus_space_handle_t, bus_size_t, u_int8_t);
+	void		(*write_2)(bus_space_handle_t, bus_size_t, u_int16_t);
+	void		(*write_4)(bus_space_handle_t, bus_size_t, u_int32_t);
+	void		(*write_8)(bus_space_handle_t, bus_size_t, u_int64_t);
+
+#define bus_space_write_1(_t, _h, _o, _v) \
+	((_t)->write_1((_h), (_o), (_v)))
+#define bus_space_write_2(_t, _h, _o, _v) \
+	((_t)->write_2((_h), (_o), (_v)))
+#define bus_space_write_4(_t, _h, _o, _v) \
+	((_t)->write_4((_h), (_o), (_v)))
+#define bus_space_write_8(_t, _h, _o, _v) \
+	((_t)->write_8((_h), (_o), (_v)))
 
 /*
  *	void bus_space_write_multi_N(bus_space_tag_t tag,
@@ -252,14 +267,23 @@ void	bus_space_write_8(bus_space_tag_t, bus_space_handle_t,
  * provided to bus space described by tag/handle/offset.
  */
 
-void	bus_space_write_multi_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int8_t *, bus_size_t);
-void	bus_space_write_multi_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int16_t *, bus_size_t);
-void	bus_space_write_multi_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int32_t *, bus_size_t);
-void	bus_space_write_multi_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*write_multi_1)(bus_space_handle_t,
+			    bus_size_t, const u_int8_t *, bus_size_t);
+	void		(*write_multi_2)(bus_space_handle_t,
+			    bus_size_t, const u_int16_t *, bus_size_t);
+	void		(*write_multi_4)(bus_space_handle_t,
+			    bus_size_t, const u_int32_t *, bus_size_t);
+	void		(*write_multi_8)(bus_space_handle_t,
+			    bus_size_t, const u_int64_t *, bus_size_t);
+
+#define bus_space_write_multi_1(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_1((_h), (_o), (_a), (_c)))
+#define bus_space_write_multi_2(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_2((_h), (_o), (_a), (_c)))
+#define bus_space_write_multi_4(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_4((_h), (_o), (_a), (_c)))
+#define bus_space_write_multi_8(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_8((_h), (_o), (_a), (_c)))
 
 /*
  *	void bus_space_write_raw_multi_N(bus_space_tag_t tag,
@@ -272,12 +296,12 @@ void	bus_space_write_multi_8(bus_space_tag_t, bus_space_handle_t,
  * possible byte-swapping should be done by these functions.
  */
 
-#define	bus_space_write_raw_multi_2(t, h, o, a, c) \
-    bus_space_write_multi_2((t), (h), (o), (const u_int16_t *)(a), (c) >> 1)
-#define	bus_space_write_raw_multi_4(t, h, o, a, c) \
-    bus_space_write_multi_4((t), (h), (o), (const u_int32_t *)(a), (c) >> 2)
-#define	bus_space_write_raw_multi_8(t, h, o, a, c) \
-    bus_space_write_multi_4((t), (h), (o), (const u_int64_t *)(a), (c) >> 3)
+#define bus_space_write_raw_multi_2(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_2((_h), (_o), (const u_int16_t *)(_a), (_c) >> 1))
+#define bus_space_write_raw_multi_4(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_4((_h), (_o), (const u_int32_t *)(_a), (_c) >> 2))
+#define bus_space_write_raw_multi_8(_t, _h, _o, _a, _c) \
+	((_t)->write_multi_8((_h), (_o), (const u_int64_t *)(_a), (_c) >> 3))
 
 /*
  *	void bus_space_write_region_N(bus_space_tag_t tag,
@@ -288,14 +312,23 @@ void	bus_space_write_multi_8(bus_space_tag_t, bus_space_handle_t,
  * to bus space described by tag/handle starting at `offset'.
  */
 
-void	bus_space_write_region_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int8_t *, bus_size_t);
-void	bus_space_write_region_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int16_t *, bus_size_t);
-void	bus_space_write_region_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int32_t *, bus_size_t);
-void	bus_space_write_region_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*write_region_1)(bus_space_handle_t,
+			    bus_size_t, const u_int8_t *, bus_size_t);
+	void		(*write_region_2)(bus_space_handle_t,
+			    bus_size_t, const u_int16_t *, bus_size_t);
+	void		(*write_region_4)(bus_space_handle_t,
+			    bus_size_t, const u_int32_t *, bus_size_t);
+	void		(*write_region_8)(bus_space_handle_t,
+			    bus_size_t, const u_int64_t *, bus_size_t);
+
+#define bus_space_write_region_1(_t, _h, _o, _a, _c) \
+	((_t)->write_region_1((_h), (_o), (_a), (_c)))
+#define bus_space_write_region_2(_t, _h, _o, _a, _c) \
+	((_t)->write_region_2((_h), (_o), (_a), (_c)))
+#define bus_space_write_region_4(_t, _h, _o, _a, _c) \
+	((_t)->write_region_4((_h), (_o), (_a), (_c)))
+#define bus_space_write_region_8(_t, _h, _o, _a, _c) \
+	((_t)->write_region_8((_h), (_o), (_a), (_c)))
 
 /*
  *	void bus_space_write_raw_region_N(bus_space_tag_t tag,
@@ -309,12 +342,12 @@ void	bus_space_write_region_8(bus_space_tag_t, bus_space_handle_t,
  * these functions.
  */
 
-#define	bus_space_write_raw_region_2(t, h, o, a, c) \
-    bus_space_write_region_2((t), (h), (o), (const u_int16_t *)(a), (c) >> 1)
-#define	bus_space_write_raw_region_4(t, h, o, a, c) \
-    bus_space_write_region_4((t), (h), (o), (const u_int32_t *)(a), (c) >> 2)
-#define	bus_space_write_raw_region_8(t, h, o, a, c) \
-    bus_space_write_region_8((t), (h), (o), (const u_int64_t *)(a), (c) >> 3)
+#define bus_space_write_raw_region_2(_t, _h, _o, _a, _c) \
+	((_t)->write_region_2((_h), (_o), (const u_int16_t *)(_a), (_c) >> 1))
+#define bus_space_write_raw_region_4(_t, _h, _o, _a, _c) \
+	((_t)->write_region_4((_h), (_o), (const u_int32_t *)(_a), (_c) >> 2))
+#define bus_space_write_raw_region_8(_t, _h, _o, _a, _c) \
+	((_t)->write_region_8((_h), (_o), (const u_int64_t *)(_a), (_c) >> 3))
 
 /*
  *	void bus_space_set_multi_N(bus_space_tag_t tag,
@@ -325,14 +358,23 @@ void	bus_space_write_region_8(bus_space_tag_t, bus_space_handle_t,
  * by tag/handle/offset `count' times.
  */
 
-void	bus_space_set_multi_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int8_t, size_t);
-void	bus_space_set_multi_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int16_t, size_t);
-void	bus_space_set_multi_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int32_t, size_t);
-void	bus_space_set_multi_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int64_t, size_t);
+	void		(*set_multi_1)(bus_space_handle_t,
+			    bus_size_t, u_int8_t, size_t);
+	void		(*set_multi_2)(bus_space_handle_t,
+			    bus_size_t, u_int16_t, size_t);
+	void		(*set_multi_4)(bus_space_handle_t,
+			    bus_size_t, u_int32_t, size_t);
+	void		(*set_multi_8)(bus_space_handle_t,
+			    bus_size_t, u_int64_t, size_t);
+
+#define bus_space_set_multi_1(_t, _h, _o, _a, _c) \
+	((_t)->set_multi_1((_h), (_o), (_a), (_c)))
+#define bus_space_set_multi_2(_t, _h, _o, _a, _c) \
+	((_t)->set_multi_2((_h), (_o), (_a), (_c)))
+#define bus_space_set_multi_4(_t, _h, _o, _a, _c) \
+	((_t)->set_multi_4((_h), (_o), (_a), (_c)))
+#define bus_space_set_multi_8(_t, _h, _o, _a, _c) \
+	((_t)->set_multi_8((_h), (_o), (_a), (_c)))
 
 /*
  *	void bus_space_set_region_N(bus_space_tag_t tag,
@@ -343,14 +385,23 @@ void	bus_space_set_multi_8(bus_space_tag_t, bus_space_handle_t,
  * by tag/handle starting at `offset'.
  */
 
-void	bus_space_set_region_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int8_t, size_t);
-void	bus_space_set_region_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int16_t, size_t);
-void	bus_space_set_region_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int32_t, size_t);
-void	bus_space_set_region_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, u_int64_t, size_t);
+	void		(*set_region_1)(bus_space_handle_t,
+			    bus_size_t, u_int8_t, size_t);
+	void		(*set_region_2)(bus_space_handle_t,
+			    bus_size_t, u_int16_t, size_t);
+	void		(*set_region_4)(bus_space_handle_t,
+			    bus_size_t, u_int32_t, size_t);
+	void		(*set_region_8)(bus_space_handle_t,
+			    bus_size_t, u_int64_t, size_t);
+
+#define bus_space_set_region_1(_t, _h, _o, _a, _c) \
+	((_t)->set_region_1((_h), (_o), (_a), (_c)))
+#define bus_space_set_region_2(_t, _h, _o, _a, _c) \
+	((_t)->set_region_2((_h), (_o), (_a), (_c)))
+#define bus_space_set_region_4(_t, _h, _o, _a, _c) \
+	((_t)->set_region_4((_h), (_o), (_a), (_c)))
+#define bus_space_set_region_8(_t, _h, _o, _a, _c) \
+	((_t)->set_region_8((_h), (_o), (_a), (_c)))
 
 /*
  *	void bus_space_copy_N(bus_space_tag_t tag,
@@ -362,14 +413,47 @@ void	bus_space_set_region_8(bus_space_tag_t, bus_space_handle_t,
  * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2.
  */
 
-void	bus_space_copy_1(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
-void	bus_space_copy_2(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
-void	bus_space_copy_4(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
-void	bus_space_copy_8(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
+	void		(*copy_1)(bus_space_handle_t,
+			    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
+	void		(*copy_2)(bus_space_handle_t,
+			    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
+	void		(*copy_4)(bus_space_handle_t,
+			    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
+	void		(*copy_8)(bus_space_handle_t,
+			    bus_size_t, bus_space_handle_t, bus_size_t, size_t);
+
+#define bus_space_copy_1(_t, _h1, _o1, _h2, _o2, _c) \
+	((_t)->copy_1((_h1), (_o1), (_h2), (_o2), (_c)))
+#define bus_space_copy_2(_t, _h1, _o1, _h2, _o2, _c) \
+	((_t)->copy_2((_h1), (_o1), (_h2), (_o2), (_c)))
+#define bus_space_copy_4(_t, _h1, _o1, _h2, _o2, _c) \
+	((_t)->copy_4((_h1), (_o1), (_h2), (_o2), (_c)))
+#define bus_space_copy_8(_t, _h1, _o1, _h2, _o2, _c) \
+	((_t)->copy_8((_h1), (_o1), (_h2), (_o2), (_c)))
+
+/*
+ *	void *bus_space_vaddr(bus_space_tag_t, bus_space_handle_t);
+ *
+ * Get the kernel virtual address for the mapped bus space.
+ * Only allowed for regions mapped with BUS_SPACE_MAP_LINEAR.
+ */
+	void *		(*vaddr)(bus_space_handle_t);
+
+#define bus_space_vaddr(_t, _h) \
+	((_t)->vaddr((_h)))
+
+/*      
+ *      paddr_t bus_space_mmap(bus_space_tag_t t, bus_addr_t base,
+ *          off_t offset, int prot, int flags);
+ * 
+ * Mmap an area of bus space.
+ */
+ 
+	paddr_t		(*mmap)(bus_addr_t, off_t, int, int);
+
+#define bus_space_mmap(_t, _a, _o, _p, _f) \
+	((_t)->mmap((_a), (_o), (_p), (_f)))
+};
 
 /*
  * Bus read/write barrier methods.
@@ -377,21 +461,43 @@ void	bus_space_copy_8(bus_space_tag_t, bus_space_handle_t,
 #define	BUS_SPACE_BARRIER_READ	0x01		/* force read barrier */
 #define	BUS_SPACE_BARRIER_WRITE	0x02		/* force write barrier */
 
-void	bus_space_barrier(bus_space_tag_t, bus_space_handle_t,
-	    bus_size_t, bus_size_t, int);
+static inline void
+bus_space_barrier(bus_space_tag_t space, bus_space_handle_t
+    handle, bus_size_t offset, bus_size_t length, int flags)
+{
+	switch (flags) {
+	case (BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE):
+		__asm __volatile("mfence");
+		break;
+	case BUS_SPACE_BARRIER_WRITE:
+		__asm __volatile("sfence");
+		break;
+	default:
+		__asm __volatile("lfence");
+		break;
+	}
+}
 
 #define	BUS_SPACE_MAP_CACHEABLE		0x0001
 #define	BUS_SPACE_MAP_LINEAR		0x0002
 #define	BUS_SPACE_MAP_PREFETCHABLE	0x0008
 
 /*
- *	void *bus_space_vaddr(bus_space_tag_t, bus_space_handle_t);
- *
- * Get the kernel virtual address for the mapped bus space.
- * Only allowed for regions mapped with BUS_SPACE_MAP_LINEAR.
+ * Values for the x86 bus space tag, not to be used directly by MI code.
+ */
+
+/* space is i/o space */
+extern const struct x86_bus_space_ops x86_bus_space_io_ops;
+#define	X86_BUS_SPACE_IO	(&x86_bus_space_io_ops)
+
+/* space is mem space */
+extern const struct x86_bus_space_ops x86_bus_space_mem_ops;
+#define X86_BUS_SPACE_MEM	(&x86_bus_space_mem_ops)
+
+/*
+ * bus_dma
  */
-#define bus_space_vaddr(t, h) \
-	((t) == X86_BUS_SPACE_IO ? (void *)(NULL) : (void *)(h))
+
 /*
  * Flags used in various bus DMA methods.
  */
@@ -639,13 +745,4 @@ int	sg_dmamap_load_physarray(bus_dma_tag_t, bus_dmamap_t, paddr_t *,
 int	sg_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
 	    bus_dma_segment_t *, int, int *, int);
 
-/*      
- *      paddr_t bus_space_mmap(bus_space_tag_t t, bus_addr_t base,
- *          off_t offset, int prot, int flags);
- * 
- * Mmap an area of bus space.
- */
- 
-paddr_t bus_space_mmap(bus_space_tag_t, bus_addr_t, off_t, int, int);
-
 #endif /* _MACHINE_BUS_H_ */