From 64e2c6738b4d49d69d697b5887f72ad07c206ab3 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 5 Apr 2018 09:09:09 -0400 Subject: io: define several IO & PIO barrier types for the asm-generic version Getting ready to harden readX()/writeX() and inX()/outX() semantics for the generic implementation. Defining two set of macros as __io_br() and __io_ar() to indicate actions to be taken before and after MMIO read. Defining two set of macros as __io_bw() and __io_aw() to indicate actions to be taken before and after MMIO write. Defining two set of macros as __io_pbw() and __io_paw() to indicate actions to be taken before and after Port IO write. Defining two set of macros as __io_pbr() and __io_par() to indicate actions to be taken before and after Port IO read. If rmb() is available for the architecture, prefer rmb() as the default implementation of __io_ar()/__io_par(). If wmb() is available for the architecture, prefer wmb() as the default implementation of __io_bw()/__io_pbw(). Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index b4531e3b2120..570433b34180 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -25,6 +25,50 @@ #define mmiowb() do {} while (0) #endif +#ifndef __io_br +#define __io_br() barrier() +#endif + +/* prevent prefetching of coherent DMA data ahead of a dma-complete */ +#ifndef __io_ar +#ifdef rmb +#define __io_ar() rmb() +#else +#define __io_ar() barrier() +#endif +#endif + +/* flush writes to coherent DMA data before possibly triggering a DMA read */ +#ifndef __io_bw +#ifdef wmb +#define __io_bw() wmb() +#else +#define __io_bw() barrier() +#endif +#endif + +/* serialize device access against a spin_unlock, usually handled there. */ +#ifndef __io_aw +#define __io_aw() barrier() +#endif + +#ifndef __io_pbw +#define __io_pbw() __io_bw() +#endif + +#ifndef __io_paw +#define __io_paw() __io_aw() +#endif + +#ifndef __io_pbr +#define __io_pbr() __io_br() +#endif + +#ifndef __io_par +#define __io_par() __io_ar() +#endif + + /* * __raw_{read,write}{b,w,l,q}() access memory in native endianness. * -- cgit v1.2.3-59-g8ed1b From 032d59e1cde9dd71bb5918e1f6529776623ee86b Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 5 Apr 2018 09:09:10 -0400 Subject: io: define stronger ordering for the default readX() implementation The default implementation of mapping readX() to __raw_readX() is wrong. readX() has stronger ordering semantics. Compiler is allowed to reorder __raw_readX() against the memory accesses following register read. Use the previously defined __io_ar() and __io_br() macros to harden code generation according to architecture support. Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index 570433b34180..d27e8af9dd5a 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -154,7 +154,12 @@ static inline void __raw_writeq(u64 value, volatile void __iomem *addr) #define readb readb static inline u8 readb(const volatile void __iomem *addr) { - return __raw_readb(addr); + u8 val; + + __io_br(); + val = __raw_readb(addr); + __io_ar(); + return val; } #endif @@ -162,7 +167,12 @@ static inline u8 readb(const volatile void __iomem *addr) #define readw readw static inline u16 readw(const volatile void __iomem *addr) { - return __le16_to_cpu(__raw_readw(addr)); + u16 val; + + __io_br(); + val = __le16_to_cpu(__raw_readw(addr)); + __io_ar(); + return val; } #endif @@ -170,7 +180,12 @@ static inline u16 readw(const volatile void __iomem *addr) #define readl readl static inline u32 readl(const volatile void __iomem *addr) { - return __le32_to_cpu(__raw_readl(addr)); + u32 val; + + __io_br(); + val = __le32_to_cpu(__raw_readl(addr)); + __io_ar(); + return val; } #endif @@ -179,7 +194,12 @@ static inline u32 readl(const volatile void __iomem *addr) #define readq readq static inline u64 readq(const volatile void __iomem *addr) { - return __le64_to_cpu(__raw_readq(addr)); + u64 val; + + __io_br(); + val = __le64_to_cpu(__raw_readq(addr)); + __io_ar(); + return val; } #endif #endif /* CONFIG_64BIT */ -- cgit v1.2.3-59-g8ed1b From 755bd04aaf4bf7c49ce8aad2677ea4d14271fc46 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 5 Apr 2018 09:09:11 -0400 Subject: io: define stronger ordering for the default writeX() implementation The default implementation of mapping writeX() to __raw_writeX() is wrong. writeX() has stronger ordering semantics. Compiler is allowed to reorder memory writes against __raw_writeX(). Use the previously defined __io_aw() and __io_bw() macros to harden code generation according to architecture support. Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index d27e8af9dd5a..964725e4f459 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -208,7 +208,9 @@ static inline u64 readq(const volatile void __iomem *addr) #define writeb writeb static inline void writeb(u8 value, volatile void __iomem *addr) { + __io_bw(); __raw_writeb(value, addr); + __io_aw(); } #endif @@ -216,7 +218,9 @@ static inline void writeb(u8 value, volatile void __iomem *addr) #define writew writew static inline void writew(u16 value, volatile void __iomem *addr) { + __io_bw(); __raw_writew(cpu_to_le16(value), addr); + __io_aw(); } #endif @@ -224,7 +228,9 @@ static inline void writew(u16 value, volatile void __iomem *addr) #define writel writel static inline void writel(u32 value, volatile void __iomem *addr) { + __io_bw(); __raw_writel(__cpu_to_le32(value), addr); + __io_aw(); } #endif @@ -233,7 +239,9 @@ static inline void writel(u32 value, volatile void __iomem *addr) #define writeq writeq static inline void writeq(u64 value, volatile void __iomem *addr) { + __io_bw(); __raw_writeq(__cpu_to_le64(value), addr); + __io_aw(); } #endif #endif /* CONFIG_64BIT */ -- cgit v1.2.3-59-g8ed1b From a7851aa54c0cdd2f62457b72aee0da5d5586e514 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 5 Apr 2018 09:09:12 -0400 Subject: io: change outX() to have their own IO barrier overrides Open code writeX() inside outX() so that outX() variants have their own overrideable Port IO barrier combinations as __io_pbw() and __io_paw() for actions to be taken before port IO and after port IO write. Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index 964725e4f459..53226d907c45 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -457,7 +457,9 @@ static inline u32 inl(unsigned long addr) #define outb outb static inline void outb(u8 value, unsigned long addr) { - writeb(value, PCI_IOBASE + addr); + __io_pbw(); + __raw_writeb(value, PCI_IOBASE + addr); + __io_paw(); } #endif @@ -465,7 +467,9 @@ static inline void outb(u8 value, unsigned long addr) #define outw outw static inline void outw(u16 value, unsigned long addr) { - writew(value, PCI_IOBASE + addr); + __io_pbw(); + __raw_writew(cpu_to_le16(value), PCI_IOBASE + addr); + __io_paw(); } #endif @@ -473,7 +477,9 @@ static inline void outw(u16 value, unsigned long addr) #define outl outl static inline void outl(u32 value, unsigned long addr) { - writel(value, PCI_IOBASE + addr); + __io_pbw(); + __raw_writel(cpu_to_le32(value), PCI_IOBASE + addr); + __io_paw(); } #endif -- cgit v1.2.3-59-g8ed1b From 87fe2d543f817300e13f0ea683f38c122737856e Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Thu, 5 Apr 2018 09:09:13 -0400 Subject: io: change inX() to have their own IO barrier overrides Open code readX() inside inX() so that inX() variants have their own overrideable Port IO barrier combinations as __io_pbr() and __io_par() for actions to be taken before port IO and after port IO read. Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index 53226d907c45..578b6883dd6a 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -433,7 +433,12 @@ static inline void writesq(volatile void __iomem *addr, const void *buffer, #define inb inb static inline u8 inb(unsigned long addr) { - return readb(PCI_IOBASE + addr); + u8 val; + + __io_pbr(); + val = __raw_readb(PCI_IOBASE + addr); + __io_par(); + return val; } #endif @@ -441,7 +446,12 @@ static inline u8 inb(unsigned long addr) #define inw inw static inline u16 inw(unsigned long addr) { - return readw(PCI_IOBASE + addr); + u16 val; + + __io_pbr(); + val = __le16_to_cpu(__raw_readw(PCI_IOBASE + addr)); + __io_par(); + return val; } #endif @@ -449,7 +459,12 @@ static inline u16 inw(unsigned long addr) #define inl inl static inline u32 inl(unsigned long addr) { - return readl(PCI_IOBASE + addr); + u32 val; + + __io_pbr(); + val = __le32_to_cpu(__raw_readl(PCI_IOBASE + addr)); + __io_par(); + return val; } #endif -- cgit v1.2.3-59-g8ed1b From 8875c55437617fa4351070656bd78e17ed8284a5 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Fri, 6 Apr 2018 14:02:45 -0400 Subject: io: change readX_relaxed() to remove barriers Now that we hardened readX() API in asm-generic version, readX_relaxed() API is violating the rules when readX_relaxed() == readX() in the default implementation. The relaxed API shouldn't have any barriers in it and it doesn't provide any ordering with respect to the memory transactions. The only requirement is for reads to be ordered with respect to each other. This is achieved by the volatile in the __raw_readX() API. Open code the relaxed API and remove any barriers in it. Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index 578b6883dd6a..fa0975da0cec 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -252,19 +252,35 @@ static inline void writeq(u64 value, volatile void __iomem *addr) * accesses. */ #ifndef readb_relaxed -#define readb_relaxed readb +#define readb_relaxed readb_relaxed +static inline u8 readb_relaxed(const volatile void __iomem *addr) +{ + return __raw_readb(addr); +} #endif #ifndef readw_relaxed -#define readw_relaxed readw +#define readw_relaxed readw_relaxed +static inline u16 readw_relaxed(const volatile void __iomem *addr) +{ + return __le16_to_cpu(__raw_readw(addr)); +} #endif #ifndef readl_relaxed -#define readl_relaxed readl +#define readl_relaxed readl_relaxed +static inline u32 readl_relaxed(const volatile void __iomem *addr) +{ + return __le32_to_cpu(__raw_readl(addr)); +} #endif #if defined(readq) && !defined(readq_relaxed) -#define readq_relaxed readq +#define readq_relaxed readq_relaxed +static inline u64 readq_relaxed(const volatile void __iomem *addr) +{ + return __le64_to_cpu(__raw_readq(addr)); +} #endif #ifndef writeb_relaxed -- cgit v1.2.3-59-g8ed1b From a71e7c44ffb7baea0c0795824afc34cc0bc1a301 Mon Sep 17 00:00:00 2001 From: Sinan Kaya Date: Fri, 6 Apr 2018 14:02:46 -0400 Subject: io: change writeX_relaxed() to remove barriers Now that we hardened writeX() API in asm-generic version, writeX_relaxed() API is violating the rules when writeX_relaxed() == writeX() in the default implementation. The relaxed API shouldn't have any barriers in it and it doesn't provide any ordering with respect to the memory transactions. The only requirement is for writes to be ordered with respect to each other. This is achieved by the volatile in the __raw_writeX() API. Open code the relaxed API and remove any barriers in it. Signed-off-by: Sinan Kaya Signed-off-by: Arnd Bergmann --- include/asm-generic/io.h | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/asm-generic/io.h b/include/asm-generic/io.h index fa0975da0cec..f4a149403be9 100644 --- a/include/asm-generic/io.h +++ b/include/asm-generic/io.h @@ -284,19 +284,35 @@ static inline u64 readq_relaxed(const volatile void __iomem *addr) #endif #ifndef writeb_relaxed -#define writeb_relaxed writeb +#define writeb_relaxed writeb_relaxed +static inline void writeb_relaxed(u8 value, volatile void __iomem *addr) +{ + __raw_writeb(value, addr); +} #endif #ifndef writew_relaxed -#define writew_relaxed writew +#define writew_relaxed writew_relaxed +static inline void writew_relaxed(u16 value, volatile void __iomem *addr) +{ + __raw_writew(cpu_to_le16(value), addr); +} #endif #ifndef writel_relaxed -#define writel_relaxed writel +#define writel_relaxed writel_relaxed +static inline void writel_relaxed(u32 value, volatile void __iomem *addr) +{ + __raw_writel(__cpu_to_le32(value), addr); +} #endif #if defined(writeq) && !defined(writeq_relaxed) -#define writeq_relaxed writeq +#define writeq_relaxed writeq_relaxed +static inline void writeq_relaxed(u64 value, volatile void __iomem *addr) +{ + __raw_writeq(__cpu_to_le64(value), addr); +} #endif /* -- cgit v1.2.3-59-g8ed1b