From a8cccdd954732a558d481407ab7c3106b89c34ae Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Fri, 11 Dec 2020 10:46:24 -0800 Subject: init: lto: ensure initcall ordering With LTO, the compiler doesn't necessarily obey the link order for initcalls, and initcall variables need globally unique names to avoid collisions at link time. This change exports __KBUILD_MODNAME and adds the initcall_id() macro, which uses it together with __COUNTER__ and __LINE__ to help ensure these variables have unique names, and moves each variable to its own section when LTO is enabled, so the correct order can be specified using a linker script. The generate_initcall_ordering.pl script uses nm to find initcalls from the object files passed to the linker, and generates a linker script that specifies the same order for initcalls that we would have without LTO. With LTO enabled, the script is called in link-vmlinux.sh through jobserver-exec to limit the number of jobs spawned. Signed-off-by: Sami Tolvanen Reviewed-by: Kees Cook Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20201211184633.3213045-8-samitolvanen@google.com --- include/linux/init.h | 52 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 7 deletions(-) (limited to 'include/linux') diff --git a/include/linux/init.h b/include/linux/init.h index e668832ef66a..a57bf0dfd75f 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -184,19 +184,57 @@ extern bool initcall_debug; * as KEEP() in the linker script. */ +/* Format: ____ */ +#define __initcall_id(fn) \ + __PASTE(__KBUILD_MODNAME, \ + __PASTE(__, \ + __PASTE(__COUNTER__, \ + __PASTE(_, \ + __PASTE(__LINE__, \ + __PASTE(_, fn)))))) + +/* Format: ____ */ +#define __initcall_name(prefix, __iid, id) \ + __PASTE(__, \ + __PASTE(prefix, \ + __PASTE(__, \ + __PASTE(__iid, id)))) + +#ifdef CONFIG_LTO_CLANG +/* + * With LTO, the compiler doesn't necessarily obey link order for + * initcalls. In order to preserve the correct order, we add each + * variable into its own section and generate a linker script (in + * scripts/link-vmlinux.sh) to specify the order of the sections. + */ +#define __initcall_section(__sec, __iid) \ + #__sec ".init.." #__iid +#else +#define __initcall_section(__sec, __iid) \ + #__sec ".init" +#endif + #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS -#define ___define_initcall(fn, id, __sec) \ +#define ____define_initcall(fn, __name, __sec) \ __ADDRESSABLE(fn) \ - asm(".section \"" #__sec ".init\", \"a\" \n" \ - "__initcall_" #fn #id ": \n" \ + asm(".section \"" __sec "\", \"a\" \n" \ + __stringify(__name) ": \n" \ ".long " #fn " - . \n" \ ".previous \n"); #else -#define ___define_initcall(fn, id, __sec) \ - static initcall_t __initcall_##fn##id __used \ - __attribute__((__section__(#__sec ".init"))) = fn; +#define ____define_initcall(fn, __name, __sec) \ + static initcall_t __name __used \ + __attribute__((__section__(__sec))) = fn; #endif +#define __unique_initcall(fn, id, __sec, __iid) \ + ____define_initcall(fn, \ + __initcall_name(initcall, __iid, id), \ + __initcall_section(__sec, __iid)) + +#define ___define_initcall(fn, id, __sec) \ + __unique_initcall(fn, id, __sec, __initcall_id(fn)) + #define __define_initcall(fn, id) ___define_initcall(fn, id, .initcall##id) /* @@ -236,7 +274,7 @@ extern bool initcall_debug; #define __exitcall(fn) \ static exitcall_t __exitcall_##fn __exit_call = fn -#define console_initcall(fn) ___define_initcall(fn,, .con_initcall) +#define console_initcall(fn) ___define_initcall(fn, con, .con_initcall) struct obs_kernel_param { const char *str; -- cgit v1.2.3-59-g8ed1b From 3578ad11f3fba07e64c26d8db68cfd3dde28c59e Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Fri, 11 Dec 2020 10:46:25 -0800 Subject: init: lto: fix PREL32 relocations With LTO, the compiler can rename static functions to avoid global naming collisions. As initcall functions are typically static, renaming can break references to them in inline assembly. This change adds a global stub with a stable name for each initcall to fix the issue when PREL32 relocations are used. Signed-off-by: Sami Tolvanen Reviewed-by: Kees Cook Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20201211184633.3213045-9-samitolvanen@google.com --- include/linux/init.h | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'include/linux') diff --git a/include/linux/init.h b/include/linux/init.h index a57bf0dfd75f..a01f01c1a5c5 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -209,26 +209,49 @@ extern bool initcall_debug; */ #define __initcall_section(__sec, __iid) \ #__sec ".init.." #__iid + +/* + * With LTO, the compiler can rename static functions to avoid + * global naming collisions. We use a global stub function for + * initcalls to create a stable symbol name whose address can be + * taken in inline assembly when PREL32 relocations are used. + */ +#define __initcall_stub(fn, __iid, id) \ + __initcall_name(initstub, __iid, id) + +#define __define_initcall_stub(__stub, fn) \ + int __init __stub(void); \ + int __init __stub(void) \ + { \ + return fn(); \ + } \ + __ADDRESSABLE(__stub) #else #define __initcall_section(__sec, __iid) \ #__sec ".init" + +#define __initcall_stub(fn, __iid, id) fn + +#define __define_initcall_stub(__stub, fn) \ + __ADDRESSABLE(fn) #endif #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS -#define ____define_initcall(fn, __name, __sec) \ - __ADDRESSABLE(fn) \ +#define ____define_initcall(fn, __stub, __name, __sec) \ + __define_initcall_stub(__stub, fn) \ asm(".section \"" __sec "\", \"a\" \n" \ __stringify(__name) ": \n" \ - ".long " #fn " - . \n" \ + ".long " __stringify(__stub) " - . \n" \ ".previous \n"); #else -#define ____define_initcall(fn, __name, __sec) \ +#define ____define_initcall(fn, __unused, __name, __sec) \ static initcall_t __name __used \ __attribute__((__section__(__sec))) = fn; #endif #define __unique_initcall(fn, id, __sec, __iid) \ ____define_initcall(fn, \ + __initcall_stub(fn, __iid, id), \ __initcall_name(initcall, __iid, id), \ __initcall_section(__sec, __iid)) -- cgit v1.2.3-59-g8ed1b From 09a4e4d9c52a3c5e39e4f409b2c083ab13c6afc2 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Fri, 11 Dec 2020 10:46:26 -0800 Subject: PCI: Fix PREL32 relocations for LTO With Clang's Link Time Optimization (LTO), the compiler can rename static functions to avoid global naming collisions. As PCI fixup functions are typically static, renaming can break references to them in inline assembly. This change adds a global stub to DECLARE_PCI_FIXUP_SECTION to fix the issue when PREL32 relocations are used. Signed-off-by: Sami Tolvanen Acked-by: Bjorn Helgaas Reviewed-by: Kees Cook Signed-off-by: Kees Cook Link: https://lore.kernel.org/r/20201211184633.3213045-10-samitolvanen@google.com --- include/linux/pci.h | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) (limited to 'include/linux') diff --git a/include/linux/pci.h b/include/linux/pci.h index b32126d26997..2276e5d00192 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1917,7 +1917,7 @@ enum pci_fixup_pass { }; #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS -#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ +#define ___DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ class_shift, hook) \ __ADDRESSABLE(hook) \ asm(".section " #sec ", \"a\" \n" \ @@ -1926,10 +1926,33 @@ enum pci_fixup_pass { ".long " #class ", " #class_shift " \n" \ ".long " #hook " - . \n" \ ".previous \n"); + +/* + * Clang's LTO may rename static functions in C, but has no way to + * handle such renamings when referenced from inline asm. To work + * around this, create global C stubs for these cases. + */ +#ifdef CONFIG_LTO_CLANG +#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ + class_shift, hook, stub) \ + void stub(struct pci_dev *dev); \ + void stub(struct pci_dev *dev) \ + { \ + hook(dev); \ + } \ + ___DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ + class_shift, stub) +#else +#define __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ + class_shift, hook, stub) \ + ___DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ + class_shift, hook) +#endif + #define DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ class_shift, hook) \ __DECLARE_PCI_FIXUP_SECTION(sec, name, vendor, device, class, \ - class_shift, hook) + class_shift, hook, __UNIQUE_ID(hook)) #else /* Anonymous variables would be nice... */ #define DECLARE_PCI_FIXUP_SECTION(section, name, vendor, device, class, \ -- cgit v1.2.3-59-g8ed1b