From d582203578a1f3d408e27bb9042e8635954cd320 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 2 May 2007 19:27:14 +0200 Subject: [PATCH] i386: PARAVIRT: Use patch site IDs computed from offset in paravirt_ops structure Use patch type identifiers derived from the offset of the operation in the paravirt_ops structure. This avoids having to maintain a separate enum for patch site types. Also, since the identifier is derived from the offset into paravirt_ops, the offset can be derived from the identifier. This is used to remove replicated information in the various callsite macros, which has been a source of bugs in the past. This patch also drops the fused save_fl+cli operation, which doesn't really add much and makes things more complex - specifically because it breaks the 1:1 relationship between identifiers and offsets. If this operation turns out to be particularly beneficial, then the right answer is to define a new entrypoint for it. Signed-off-by: Jeremy Fitzhardinge Signed-off-by: Andi Kleen Cc: Rusty Russell Cc: Zachary Amsden --- include/asm-i386/paravirt.h | 177 +++++++++++++++++++++++--------------------- 1 file changed, 92 insertions(+), 85 deletions(-) (limited to 'include/asm-i386/paravirt.h') diff --git a/include/asm-i386/paravirt.h b/include/asm-i386/paravirt.h index b4cc2fc4031e..1dbc01f4ed4d 100644 --- a/include/asm-i386/paravirt.h +++ b/include/asm-i386/paravirt.h @@ -4,19 +4,8 @@ * para-virtualization: those hooks are defined here. */ #ifdef CONFIG_PARAVIRT -#include #include -/* These are the most performance critical ops, so we want to be able to patch - * callers */ -#define PARAVIRT_IRQ_DISABLE 0 -#define PARAVIRT_IRQ_ENABLE 1 -#define PARAVIRT_RESTORE_FLAGS 2 -#define PARAVIRT_SAVE_FLAGS 3 -#define PARAVIRT_SAVE_FLAGS_IRQ_DISABLE 4 -#define PARAVIRT_INTERRUPT_RETURN 5 -#define PARAVIRT_STI_SYSEXIT 6 - /* Bitmask of what can be clobbered: usually at least eax. */ #define CLBR_NONE 0x0 #define CLBR_EAX 0x1 @@ -191,6 +180,28 @@ struct paravirt_ops extern struct paravirt_ops paravirt_ops; +#define PARAVIRT_PATCH(x) \ + (offsetof(struct paravirt_ops, x) / sizeof(void *)) + +#define paravirt_type(type) \ + [paravirt_typenum] "i" (PARAVIRT_PATCH(type)) +#define paravirt_clobber(clobber) \ + [paravirt_clobber] "i" (clobber) + +#define PARAVIRT_CALL "call *paravirt_ops+%c[paravirt_typenum]*4;" + +#define _paravirt_alt(insn_string, type, clobber) \ + "771:\n\t" insn_string "\n" "772:\n" \ + ".pushsection .parainstructions,\"a\"\n" \ + " .long 771b\n" \ + " .byte " type "\n" \ + " .byte 772b-771b\n" \ + " .short " clobber "\n" \ + ".popsection\n" + +#define paravirt_alt(insn_string) \ + _paravirt_alt(insn_string, "%c[paravirt_typenum]", "%c[paravirt_clobber]") + #define paravirt_enabled() (paravirt_ops.paravirt_enabled) static inline void load_esp0(struct tss_struct *tss, @@ -515,93 +526,89 @@ struct paravirt_patch_site { extern struct paravirt_patch_site __parainstructions[], __parainstructions_end[]; -#define paravirt_alt(insn_string, typenum, clobber) \ - "771:\n\t" insn_string "\n" "772:\n" \ - ".pushsection .parainstructions,\"a\"\n" \ - " .long 771b\n" \ - " .byte " __stringify(typenum) "\n" \ - " .byte 772b-771b\n" \ - " .short " __stringify(clobber) "\n" \ - ".popsection" - static inline unsigned long __raw_local_save_flags(void) { unsigned long f; - __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" - "call *%1;" - "popl %%edx; popl %%ecx", - PARAVIRT_SAVE_FLAGS, CLBR_NONE) - : "=a"(f): "m"(paravirt_ops.save_fl) - : "memory", "cc"); + asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" + PARAVIRT_CALL + "popl %%edx; popl %%ecx") + : "=a"(f) + : paravirt_type(save_fl), + paravirt_clobber(CLBR_NONE) + : "memory", "cc"); return f; } static inline void raw_local_irq_restore(unsigned long f) { - __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" - "call *%1;" - "popl %%edx; popl %%ecx", - PARAVIRT_RESTORE_FLAGS, CLBR_EAX) - : "=a"(f) : "m" (paravirt_ops.restore_fl), "0"(f) - : "memory", "cc"); + asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" + PARAVIRT_CALL + "popl %%edx; popl %%ecx") + : "=a"(f) + : "0"(f), + paravirt_type(restore_fl), + paravirt_clobber(CLBR_EAX) + : "memory", "cc"); } static inline void raw_local_irq_disable(void) { - __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" - "call *%0;" - "popl %%edx; popl %%ecx", - PARAVIRT_IRQ_DISABLE, CLBR_EAX) - : : "m" (paravirt_ops.irq_disable) - : "memory", "eax", "cc"); + asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" + PARAVIRT_CALL + "popl %%edx; popl %%ecx") + : + : paravirt_type(irq_disable), + paravirt_clobber(CLBR_EAX) + : "memory", "eax", "cc"); } static inline void raw_local_irq_enable(void) { - __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" - "call *%0;" - "popl %%edx; popl %%ecx", - PARAVIRT_IRQ_ENABLE, CLBR_EAX) - : : "m" (paravirt_ops.irq_enable) - : "memory", "eax", "cc"); + asm volatile(paravirt_alt("pushl %%ecx; pushl %%edx;" + PARAVIRT_CALL + "popl %%edx; popl %%ecx") + : + : paravirt_type(irq_enable), + paravirt_clobber(CLBR_EAX) + : "memory", "eax", "cc"); } static inline unsigned long __raw_local_irq_save(void) { unsigned long f; - __asm__ __volatile__(paravirt_alt( "pushl %%ecx; pushl %%edx;" - "call *%1; pushl %%eax;" - "call *%2; popl %%eax;" - "popl %%edx; popl %%ecx", - PARAVIRT_SAVE_FLAGS_IRQ_DISABLE, - CLBR_NONE) - : "=a"(f) - : "m" (paravirt_ops.save_fl), - "m" (paravirt_ops.irq_disable) - : "memory", "cc"); + f = __raw_local_save_flags(); + raw_local_irq_disable(); return f; } -#define CLI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \ - "call *paravirt_ops+%c[irq_disable];" \ - "popl %%edx; popl %%ecx", \ - PARAVIRT_IRQ_DISABLE, CLBR_EAX) +#define CLI_STRING \ + _paravirt_alt("pushl %%ecx; pushl %%edx;" \ + "call *paravirt_ops+%c[paravirt_cli_type]*4;" \ + "popl %%edx; popl %%ecx", \ + "%c[paravirt_cli_type]", "%c[paravirt_clobber]") + +#define STI_STRING \ + _paravirt_alt("pushl %%ecx; pushl %%edx;" \ + "call *paravirt_ops+%c[paravirt_sti_type]*4;" \ + "popl %%edx; popl %%ecx", \ + "%c[paravirt_sti_type]", "%c[paravirt_clobber]") -#define STI_STRING paravirt_alt("pushl %%ecx; pushl %%edx;" \ - "call *paravirt_ops+%c[irq_enable];" \ - "popl %%edx; popl %%ecx", \ - PARAVIRT_IRQ_ENABLE, CLBR_EAX) #define CLI_STI_CLOBBERS , "%eax" -#define CLI_STI_INPUT_ARGS \ +#define CLI_STI_INPUT_ARGS \ , \ - [irq_disable] "i" (offsetof(struct paravirt_ops, irq_disable)), \ - [irq_enable] "i" (offsetof(struct paravirt_ops, irq_enable)) + [paravirt_cli_type] "i" (PARAVIRT_PATCH(irq_disable)), \ + [paravirt_sti_type] "i" (PARAVIRT_PATCH(irq_enable)), \ + paravirt_clobber(CLBR_EAX) + +#undef PARAVIRT_CALL #else /* __ASSEMBLY__ */ -#define PARA_PATCH(ptype, clobbers, ops) \ +#define PARA_PATCH(off) ((off) / 4) + +#define PARA_SITE(ptype, clobbers, ops) \ 771:; \ ops; \ 772:; \ @@ -612,25 +619,25 @@ static inline unsigned long __raw_local_irq_save(void) .short clobbers; \ .popsection -#define INTERRUPT_RETURN \ - PARA_PATCH(PARAVIRT_INTERRUPT_RETURN, CLBR_ANY, \ - jmp *%cs:paravirt_ops+PARAVIRT_iret) - -#define DISABLE_INTERRUPTS(clobbers) \ - PARA_PATCH(PARAVIRT_IRQ_DISABLE, clobbers, \ - pushl %ecx; pushl %edx; \ - call *paravirt_ops+PARAVIRT_irq_disable; \ - popl %edx; popl %ecx) \ - -#define ENABLE_INTERRUPTS(clobbers) \ - PARA_PATCH(PARAVIRT_IRQ_ENABLE, clobbers, \ - pushl %ecx; pushl %edx; \ - call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ - popl %edx; popl %ecx) - -#define ENABLE_INTERRUPTS_SYSEXIT \ - PARA_PATCH(PARAVIRT_STI_SYSEXIT, CLBR_ANY, \ - jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) +#define INTERRUPT_RETURN \ + PARA_SITE(PARA_PATCH(PARAVIRT_iret), CLBR_ANY, \ + jmp *%cs:paravirt_ops+PARAVIRT_iret) + +#define DISABLE_INTERRUPTS(clobbers) \ + PARA_SITE(PARA_PATCH(PARAVIRT_irq_disable), clobbers, \ + pushl %ecx; pushl %edx; \ + call *%cs:paravirt_ops+PARAVIRT_irq_disable; \ + popl %edx; popl %ecx) \ + +#define ENABLE_INTERRUPTS(clobbers) \ + PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable), clobbers, \ + pushl %ecx; pushl %edx; \ + call *%cs:paravirt_ops+PARAVIRT_irq_enable; \ + popl %edx; popl %ecx) + +#define ENABLE_INTERRUPTS_SYSEXIT \ + PARA_SITE(PARA_PATCH(PARAVIRT_irq_enable_sysexit), CLBR_ANY, \ + jmp *%cs:paravirt_ops+PARAVIRT_irq_enable_sysexit) #define GET_CR0_INTO_EAX \ call *paravirt_ops+PARAVIRT_read_cr0 -- cgit v1.2.3-59-g8ed1b