819165fb34
Eric pointed out overly restrictive constraints in atomic64_set(), but there are issues throughout the file. In the cited case, %ebx and %ecx are inputs only (don't get changed by either of the two low level implementations). This was also the case elsewhere. Further in many cases early-clobber indicators were missing. Finally, the previous implementation rolled a custom alternative instruction macro from scratch, rather than using alternative_call() (which was introduced with the commit that the description of the change in question actually refers to). Adjusting has the benefit of not hiding referenced symbols from the compiler, which however requires them to be declared not just in the exporting source file (which, as a desirable side effect, in turn allows that exporting file to become a real 5-line stub). This patch does not eliminate the overly restrictive memory clobbers, however: Doing so would occasionally make the compiler set up a second register for accessing the memory object (to satisfy the added "m" constraint), and it's not clear which of the two non-optimal alternatives is better. v2: Re-do the declaration and exporting of the internal symbols. Reported-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: Jan Beulich <jbeulich@suse.com> Link: http://lkml.kernel.org/r/4F19A2A5020000780006E0D9@nat28.tlf.novell.com Cc: Luca Barbieri <luca@luca-barbieri.com> Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
195 lines
6.7 KiB
C
195 lines
6.7 KiB
C
#ifndef _ASM_X86_ALTERNATIVE_H
|
|
#define _ASM_X86_ALTERNATIVE_H
|
|
|
|
#include <linux/types.h>
|
|
#include <linux/stddef.h>
|
|
#include <linux/stringify.h>
|
|
#include <asm/asm.h>
|
|
|
|
/*
|
|
* Alternative inline assembly for SMP.
|
|
*
|
|
* The LOCK_PREFIX macro defined here replaces the LOCK and
|
|
* LOCK_PREFIX macros used everywhere in the source tree.
|
|
*
|
|
* SMP alternatives use the same data structures as the other
|
|
* alternatives and the X86_FEATURE_UP flag to indicate the case of a
|
|
* UP system running a SMP kernel. The existing apply_alternatives()
|
|
* works fine for patching a SMP kernel for UP.
|
|
*
|
|
* The SMP alternative tables can be kept after boot and contain both
|
|
* UP and SMP versions of the instructions to allow switching back to
|
|
* SMP at runtime, when hotplugging in a new CPU, which is especially
|
|
* useful in virtualized environments.
|
|
*
|
|
* The very common lock prefix is handled as special case in a
|
|
* separate table which is a pure address list without replacement ptr
|
|
* and size information. That keeps the table sizes small.
|
|
*/
|
|
|
|
#ifdef CONFIG_SMP
|
|
#define LOCK_PREFIX_HERE \
|
|
".section .smp_locks,\"a\"\n" \
|
|
".balign 4\n" \
|
|
".long 671f - .\n" /* offset */ \
|
|
".previous\n" \
|
|
"671:"
|
|
|
|
#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
|
|
|
|
#else /* ! CONFIG_SMP */
|
|
#define LOCK_PREFIX_HERE ""
|
|
#define LOCK_PREFIX ""
|
|
#endif
|
|
|
|
struct alt_instr {
|
|
s32 instr_offset; /* original instruction */
|
|
s32 repl_offset; /* offset to replacement instruction */
|
|
u16 cpuid; /* cpuid bit set for replacement */
|
|
u8 instrlen; /* length of original instruction */
|
|
u8 replacementlen; /* length of new instruction, <= instrlen */
|
|
};
|
|
|
|
extern void alternative_instructions(void);
|
|
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
|
|
|
|
struct module;
|
|
|
|
#ifdef CONFIG_SMP
|
|
extern void alternatives_smp_module_add(struct module *mod, char *name,
|
|
void *locks, void *locks_end,
|
|
void *text, void *text_end);
|
|
extern void alternatives_smp_module_del(struct module *mod);
|
|
extern void alternatives_smp_switch(int smp);
|
|
extern int alternatives_text_reserved(void *start, void *end);
|
|
extern bool skip_smp_alternatives;
|
|
#else
|
|
static inline void alternatives_smp_module_add(struct module *mod, char *name,
|
|
void *locks, void *locks_end,
|
|
void *text, void *text_end) {}
|
|
static inline void alternatives_smp_module_del(struct module *mod) {}
|
|
static inline void alternatives_smp_switch(int smp) {}
|
|
static inline int alternatives_text_reserved(void *start, void *end)
|
|
{
|
|
return 0;
|
|
}
|
|
#endif /* CONFIG_SMP */
|
|
|
|
/* alternative assembly primitive: */
|
|
#define ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
\
|
|
"661:\n\t" oldinstr "\n662:\n" \
|
|
".section .altinstructions,\"a\"\n" \
|
|
" .long 661b - .\n" /* label */ \
|
|
" .long 663f - .\n" /* new instruction */ \
|
|
" .word " __stringify(feature) "\n" /* feature bit */ \
|
|
" .byte 662b-661b\n" /* sourcelen */ \
|
|
" .byte 664f-663f\n" /* replacementlen */ \
|
|
".previous\n" \
|
|
".section .discard,\"aw\",@progbits\n" \
|
|
" .byte 0xff + (664f-663f) - (662b-661b)\n" /* rlen <= slen */ \
|
|
".previous\n" \
|
|
".section .altinstr_replacement, \"ax\"\n" \
|
|
"663:\n\t" newinstr "\n664:\n" /* replacement */ \
|
|
".previous"
|
|
|
|
/*
|
|
* This must be included *after* the definition of ALTERNATIVE due to
|
|
* <asm/arch_hweight.h>
|
|
*/
|
|
#include <asm/cpufeature.h>
|
|
|
|
/*
|
|
* Alternative instructions for different CPU types or capabilities.
|
|
*
|
|
* This allows to use optimized instructions even on generic binary
|
|
* kernels.
|
|
*
|
|
* length of oldinstr must be longer or equal the length of newinstr
|
|
* It can be padded with nops as needed.
|
|
*
|
|
* For non barrier like inlines please define new variants
|
|
* without volatile and memory clobber.
|
|
*/
|
|
#define alternative(oldinstr, newinstr, feature) \
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
|
|
|
|
/*
|
|
* Alternative inline assembly with input.
|
|
*
|
|
* Pecularities:
|
|
* No memory clobber here.
|
|
* Argument numbers start with 1.
|
|
* Best is to use constraints that are fixed size (like (%1) ... "r")
|
|
* If you use variable sized constraints like "m" or "g" in the
|
|
* replacement make sure to pad to the worst case length.
|
|
* Leaving an unused argument 0 to keep API compatibility.
|
|
*/
|
|
#define alternative_input(oldinstr, newinstr, feature, input...) \
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
: : "i" (0), ## input)
|
|
|
|
/* Like alternative_input, but with a single output argument */
|
|
#define alternative_io(oldinstr, newinstr, feature, output, input...) \
|
|
asm volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
|
|
: output : "i" (0), ## input)
|
|
|
|
/* Like alternative_io, but for replacing a direct call with another one. */
|
|
#define alternative_call(oldfunc, newfunc, feature, output, input...) \
|
|
asm volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
|
|
: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
|
|
|
|
/*
|
|
* use this macro(s) if you need more than one output parameter
|
|
* in alternative_io
|
|
*/
|
|
#define ASM_OUTPUT2(a...) a
|
|
|
|
/*
|
|
* use this macro if you need clobbers but no inputs in
|
|
* alternative_{input,io,call}()
|
|
*/
|
|
#define ASM_NO_INPUT_CLOBBER(clbr...) "i" (0) : clbr
|
|
|
|
struct paravirt_patch_site;
|
|
#ifdef CONFIG_PARAVIRT
|
|
void apply_paravirt(struct paravirt_patch_site *start,
|
|
struct paravirt_patch_site *end);
|
|
#else
|
|
static inline void apply_paravirt(struct paravirt_patch_site *start,
|
|
struct paravirt_patch_site *end)
|
|
{}
|
|
#define __parainstructions NULL
|
|
#define __parainstructions_end NULL
|
|
#endif
|
|
|
|
extern void *text_poke_early(void *addr, const void *opcode, size_t len);
|
|
|
|
/*
|
|
* Clear and restore the kernel write-protection flag on the local CPU.
|
|
* Allows the kernel to edit read-only pages.
|
|
* Side-effect: any interrupt handler running between save and restore will have
|
|
* the ability to write to read-only pages.
|
|
*
|
|
* Warning:
|
|
* Code patching in the UP case is safe if NMIs and MCE handlers are stopped and
|
|
* no thread can be preempted in the instructions being modified (no iret to an
|
|
* invalid instruction possible) or if the instructions are changed from a
|
|
* consistent state to another consistent state atomically.
|
|
* More care must be taken when modifying code in the SMP case because of
|
|
* Intel's errata. text_poke_smp() takes care that errata, but still
|
|
* doesn't support NMI/MCE handler code modifying.
|
|
* On the local CPU you need to be protected again NMI or MCE handlers seeing an
|
|
* inconsistent instruction while you patch.
|
|
*/
|
|
struct text_poke_param {
|
|
void *addr;
|
|
const void *opcode;
|
|
size_t len;
|
|
};
|
|
|
|
extern void *text_poke(void *addr, const void *opcode, size_t len);
|
|
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
|
|
extern void text_poke_smp_batch(struct text_poke_param *params, int n);
|
|
|
|
#endif /* _ASM_X86_ALTERNATIVE_H */
|