113fc5a6e8
xchg() and cmpxchg() modify their memory operands, not merely read them. For some versions of gcc the "memory" clobber has apparently dealt with the situation, but not for all. Originally-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: H. Peter Anvin <hpa@zytor.com> Cc: Glauber Costa <glommer@redhat.com> Cc: Avi Kivity <avi@redhat.com> Cc: Peter Palfrader <peter@palfrader.org> Cc: Greg KH <gregkh@suse.de> Cc: Alan Cox <alan@lxorguk.ukuu.org.uk> Cc: Zachary Amsden <zamsden@redhat.com> Cc: Marcelo Tosatti <mtosatti@redhat.com> Cc: <stable@kernel.org> LKML-Reference: <4C4F7277.8050306@zytor.com>
303 lines
8.4 KiB
C
303 lines
8.4 KiB
C
#ifndef _ASM_X86_CMPXCHG_32_H
|
|
#define _ASM_X86_CMPXCHG_32_H
|
|
|
|
#include <linux/bitops.h> /* for LOCK_PREFIX */
|
|
|
|
/*
|
|
* Note: if you use set64_bit(), __cmpxchg64(), or their variants, you
|
|
* you need to test for the feature in boot_cpu_data.
|
|
*/
|
|
|
|
extern void __xchg_wrong_size(void);
|
|
|
|
/*
|
|
* Note: no "lock" prefix even on SMP: xchg always implies lock anyway
|
|
* Note 2: xchg has side effect, so that attribute volatile is necessary,
|
|
* but generally the primitive is invalid, *ptr is output argument. --ANK
|
|
*/
|
|
|
|
struct __xchg_dummy {
|
|
unsigned long a[100];
|
|
};
|
|
#define __xg(x) ((struct __xchg_dummy *)(x))
|
|
|
|
#define __xchg(x, ptr, size) \
|
|
({ \
|
|
__typeof(*(ptr)) __x = (x); \
|
|
switch (size) { \
|
|
case 1: \
|
|
asm volatile("xchgb %b0,%1" \
|
|
: "=q" (__x), "+m" (*__xg(ptr)) \
|
|
: "0" (__x) \
|
|
: "memory"); \
|
|
break; \
|
|
case 2: \
|
|
asm volatile("xchgw %w0,%1" \
|
|
: "=r" (__x), "+m" (*__xg(ptr)) \
|
|
: "0" (__x) \
|
|
: "memory"); \
|
|
break; \
|
|
case 4: \
|
|
asm volatile("xchgl %0,%1" \
|
|
: "=r" (__x), "+m" (*__xg(ptr)) \
|
|
: "0" (__x) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
__xchg_wrong_size(); \
|
|
} \
|
|
__x; \
|
|
})
|
|
|
|
#define xchg(ptr, v) \
|
|
__xchg((v), (ptr), sizeof(*ptr))
|
|
|
|
/*
|
|
* The semantics of XCHGCMP8B are a bit strange, this is why
|
|
* there is a loop and the loading of %%eax and %%edx has to
|
|
* be inside. This inlines well in most cases, the cached
|
|
* cost is around ~38 cycles. (in the future we might want
|
|
* to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that
|
|
* might have an implicit FPU-save as a cost, so it's not
|
|
* clear which path to go.)
|
|
*
|
|
* cmpxchg8b must be used with the lock prefix here to allow
|
|
* the instruction to be executed atomically, see page 3-102
|
|
* of the instruction set reference 24319102.pdf. We need
|
|
* the reader side to see the coherent 64bit value.
|
|
*/
|
|
static inline void __set_64bit(unsigned long long *ptr,
|
|
unsigned int low, unsigned int high)
|
|
{
|
|
asm volatile("\n1:\t"
|
|
"movl (%1), %%eax\n\t"
|
|
"movl 4(%1), %%edx\n\t"
|
|
LOCK_PREFIX "cmpxchg8b (%1)\n\t"
|
|
"jnz 1b"
|
|
: "=m" (*ptr)
|
|
: "D" (ptr),
|
|
"b" (low),
|
|
"c" (high)
|
|
: "ax", "dx", "memory");
|
|
}
|
|
|
|
static inline void __set_64bit_constant(unsigned long long *ptr,
|
|
unsigned long long value)
|
|
{
|
|
__set_64bit(ptr, (unsigned int)value, (unsigned int)(value >> 32));
|
|
}
|
|
|
|
#define ll_low(x) *(((unsigned int *)&(x)) + 0)
|
|
#define ll_high(x) *(((unsigned int *)&(x)) + 1)
|
|
|
|
static inline void __set_64bit_var(unsigned long long *ptr,
|
|
unsigned long long value)
|
|
{
|
|
__set_64bit(ptr, ll_low(value), ll_high(value));
|
|
}
|
|
|
|
#define set_64bit(ptr, value) \
|
|
(__builtin_constant_p((value)) \
|
|
? __set_64bit_constant((ptr), (value)) \
|
|
: __set_64bit_var((ptr), (value)))
|
|
|
|
#define _set_64bit(ptr, value) \
|
|
(__builtin_constant_p(value) \
|
|
? __set_64bit(ptr, (unsigned int)(value), \
|
|
(unsigned int)((value) >> 32)) \
|
|
: __set_64bit(ptr, ll_low((value)), ll_high((value))))
|
|
|
|
extern void __cmpxchg_wrong_size(void);
|
|
|
|
/*
|
|
* Atomic compare and exchange. Compare OLD with MEM, if identical,
|
|
* store NEW in MEM. Return the initial value in MEM. Success is
|
|
* indicated by comparing RETURN with OLD.
|
|
*/
|
|
#define __raw_cmpxchg(ptr, old, new, size, lock) \
|
|
({ \
|
|
__typeof__(*(ptr)) __ret; \
|
|
__typeof__(*(ptr)) __old = (old); \
|
|
__typeof__(*(ptr)) __new = (new); \
|
|
switch (size) { \
|
|
case 1: \
|
|
asm volatile(lock "cmpxchgb %b2,%1" \
|
|
: "=a" (__ret), "+m" (*__xg(ptr)) \
|
|
: "q" (__new), "0" (__old) \
|
|
: "memory"); \
|
|
break; \
|
|
case 2: \
|
|
asm volatile(lock "cmpxchgw %w2,%1" \
|
|
: "=a" (__ret), "+m" (*__xg(ptr)) \
|
|
: "r" (__new), "0" (__old) \
|
|
: "memory"); \
|
|
break; \
|
|
case 4: \
|
|
asm volatile(lock "cmpxchgl %2,%1" \
|
|
: "=a" (__ret), "+m" (*__xg(ptr)) \
|
|
: "r" (__new), "0" (__old) \
|
|
: "memory"); \
|
|
break; \
|
|
default: \
|
|
__cmpxchg_wrong_size(); \
|
|
} \
|
|
__ret; \
|
|
})
|
|
|
|
#define __cmpxchg(ptr, old, new, size) \
|
|
__raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX)
|
|
|
|
#define __sync_cmpxchg(ptr, old, new, size) \
|
|
__raw_cmpxchg((ptr), (old), (new), (size), "lock; ")
|
|
|
|
#define __cmpxchg_local(ptr, old, new, size) \
|
|
__raw_cmpxchg((ptr), (old), (new), (size), "")
|
|
|
|
#ifdef CONFIG_X86_CMPXCHG
|
|
#define __HAVE_ARCH_CMPXCHG 1
|
|
|
|
#define cmpxchg(ptr, old, new) \
|
|
__cmpxchg((ptr), (old), (new), sizeof(*ptr))
|
|
|
|
#define sync_cmpxchg(ptr, old, new) \
|
|
__sync_cmpxchg((ptr), (old), (new), sizeof(*ptr))
|
|
|
|
#define cmpxchg_local(ptr, old, new) \
|
|
__cmpxchg_local((ptr), (old), (new), sizeof(*ptr))
|
|
#endif
|
|
|
|
#ifdef CONFIG_X86_CMPXCHG64
|
|
#define cmpxchg64(ptr, o, n) \
|
|
((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \
|
|
(unsigned long long)(n)))
|
|
#define cmpxchg64_local(ptr, o, n) \
|
|
((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \
|
|
(unsigned long long)(n)))
|
|
#endif
|
|
|
|
static inline unsigned long long __cmpxchg64(volatile void *ptr,
|
|
unsigned long long old,
|
|
unsigned long long new)
|
|
{
|
|
unsigned long long prev;
|
|
asm volatile(LOCK_PREFIX "cmpxchg8b %1"
|
|
: "=A" (prev),
|
|
"+m" (*__xg(ptr))
|
|
: "b" ((unsigned long)new),
|
|
"c" ((unsigned long)(new >> 32)),
|
|
"0" (old)
|
|
: "memory");
|
|
return prev;
|
|
}
|
|
|
|
static inline unsigned long long __cmpxchg64_local(volatile void *ptr,
|
|
unsigned long long old,
|
|
unsigned long long new)
|
|
{
|
|
unsigned long long prev;
|
|
asm volatile("cmpxchg8b %1"
|
|
: "=A" (prev),
|
|
"+m" (*__xg(ptr))
|
|
: "b" ((unsigned long)new),
|
|
"c" ((unsigned long)(new >> 32)),
|
|
"0" (old)
|
|
: "memory");
|
|
return prev;
|
|
}
|
|
|
|
#ifndef CONFIG_X86_CMPXCHG
|
|
/*
|
|
* Building a kernel capable running on 80386. It may be necessary to
|
|
* simulate the cmpxchg on the 80386 CPU. For that purpose we define
|
|
* a function for each of the sizes we support.
|
|
*/
|
|
|
|
extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8);
|
|
extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16);
|
|
extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32);
|
|
|
|
static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old,
|
|
unsigned long new, int size)
|
|
{
|
|
switch (size) {
|
|
case 1:
|
|
return cmpxchg_386_u8(ptr, old, new);
|
|
case 2:
|
|
return cmpxchg_386_u16(ptr, old, new);
|
|
case 4:
|
|
return cmpxchg_386_u32(ptr, old, new);
|
|
}
|
|
return old;
|
|
}
|
|
|
|
#define cmpxchg(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) __ret; \
|
|
if (likely(boot_cpu_data.x86 > 3)) \
|
|
__ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \
|
|
(unsigned long)(o), (unsigned long)(n), \
|
|
sizeof(*(ptr))); \
|
|
else \
|
|
__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
|
|
(unsigned long)(o), (unsigned long)(n), \
|
|
sizeof(*(ptr))); \
|
|
__ret; \
|
|
})
|
|
#define cmpxchg_local(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) __ret; \
|
|
if (likely(boot_cpu_data.x86 > 3)) \
|
|
__ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \
|
|
(unsigned long)(o), (unsigned long)(n), \
|
|
sizeof(*(ptr))); \
|
|
else \
|
|
__ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \
|
|
(unsigned long)(o), (unsigned long)(n), \
|
|
sizeof(*(ptr))); \
|
|
__ret; \
|
|
})
|
|
#endif
|
|
|
|
#ifndef CONFIG_X86_CMPXCHG64
|
|
/*
|
|
* Building a kernel capable running on 80386 and 80486. It may be necessary
|
|
* to simulate the cmpxchg8b on the 80386 and 80486 CPU.
|
|
*/
|
|
|
|
extern unsigned long long cmpxchg_486_u64(volatile void *, u64, u64);
|
|
|
|
#define cmpxchg64(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) __ret; \
|
|
__typeof__(*(ptr)) __old = (o); \
|
|
__typeof__(*(ptr)) __new = (n); \
|
|
alternative_io(LOCK_PREFIX_HERE \
|
|
"call cmpxchg8b_emu", \
|
|
"lock; cmpxchg8b (%%esi)" , \
|
|
X86_FEATURE_CX8, \
|
|
"=A" (__ret), \
|
|
"S" ((ptr)), "0" (__old), \
|
|
"b" ((unsigned int)__new), \
|
|
"c" ((unsigned int)(__new>>32)) \
|
|
: "memory"); \
|
|
__ret; })
|
|
|
|
|
|
|
|
#define cmpxchg64_local(ptr, o, n) \
|
|
({ \
|
|
__typeof__(*(ptr)) __ret; \
|
|
if (likely(boot_cpu_data.x86 > 4)) \
|
|
__ret = (__typeof__(*(ptr)))__cmpxchg64_local((ptr), \
|
|
(unsigned long long)(o), \
|
|
(unsigned long long)(n)); \
|
|
else \
|
|
__ret = (__typeof__(*(ptr)))cmpxchg_486_u64((ptr), \
|
|
(unsigned long long)(o), \
|
|
(unsigned long long)(n)); \
|
|
__ret; \
|
|
})
|
|
|
|
#endif
|
|
|
|
#endif /* _ASM_X86_CMPXCHG_32_H */
|