9a0b5817ad
Implement SMP alternatives, i.e. switching at runtime between different
code versions for UP and SMP. The code can patch both SMP->UP and UP->SMP.
The UP->SMP case is useful for CPU hotplug.
With CONFIG_CPU_HOTPLUG enabled the code switches to UP at boot time and
when the number of CPUs goes down to 1, and switches to SMP when the number
of CPUs goes up to 2.
Without CONFIG_CPU_HOTPLUG or on non-SMP-capable systems the code is
patched once at boot time (if needed) and the tables are released
afterwards.
The changes in detail:
* The current alternatives bits are moved to a separate file,
the SMP alternatives code is added there.
* The patch adds some new elf sections to the kernel:
.smp_altinstructions
like .altinstructions, also contains a list
of alt_instr structs.
.smp_altinstr_replacement
like .altinstr_replacement, but also has some space to
save original instruction before replaving it.
.smp_locks
list of pointers to lock prefixes which can be nop'ed
out on UP.
The first two are used to replace more complex instruction
sequences such as spinlocks and semaphores. It would be possible
to deal with the lock prefixes with that as well, but by handling
them as special case the table sizes become much smaller.
* The sections are page-aligned and padded up to page size, so they
can be free if they are not needed.
* Splitted the code to release init pages to a separate function and
use it to release the elf sections if they are unused.
Signed-off-by: Gerd Hoffmann <kraxel@suse.de>
Signed-off-by: Chuck Ebbert <76306.1226@compuserve.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
138 lines
4.1 KiB
C
138 lines
4.1 KiB
C
/*
|
|
* Assembly implementation of the mutex fastpath, based on atomic
|
|
* decrement/increment.
|
|
*
|
|
* started by Ingo Molnar:
|
|
*
|
|
* Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
|
|
*/
|
|
#ifndef _ASM_MUTEX_H
|
|
#define _ASM_MUTEX_H
|
|
|
|
#include "asm/alternative.h"
|
|
|
|
/**
|
|
* __mutex_fastpath_lock - try to take the lock by moving the count
|
|
* from 1 to a 0 value
|
|
* @count: pointer of type atomic_t
|
|
* @fn: function to call if the original value was not 1
|
|
*
|
|
* Change the count from 1 to a value lower than 1, and call <fn> if it
|
|
* wasn't 1 originally. This function MUST leave the value lower than 1
|
|
* even when the "1" assertion wasn't true.
|
|
*/
|
|
#define __mutex_fastpath_lock(count, fail_fn) \
|
|
do { \
|
|
unsigned int dummy; \
|
|
\
|
|
typecheck(atomic_t *, count); \
|
|
typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \
|
|
\
|
|
__asm__ __volatile__( \
|
|
LOCK_PREFIX " decl (%%eax) \n" \
|
|
" js 2f \n" \
|
|
"1: \n" \
|
|
\
|
|
LOCK_SECTION_START("") \
|
|
"2: call "#fail_fn" \n" \
|
|
" jmp 1b \n" \
|
|
LOCK_SECTION_END \
|
|
\
|
|
:"=a" (dummy) \
|
|
: "a" (count) \
|
|
: "memory", "ecx", "edx"); \
|
|
} while (0)
|
|
|
|
|
|
/**
|
|
* __mutex_fastpath_lock_retval - try to take the lock by moving the count
|
|
* from 1 to a 0 value
|
|
* @count: pointer of type atomic_t
|
|
* @fail_fn: function to call if the original value was not 1
|
|
*
|
|
* Change the count from 1 to a value lower than 1, and call <fail_fn> if it
|
|
* wasn't 1 originally. This function returns 0 if the fastpath succeeds,
|
|
* or anything the slow path function returns
|
|
*/
|
|
static inline int
|
|
__mutex_fastpath_lock_retval(atomic_t *count,
|
|
int fastcall (*fail_fn)(atomic_t *))
|
|
{
|
|
if (unlikely(atomic_dec_return(count) < 0))
|
|
return fail_fn(count);
|
|
else
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* __mutex_fastpath_unlock - try to promote the mutex from 0 to 1
|
|
* @count: pointer of type atomic_t
|
|
* @fail_fn: function to call if the original value was not 0
|
|
*
|
|
* try to promote the mutex from 0 to 1. if it wasn't 0, call <fail_fn>.
|
|
* In the failure case, this function is allowed to either set the value
|
|
* to 1, or to set it to a value lower than 1.
|
|
*
|
|
* If the implementation sets it to a value of lower than 1, the
|
|
* __mutex_slowpath_needs_to_unlock() macro needs to return 1, it needs
|
|
* to return 0 otherwise.
|
|
*/
|
|
#define __mutex_fastpath_unlock(count, fail_fn) \
|
|
do { \
|
|
unsigned int dummy; \
|
|
\
|
|
typecheck(atomic_t *, count); \
|
|
typecheck_fn(fastcall void (*)(atomic_t *), fail_fn); \
|
|
\
|
|
__asm__ __volatile__( \
|
|
LOCK_PREFIX " incl (%%eax) \n" \
|
|
" jle 2f \n" \
|
|
"1: \n" \
|
|
\
|
|
LOCK_SECTION_START("") \
|
|
"2: call "#fail_fn" \n" \
|
|
" jmp 1b \n" \
|
|
LOCK_SECTION_END \
|
|
\
|
|
:"=a" (dummy) \
|
|
: "a" (count) \
|
|
: "memory", "ecx", "edx"); \
|
|
} while (0)
|
|
|
|
#define __mutex_slowpath_needs_to_unlock() 1
|
|
|
|
/**
|
|
* __mutex_fastpath_trylock - try to acquire the mutex, without waiting
|
|
*
|
|
* @count: pointer of type atomic_t
|
|
* @fail_fn: fallback function
|
|
*
|
|
* Change the count from 1 to a value lower than 1, and return 0 (failure)
|
|
* if it wasn't 1 originally, or return 1 (success) otherwise. This function
|
|
* MUST leave the value lower than 1 even when the "1" assertion wasn't true.
|
|
* Additionally, if the value was < 0 originally, this function must not leave
|
|
* it to 0 on failure.
|
|
*/
|
|
static inline int
|
|
__mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
|
|
{
|
|
/*
|
|
* We have two variants here. The cmpxchg based one is the best one
|
|
* because it never induce a false contention state. It is included
|
|
* here because architectures using the inc/dec algorithms over the
|
|
* xchg ones are much more likely to support cmpxchg natively.
|
|
*
|
|
* If not we fall back to the spinlock based variant - that is
|
|
* just as efficient (and simpler) as a 'destructive' probing of
|
|
* the mutex state would be.
|
|
*/
|
|
#ifdef __HAVE_ARCH_CMPXCHG
|
|
if (likely(atomic_cmpxchg(count, 1, 0) == 1))
|
|
return 1;
|
|
return 0;
|
|
#else
|
|
return fail_fn(count);
|
|
#endif
|
|
}
|
|
|
|
#endif
|