23ca4bba3e
* Introduce a new PER_CPU macro called "EARLY_PER_CPU". This is
used by some per_cpu variables that are initialized and accessed
before there are per_cpu areas allocated.
["Early" in respect to per_cpu variables is "earlier than the per_cpu
areas have been setup".]
This patchset adds these new macros:
DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)
EXPORT_EARLY_PER_CPU_SYMBOL(_name)
DECLARE_EARLY_PER_CPU(_type, _name)
early_per_cpu_ptr(_name)
early_per_cpu_map(_name, _idx)
early_per_cpu(_name, _cpu)
The DEFINE macro defines the per_cpu variable as well as the early
map and pointer. It also initializes the per_cpu variable and map
elements to "_initvalue". The early_* macros provide access to
the initial map (usually setup during system init) and the early
pointer. This pointer is initialized to point to the early map
but is then NULL'ed when the actual per_cpu areas are setup. After
that the per_cpu variable is the correct access to the variable.
The early_per_cpu() macro is not very efficient but does show how to
access the variable if you have a function that can be called both
"early" and "late". It tests the early ptr to be NULL, and if not
then it's still valid. Otherwise, the per_cpu variable is used
instead:
#define early_per_cpu(_name, _cpu) \
(early_per_cpu_ptr(_name) ? \
early_per_cpu_ptr(_name)[_cpu] : \
per_cpu(_name, _cpu))
A better method is to actually check the pointer manually. In the
case below, numa_set_node can be called both "early" and "late":
void __cpuinit numa_set_node(int cpu, int node)
{
int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map);
if (cpu_to_node_map)
cpu_to_node_map[cpu] = node;
else
per_cpu(x86_cpu_to_node_map, cpu) = node;
}
* Add a flag "arch_provides_topology_pointers" that indicates pointers
to topology cpumask_t maps are available. Otherwise, use the function
returning the cpumask_t value. This is useful if cpumask_t set size
is very large to avoid copying data on to/off of the stack.
* The coverage of CONFIG_DEBUG_PER_CPU_MAPS has been increased while
the non-debug case has been optimized a bit.
* Remove an unreferenced compiler warning in drivers/base/topology.c
* Clean up #ifdef in setup.c
For inclusion into sched-devel/latest tree.
Based on:
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git
+ sched-devel/latest .../mingo/linux-2.6-sched-devel.git
Signed-off-by: Mike Travis <travis@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
192 lines
4.8 KiB
C
192 lines
4.8 KiB
C
#ifndef _ASM_X86_PERCPU_H_
|
|
#define _ASM_X86_PERCPU_H_
|
|
|
|
#ifdef CONFIG_X86_64
|
|
#include <linux/compiler.h>
|
|
|
|
/* Same as asm-generic/percpu.h, except that we store the per cpu offset
|
|
in the PDA. Longer term the PDA and every per cpu variable
|
|
should be just put into a single section and referenced directly
|
|
from %gs */
|
|
|
|
#ifdef CONFIG_SMP
|
|
#include <asm/pda.h>
|
|
|
|
#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
|
|
#define __my_cpu_offset read_pda(data_offset)
|
|
|
|
#define per_cpu_offset(x) (__per_cpu_offset(x))
|
|
|
|
#endif
|
|
#include <asm-generic/percpu.h>
|
|
|
|
DECLARE_PER_CPU(struct x8664_pda, pda);
|
|
|
|
#else /* CONFIG_X86_64 */
|
|
|
|
#ifdef __ASSEMBLY__
|
|
|
|
/*
|
|
* PER_CPU finds an address of a per-cpu variable.
|
|
*
|
|
* Args:
|
|
* var - variable name
|
|
* reg - 32bit register
|
|
*
|
|
* The resulting address is stored in the "reg" argument.
|
|
*
|
|
* Example:
|
|
* PER_CPU(cpu_gdt_descr, %ebx)
|
|
*/
|
|
#ifdef CONFIG_SMP
|
|
#define PER_CPU(var, reg) \
|
|
movl %fs:per_cpu__##this_cpu_off, reg; \
|
|
lea per_cpu__##var(reg), reg
|
|
#define PER_CPU_VAR(var) %fs:per_cpu__##var
|
|
#else /* ! SMP */
|
|
#define PER_CPU(var, reg) \
|
|
movl $per_cpu__##var, reg
|
|
#define PER_CPU_VAR(var) per_cpu__##var
|
|
#endif /* SMP */
|
|
|
|
#else /* ...!ASSEMBLY */
|
|
|
|
/*
|
|
* PER_CPU finds an address of a per-cpu variable.
|
|
*
|
|
* Args:
|
|
* var - variable name
|
|
* cpu - 32bit register containing the current CPU number
|
|
*
|
|
* The resulting address is stored in the "cpu" argument.
|
|
*
|
|
* Example:
|
|
* PER_CPU(cpu_gdt_descr, %ebx)
|
|
*/
|
|
#ifdef CONFIG_SMP
|
|
|
|
#define __my_cpu_offset x86_read_percpu(this_cpu_off)
|
|
|
|
/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
|
|
#define __percpu_seg "%%fs:"
|
|
|
|
#else /* !SMP */
|
|
|
|
#define __percpu_seg ""
|
|
|
|
#endif /* SMP */
|
|
|
|
#include <asm-generic/percpu.h>
|
|
|
|
/* We can use this directly for local CPU (faster). */
|
|
DECLARE_PER_CPU(unsigned long, this_cpu_off);
|
|
|
|
/* For arch-specific code, we can use direct single-insn ops (they
|
|
* don't give an lvalue though). */
|
|
extern void __bad_percpu_size(void);
|
|
|
|
#define percpu_to_op(op, var, val) \
|
|
do { \
|
|
typedef typeof(var) T__; \
|
|
if (0) { \
|
|
T__ tmp__; \
|
|
tmp__ = (val); \
|
|
} \
|
|
switch (sizeof(var)) { \
|
|
case 1: \
|
|
asm(op "b %1,"__percpu_seg"%0" \
|
|
: "+m" (var) \
|
|
: "ri" ((T__)val)); \
|
|
break; \
|
|
case 2: \
|
|
asm(op "w %1,"__percpu_seg"%0" \
|
|
: "+m" (var) \
|
|
: "ri" ((T__)val)); \
|
|
break; \
|
|
case 4: \
|
|
asm(op "l %1,"__percpu_seg"%0" \
|
|
: "+m" (var) \
|
|
: "ri" ((T__)val)); \
|
|
break; \
|
|
default: __bad_percpu_size(); \
|
|
} \
|
|
} while (0)
|
|
|
|
#define percpu_from_op(op, var) \
|
|
({ \
|
|
typeof(var) ret__; \
|
|
switch (sizeof(var)) { \
|
|
case 1: \
|
|
asm(op "b "__percpu_seg"%1,%0" \
|
|
: "=r" (ret__) \
|
|
: "m" (var)); \
|
|
break; \
|
|
case 2: \
|
|
asm(op "w "__percpu_seg"%1,%0" \
|
|
: "=r" (ret__) \
|
|
: "m" (var)); \
|
|
break; \
|
|
case 4: \
|
|
asm(op "l "__percpu_seg"%1,%0" \
|
|
: "=r" (ret__) \
|
|
: "m" (var)); \
|
|
break; \
|
|
default: __bad_percpu_size(); \
|
|
} \
|
|
ret__; \
|
|
})
|
|
|
|
#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
|
|
#define x86_write_percpu(var, val) percpu_to_op("mov", per_cpu__##var, val)
|
|
#define x86_add_percpu(var, val) percpu_to_op("add", per_cpu__##var, val)
|
|
#define x86_sub_percpu(var, val) percpu_to_op("sub", per_cpu__##var, val)
|
|
#define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val)
|
|
#endif /* !__ASSEMBLY__ */
|
|
#endif /* !CONFIG_X86_64 */
|
|
|
|
#ifdef CONFIG_SMP
|
|
|
|
/*
|
|
* Define the "EARLY_PER_CPU" macros. These are used for some per_cpu
|
|
* variables that are initialized and accessed before there are per_cpu
|
|
* areas allocated.
|
|
*/
|
|
|
|
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
|
|
DEFINE_PER_CPU(_type, _name) = _initvalue; \
|
|
__typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
|
|
{ [0 ... NR_CPUS-1] = _initvalue }; \
|
|
__typeof__(_type) *_name##_early_ptr = _name##_early_map
|
|
|
|
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
|
|
EXPORT_PER_CPU_SYMBOL(_name)
|
|
|
|
#define DECLARE_EARLY_PER_CPU(_type, _name) \
|
|
DECLARE_PER_CPU(_type, _name); \
|
|
extern __typeof__(_type) *_name##_early_ptr; \
|
|
extern __typeof__(_type) _name##_early_map[]
|
|
|
|
#define early_per_cpu_ptr(_name) (_name##_early_ptr)
|
|
#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
|
|
#define early_per_cpu(_name, _cpu) \
|
|
(early_per_cpu_ptr(_name) ? \
|
|
early_per_cpu_ptr(_name)[_cpu] : \
|
|
per_cpu(_name, _cpu))
|
|
|
|
#else /* !CONFIG_SMP */
|
|
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
|
|
DEFINE_PER_CPU(_type, _name) = _initvalue
|
|
|
|
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
|
|
EXPORT_PER_CPU_SYMBOL(_name)
|
|
|
|
#define DECLARE_EARLY_PER_CPU(_type, _name) \
|
|
DECLARE_PER_CPU(_type, _name)
|
|
|
|
#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
|
|
#define early_per_cpu_ptr(_name) NULL
|
|
/* no early_per_cpu_map() */
|
|
|
|
#endif /* !CONFIG_SMP */
|
|
|
|
#endif /* _ASM_X86_PERCPU_H_ */
|