Techwizz/kernel-fxtec-pro1x
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu into for-3.17/core

Browse source 
Merge the percpu_ref changes from Tejun, he says they are stable now.
This commit is contained in:
Jens Axboe 2014-07-01 10:19:04 -06:00
commit 17737d3b59
11 changed files with 838 additions and 826 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
arch/x86/include/asm/percpu.h
View file

@ -52,10 +52,9 @@
* Compared to the generic __my_cpu_offset version, the following * Compared to the generic __my_cpu_offset version, the following
* saves one instruction and avoids clobbering a temp register. * saves one instruction and avoids clobbering a temp register.
*/ */
#define raw_cpu_ptr(ptr) \ #define arch_raw_cpu_ptr(ptr) \
({ \ ({ \
unsigned long tcp_ptr__; \ unsigned long tcp_ptr__; \
__verify_pcpu_ptr(ptr); \
asm volatile("add " __percpu_arg(1) ", %0" \ asm volatile("add " __percpu_arg(1) ", %0" \
: "=r" (tcp_ptr__) \ : "=r" (tcp_ptr__) \
: "m" (this_cpu_off), "0" (ptr)); \ : "m" (this_cpu_off), "0" (ptr)); \

4
drivers/target/target_core_tpg.c
View file

@ -825,7 +825,7 @@ int core_tpg_add_lun(
ret = core_dev_export(dev, tpg, lun); ret = core_dev_export(dev, tpg, lun);
if (ret < 0) { if (ret < 0) {
percpu_ref_cancel_init(&lun->lun_ref); percpu_ref_exit(&lun->lun_ref);
return ret; return ret;
} }
@ -880,5 +880,7 @@ int core_tpg_post_dellun(
lun->lun_status = TRANSPORT_LUN_STATUS_FREE; lun->lun_status = TRANSPORT_LUN_STATUS_FREE;
spin_unlock(&tpg->tpg_lun_lock); spin_unlock(&tpg->tpg_lun_lock);
percpu_ref_exit(&lun->lun_ref);
return 0; return 0;
} }

6
fs/aio.c
View file

@ -506,6 +506,8 @@ static void free_ioctx(struct work_struct *work)
aio_free_ring(ctx); aio_free_ring(ctx);
free_percpu(ctx->cpu); free_percpu(ctx->cpu);
percpu_ref_exit(&ctx->reqs);
percpu_ref_exit(&ctx->users);
kmem_cache_free(kioctx_cachep, ctx); kmem_cache_free(kioctx_cachep, ctx);
} }
@ -715,8 +717,8 @@ static struct kioctx *ioctx_alloc(unsigned nr_events)
err: err:
mutex_unlock(&ctx->ring_lock); mutex_unlock(&ctx->ring_lock);
free_percpu(ctx->cpu); free_percpu(ctx->cpu);
free_percpu(ctx->reqs.pcpu_count); percpu_ref_exit(&ctx->reqs);
free_percpu(ctx->users.pcpu_count); percpu_ref_exit(&ctx->users);
kmem_cache_free(kioctx_cachep, ctx); kmem_cache_free(kioctx_cachep, ctx);
pr_debug("error allocating ioctx %d\n", err); pr_debug("error allocating ioctx %d\n", err);
return ERR_PTR(err); return ERR_PTR(err);

418
include/asm-generic/percpu.h
View file

@ -36,55 +36,17 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
#endif #endif
/* /*
* Add a offset to a pointer but keep the pointer as is. * Arch may define arch_raw_cpu_ptr() to provide more efficient address
* * translations for raw_cpu_ptr().
* Only S390 provides its own means of moving the pointer.
*/ */
#ifndef SHIFT_PERCPU_PTR #ifndef arch_raw_cpu_ptr
/* Weird cast keeps both GCC and sparse happy. */ #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
#define SHIFT_PERCPU_PTR(__p, __offset) ({ \
__verify_pcpu_ptr((__p)); \
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \
})
#endif #endif
/*
* A percpu variable may point to a discarded regions. The following are
* established ways to produce a usable pointer from the percpu variable
* offset.
*/
#define per_cpu(var, cpu) \
(*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
#ifndef raw_cpu_ptr
#define raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
#endif
#ifdef CONFIG_DEBUG_PREEMPT
#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
#else
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
#endif
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
extern void setup_per_cpu_areas(void); extern void setup_per_cpu_areas(void);
#endif #endif
#else /* ! SMP */
#define VERIFY_PERCPU_PTR(__p) ({ \
__verify_pcpu_ptr((__p)); \
(typeof(*(__p)) __kernel __force *)(__p); \
})
#define per_cpu(var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var))))
#define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var)))
#define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var)))
#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
#define raw_cpu_ptr(ptr) this_cpu_ptr(ptr)
#endif /* SMP */ #endif /* SMP */
#ifndef PER_CPU_BASE_SECTION #ifndef PER_CPU_BASE_SECTION
@ -95,25 +57,6 @@ extern void setup_per_cpu_areas(void);
#endif #endif
#endif #endif
#ifdef CONFIG_SMP
#ifdef MODULE
#define PER_CPU_SHARED_ALIGNED_SECTION ""
#define PER_CPU_ALIGNED_SECTION ""
#else
#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
#endif
#define PER_CPU_FIRST_SECTION "..first"
#else
#define PER_CPU_SHARED_ALIGNED_SECTION ""
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
#define PER_CPU_FIRST_SECTION ""
#endif
#ifndef PER_CPU_ATTRIBUTES #ifndef PER_CPU_ATTRIBUTES
#define PER_CPU_ATTRIBUTES #define PER_CPU_ATTRIBUTES
#endif #endif
@ -122,7 +65,356 @@ extern void setup_per_cpu_areas(void);
#define PER_CPU_DEF_ATTRIBUTES #define PER_CPU_DEF_ATTRIBUTES
#endif #endif
/* Keep until we have removed all uses of __this_cpu_ptr */ #define raw_cpu_generic_to_op(pcp, val, op) \
#define __this_cpu_ptr raw_cpu_ptr do { \
*raw_cpu_ptr(&(pcp)) op val; \
} while (0)
#define raw_cpu_generic_add_return(pcp, val) \
({ \
raw_cpu_add(pcp, val); \
raw_cpu_read(pcp); \
})
#define raw_cpu_generic_xchg(pcp, nval) \
({ \
typeof(pcp) __ret; \
__ret = raw_cpu_read(pcp); \
raw_cpu_write(pcp, nval); \
__ret; \
})
#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
({ \
typeof(pcp) __ret; \
__ret = raw_cpu_read(pcp); \
if (__ret == (oval)) \
raw_cpu_write(pcp, nval); \
__ret; \
})
#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({ \
int __ret = 0; \
if (raw_cpu_read(pcp1) == (oval1) && \
raw_cpu_read(pcp2) == (oval2)) { \
raw_cpu_write(pcp1, nval1); \
raw_cpu_write(pcp2, nval2); \
__ret = 1; \
} \
(__ret); \
})
#define this_cpu_generic_read(pcp) \
({ \
typeof(pcp) __ret; \
preempt_disable(); \
__ret = *this_cpu_ptr(&(pcp)); \
preempt_enable(); \
__ret; \
})
#define this_cpu_generic_to_op(pcp, val, op) \
do { \
unsigned long __flags; \
raw_local_irq_save(__flags); \
*raw_cpu_ptr(&(pcp)) op val; \
raw_local_irq_restore(__flags); \
} while (0)
#define this_cpu_generic_add_return(pcp, val) \
({ \
typeof(pcp) __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
raw_cpu_add(pcp, val); \
__ret = raw_cpu_read(pcp); \
raw_local_irq_restore(__flags); \
__ret; \
})
#define this_cpu_generic_xchg(pcp, nval) \
({ \
typeof(pcp) __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__ret = raw_cpu_read(pcp); \
raw_cpu_write(pcp, nval); \
raw_local_irq_restore(__flags); \
__ret; \
})
#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
({ \
typeof(pcp) __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__ret = raw_cpu_read(pcp); \
if (__ret == (oval)) \
raw_cpu_write(pcp, nval); \
raw_local_irq_restore(__flags); \
__ret; \
})
#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({ \
int __ret; \
unsigned long __flags; \
raw_local_irq_save(__flags); \
__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
oval1, oval2, nval1, nval2); \
raw_local_irq_restore(__flags); \
__ret; \
})
#ifndef raw_cpu_read_1
#define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
#endif
#ifndef raw_cpu_read_2
#define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
#endif
#ifndef raw_cpu_read_4
#define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
#endif
#ifndef raw_cpu_read_8
#define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
#endif
#ifndef raw_cpu_write_1
#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_write_2
#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_write_4
#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_write_8
#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef raw_cpu_add_1
#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_add_2
#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_add_4
#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_add_8
#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef raw_cpu_and_1
#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_and_2
#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_and_4
#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_and_8
#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef raw_cpu_or_1
#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_or_2
#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_or_4
#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_or_8
#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef raw_cpu_add_return_1
#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_add_return_2
#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_add_return_4
#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_add_return_8
#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
#endif
#ifndef raw_cpu_xchg_1
#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_xchg_2
#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_xchg_4
#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_xchg_8
#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
#endif
#ifndef raw_cpu_cmpxchg_1
#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_2
#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_4
#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_8
#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
raw_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef raw_cpu_cmpxchg_double_1
#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef raw_cpu_cmpxchg_double_2
#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef raw_cpu_cmpxchg_double_4
#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef raw_cpu_cmpxchg_double_8
#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_read_1
#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_read_2
#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_read_4
#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_read_8
#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
#endif
#ifndef this_cpu_write_1
#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_write_2
#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_write_4
#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_write_8
#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
#endif
#ifndef this_cpu_add_1
#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_add_2
#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_add_4
#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_add_8
#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
#endif
#ifndef this_cpu_and_1
#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_and_2
#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_and_4
#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_and_8
#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
#endif
#ifndef this_cpu_or_1
#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_or_2
#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_or_4
#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_or_8
#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
#endif
#ifndef this_cpu_add_return_1
#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_add_return_2
#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_add_return_4
#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_add_return_8
#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
#endif
#ifndef this_cpu_xchg_1
#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_xchg_2
#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_xchg_4
#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_xchg_8
#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
#endif
#ifndef this_cpu_cmpxchg_1
#define this_cpu_cmpxchg_1(pcp, oval, nval) \
this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_2
#define this_cpu_cmpxchg_2(pcp, oval, nval) \
this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_4
#define this_cpu_cmpxchg_4(pcp, oval, nval) \
this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_8
#define this_cpu_cmpxchg_8(pcp, oval, nval) \
this_cpu_generic_cmpxchg(pcp, oval, nval)
#endif
#ifndef this_cpu_cmpxchg_double_1
#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_cmpxchg_double_2
#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_cmpxchg_double_4
#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#ifndef this_cpu_cmpxchg_double_8
#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
#endif
#endif /* _ASM_GENERIC_PERCPU_H_ */ #endif /* _ASM_GENERIC_PERCPU_H_ */

380
include/linux/percpu-defs.h
View file

@ -1,6 +1,40 @@
/*
* linux/percpu-defs.h - basic definitions for percpu areas
*
* DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
*
* This file is separate from linux/percpu.h to avoid cyclic inclusion
* dependency from arch header files. Only to be included from
* asm/percpu.h.
*
* This file includes macros necessary to declare percpu sections and
* variables, and definitions of percpu accessors and operations. It
* should provide enough percpu features to arch header files even when
* they can only include asm/percpu.h to avoid cyclic inclusion dependency.
*/
#ifndef _LINUX_PERCPU_DEFS_H #ifndef _LINUX_PERCPU_DEFS_H
#define _LINUX_PERCPU_DEFS_H #define _LINUX_PERCPU_DEFS_H
#ifdef CONFIG_SMP
#ifdef MODULE
#define PER_CPU_SHARED_ALIGNED_SECTION ""
#define PER_CPU_ALIGNED_SECTION ""
#else
#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
#endif
#define PER_CPU_FIRST_SECTION "..first"
#else
#define PER_CPU_SHARED_ALIGNED_SECTION ""
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
#define PER_CPU_FIRST_SECTION ""
#endif
/* /*
* Base implementations of per-CPU variable declarations and definitions, where * Base implementations of per-CPU variable declarations and definitions, where
* the section in which the variable is to be placed is provided by the * the section in which the variable is to be placed is provided by the
@ -18,19 +52,6 @@
#define __PCPU_DUMMY_ATTRS \ #define __PCPU_DUMMY_ATTRS \
__attribute__((section(".discard"), unused)) __attribute__((section(".discard"), unused))
/*
* Macro which verifies @ptr is a percpu pointer without evaluating
* @ptr. This is to be used in percpu accessors to verify that the
* input parameter is a percpu pointer.
*
* + 0 is required in order to convert the pointer type from a
* potential array type to a pointer to a single item of the array.
*/
#define __verify_pcpu_ptr(ptr) do { \
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
(void)__vpp_verify; \
} while (0)
/* /*
* s390 and alpha modules require percpu variables to be defined as * s390 and alpha modules require percpu variables to be defined as
* weak to force the compiler to generate GOT based external * weak to force the compiler to generate GOT based external
@ -164,4 +185,337 @@
#define EXPORT_PER_CPU_SYMBOL_GPL(var) #define EXPORT_PER_CPU_SYMBOL_GPL(var)
#endif #endif
/*
* Accessors and operations.
*/
#ifndef __ASSEMBLY__
/*
* __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
* @ptr and is invoked once before a percpu area is accessed by all
* accessors and operations. This is performed in the generic part of
* percpu and arch overrides don't need to worry about it; however, if an
* arch wants to implement an arch-specific percpu accessor or operation,
* it may use __verify_pcpu_ptr() to verify the parameters.
*
* + 0 is required in order to convert the pointer type from a
* potential array type to a pointer to a single item of the array.
*/
#define __verify_pcpu_ptr(ptr) \
do { \
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
(void)__vpp_verify; \
} while (0)
#ifdef CONFIG_SMP
/*
* Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE()
* to prevent the compiler from making incorrect assumptions about the
* pointer value. The weird cast keeps both GCC and sparse happy.
*/
#define SHIFT_PERCPU_PTR(__p, __offset) \
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
#define per_cpu_ptr(ptr, cpu) \
({ \
__verify_pcpu_ptr(ptr); \
SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
})
#define raw_cpu_ptr(ptr) \
({ \
__verify_pcpu_ptr(ptr); \
arch_raw_cpu_ptr(ptr); \
})
#ifdef CONFIG_DEBUG_PREEMPT
#define this_cpu_ptr(ptr) \
({ \
__verify_pcpu_ptr(ptr); \
SHIFT_PERCPU_PTR(ptr, my_cpu_offset); \
})
#else
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
#endif
#else /* CONFIG_SMP */
#define VERIFY_PERCPU_PTR(__p) \
({ \
__verify_pcpu_ptr(__p); \
(typeof(*(__p)) __kernel __force *)(__p); \
})
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
#endif /* CONFIG_SMP */
#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
/* keep until we have removed all uses of __this_cpu_ptr */
#define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
/*
* Must be an lvalue. Since @var must be a simple identifier,
* we force a syntax error here if it isn't.
*/
#define get_cpu_var(var) \
(*({ \
preempt_disable(); \
this_cpu_ptr(&var); \
}))
/*
* The weird & is necessary because sparse considers (void)(var) to be
* a direct dereference of percpu variable (var).
*/
#define put_cpu_var(var) \
do { \
(void)&(var); \
preempt_enable(); \
} while (0)
#define get_cpu_ptr(var) \
({ \
preempt_disable(); \
this_cpu_ptr(var); \
})
#define put_cpu_ptr(var) \
do { \
(void)(var); \
preempt_enable(); \
} while (0)
/*
* Branching function to split up a function into a set of functions that
* are called for different scalar sizes of the objects handled.
*/
extern void __bad_size_call_parameter(void);
#ifdef CONFIG_DEBUG_PREEMPT
extern void __this_cpu_preempt_check(const char *op);
#else
static inline void __this_cpu_preempt_check(const char *op) { }
#endif
#define __pcpu_size_call_return(stem, variable) \
({ \
typeof(variable) pscr_ret__; \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: pscr_ret__ = stem##1(variable); break; \
case 2: pscr_ret__ = stem##2(variable); break; \
case 4: pscr_ret__ = stem##4(variable); break; \
case 8: pscr_ret__ = stem##8(variable); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pscr_ret__; \
})
#define __pcpu_size_call_return2(stem, variable, ...) \
({ \
typeof(variable) pscr2_ret__; \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pscr2_ret__; \
})
/*
* Special handling for cmpxchg_double. cmpxchg_double is passed two
* percpu variables. The first has to be aligned to a double word
* boundary and the second has to follow directly thereafter.
* We enforce this on all architectures even if they don't support
* a double cmpxchg instruction, since it's a cheap requirement, and it
* avoids breaking the requirement for architectures with the instruction.
*/
#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
({ \
bool pdcrb_ret__; \
__verify_pcpu_ptr(&(pcp1)); \
BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
VM_BUG_ON((unsigned long)(&(pcp2)) != \
(unsigned long)(&(pcp1)) + sizeof(pcp1)); \
switch(sizeof(pcp1)) { \
case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pdcrb_ret__; \
})
#define __pcpu_size_call(stem, variable, ...) \
do { \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: stem##1(variable, __VA_ARGS__);break; \
case 2: stem##2(variable, __VA_ARGS__);break; \
case 4: stem##4(variable, __VA_ARGS__);break; \
case 8: stem##8(variable, __VA_ARGS__);break; \
default: \
__bad_size_call_parameter();break; \
} \
} while (0)
/*
* this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
*
* Optimized manipulation for memory allocated through the per cpu
* allocator or for addresses of per cpu variables.
*
* These operation guarantee exclusivity of access for other operations
* on the *same* processor. The assumption is that per cpu data is only
* accessed by a single processor instance (the current one).
*
* The arch code can provide optimized implementation by defining macros
* for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
* cpu atomic operations for 2 byte sized RMW actions. If arch code does
* not provide operations for a scalar size then the fallback in the
* generic code will be used.
*
* cmpxchg_double replaces two adjacent scalars at once. The first two
* parameters are per cpu variables which have to be of the same size. A
* truth value is returned to indicate success or failure (since a double
* register result is difficult to handle). There is very limited hardware
* support for these operations, so only certain sizes may work.
*/
/*
* Operations for contexts where we do not want to do any checks for
* preemptions. Unless strictly necessary, always use [__]this_cpu_*()
* instead.
*
* If there is no other protection through preempt disable and/or disabling
* interupts then one of these RMW operations can show unexpected behavior
* because the execution thread was rescheduled on another processor or an
* interrupt occurred and the same percpu variable was modified from the
* interrupt context.
*/
#define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp)
#define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val)
#define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val)
#define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val)
#define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val)
#define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
#define raw_cpu_cmpxchg(pcp, oval, nval) \
__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
/*
* Operations for contexts that are safe from preemption/interrupts. These
* operations verify that preemption is disabled.
*/
#define __this_cpu_read(pcp) \
({ \
__this_cpu_preempt_check("read"); \
raw_cpu_read(pcp); \
})
#define __this_cpu_write(pcp, val) \
({ \
__this_cpu_preempt_check("write"); \
raw_cpu_write(pcp, val); \
})
#define __this_cpu_add(pcp, val) \
({ \
__this_cpu_preempt_check("add"); \
raw_cpu_add(pcp, val); \
})
#define __this_cpu_and(pcp, val) \
({ \
__this_cpu_preempt_check("and"); \
raw_cpu_and(pcp, val); \
})
#define __this_cpu_or(pcp, val) \
({ \
__this_cpu_preempt_check("or"); \
raw_cpu_or(pcp, val); \
})
#define __this_cpu_add_return(pcp, val) \
({ \
__this_cpu_preempt_check("add_return"); \
raw_cpu_add_return(pcp, val); \
})
#define __this_cpu_xchg(pcp, nval) \
({ \
__this_cpu_preempt_check("xchg"); \
raw_cpu_xchg(pcp, nval); \
})
#define __this_cpu_cmpxchg(pcp, oval, nval) \
({ \
__this_cpu_preempt_check("cmpxchg"); \
raw_cpu_cmpxchg(pcp, oval, nval); \
})
#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({ __this_cpu_preempt_check("cmpxchg_double"); \
raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
})
#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
/*
* Operations with implied preemption protection. These operations can be
* used without worrying about preemption. Note that interrupts may still
* occur while an operation is in progress and if the interrupt modifies
* the variable too then RMW actions may not be reliable.
*/
#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
#define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
#define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, pcp, val)
#define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, pcp, val)
#define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
#define this_cpu_cmpxchg(pcp, oval, nval) \
__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
#endif /* __ASSEMBLY__ */
#endif /* _LINUX_PERCPU_DEFS_H */ #endif /* _LINUX_PERCPU_DEFS_H */

64
include/linux/percpu-refcount.h
View file

@ -57,11 +57,9 @@ struct percpu_ref {
atomic_t count; atomic_t count;
/* /*
* The low bit of the pointer indicates whether the ref is in percpu * The low bit of the pointer indicates whether the ref is in percpu
* mode; if set, then get/put will manipulate the atomic_t (this is a * mode; if set, then get/put will manipulate the atomic_t.
* hack because we need to keep the pointer around for
* percpu_ref_kill_rcu())
*/ */
unsigned __percpu *pcpu_count; unsigned long pcpu_count_ptr;
percpu_ref_func_t *release; percpu_ref_func_t *release;
percpu_ref_func_t *confirm_kill; percpu_ref_func_t *confirm_kill;
struct rcu_head rcu; struct rcu_head rcu;
@ -69,7 +67,8 @@ struct percpu_ref {
int __must_check percpu_ref_init(struct percpu_ref *ref, int __must_check percpu_ref_init(struct percpu_ref *ref,
percpu_ref_func_t *release); percpu_ref_func_t *release);
void percpu_ref_cancel_init(struct percpu_ref *ref); void percpu_ref_reinit(struct percpu_ref *ref);
void percpu_ref_exit(struct percpu_ref *ref);
void percpu_ref_kill_and_confirm(struct percpu_ref *ref, void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
percpu_ref_func_t *confirm_kill); percpu_ref_func_t *confirm_kill);
@ -88,12 +87,28 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
return percpu_ref_kill_and_confirm(ref, NULL); return percpu_ref_kill_and_confirm(ref, NULL);
} }
#define PCPU_STATUS_BITS 2
#define PCPU_STATUS_MASK ((1 << PCPU_STATUS_BITS) - 1)
#define PCPU_REF_PTR 0
#define PCPU_REF_DEAD 1 #define PCPU_REF_DEAD 1
#define REF_STATUS(count) (((unsigned long) count) & PCPU_STATUS_MASK) /*
* Internal helper. Don't use outside percpu-refcount proper. The
* function doesn't return the pointer and let the caller test it for NULL
* because doing so forces the compiler to generate two conditional
* branches as it can't assume that @ref->pcpu_count is not NULL.
*/
static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
unsigned __percpu **pcpu_countp)
{
unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);
/* paired with smp_store_release() in percpu_ref_reinit() */
smp_read_barrier_depends();
if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
return false;
*pcpu_countp = (unsigned __percpu *)pcpu_ptr;
return true;
}
/** /**
* percpu_ref_get - increment a percpu refcount * percpu_ref_get - increment a percpu refcount
@ -107,9 +122,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
rcu_read_lock_sched(); rcu_read_lock_sched();
pcpu_count = ACCESS_ONCE(ref->pcpu_count); if (__pcpu_ref_alive(ref, &pcpu_count))
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
this_cpu_inc(*pcpu_count); this_cpu_inc(*pcpu_count);
else else
atomic_inc(&ref->count); atomic_inc(&ref->count);
@ -133,9 +146,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
rcu_read_lock_sched(); rcu_read_lock_sched();
pcpu_count = ACCESS_ONCE(ref->pcpu_count); if (__pcpu_ref_alive(ref, &pcpu_count)) {
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
this_cpu_inc(*pcpu_count); this_cpu_inc(*pcpu_count);
ret = true; ret = true;
} else { } else {
@ -168,9 +179,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
rcu_read_lock_sched(); rcu_read_lock_sched();
pcpu_count = ACCESS_ONCE(ref->pcpu_count); if (__pcpu_ref_alive(ref, &pcpu_count)) {
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
this_cpu_inc(*pcpu_count); this_cpu_inc(*pcpu_count);
ret = true; ret = true;
} }
@ -193,9 +202,7 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
rcu_read_lock_sched(); rcu_read_lock_sched();
pcpu_count = ACCESS_ONCE(ref->pcpu_count); if (__pcpu_ref_alive(ref, &pcpu_count))
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
this_cpu_dec(*pcpu_count); this_cpu_dec(*pcpu_count);
else if (unlikely(atomic_dec_and_test(&ref->count))) else if (unlikely(atomic_dec_and_test(&ref->count)))
ref->release(ref); ref->release(ref);
@ -203,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
rcu_read_unlock_sched(); rcu_read_unlock_sched();
} }
/**
* percpu_ref_is_zero - test whether a percpu refcount reached zero
* @ref: percpu_ref to test
*
* Returns %true if @ref reached zero.
*/
static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count;
if (__pcpu_ref_alive(ref, &pcpu_count))
return false;
return !atomic_read(&ref->count);
}
#endif #endif

673
include/linux/percpu.h
View file

@ -23,32 +23,6 @@
PERCPU_MODULE_RESERVE) PERCPU_MODULE_RESERVE)
#endif #endif
/*
* Must be an lvalue. Since @var must be a simple identifier,
* we force a syntax error here if it isn't.
*/
#define get_cpu_var(var) (*({ \
preempt_disable(); \
this_cpu_ptr(&var); }))
/*
* The weird & is necessary because sparse considers (void)(var) to be
* a direct dereference of percpu variable (var).
*/
#define put_cpu_var(var) do { \
(void)&(var); \
preempt_enable(); \
} while (0)
#define get_cpu_ptr(var) ({ \
preempt_disable(); \
this_cpu_ptr(var); })
#define put_cpu_ptr(var) do { \
(void)(var); \
preempt_enable(); \
} while (0)
/* minimum unit size, also is the maximum supported allocation size */ /* minimum unit size, also is the maximum supported allocation size */
#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
@ -140,17 +114,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
pcpu_fc_populate_pte_fn_t populate_pte_fn); pcpu_fc_populate_pte_fn_t populate_pte_fn);
#endif #endif
/*
* Use this to get to a cpu's version of the per-cpu object
* dynamically allocated. Non-atomic access to the current CPU's
* version should probably be combined with get_cpu()/put_cpu().
*/
#ifdef CONFIG_SMP
#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
#else
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
#endif
extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align); extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
extern bool is_kernel_percpu_address(unsigned long addr); extern bool is_kernel_percpu_address(unsigned long addr);
@ -166,640 +129,4 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
#define alloc_percpu(type) \ #define alloc_percpu(type) \
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type)) (typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
/*
* Branching function to split up a function into a set of functions that
* are called for different scalar sizes of the objects handled.
*/
extern void __bad_size_call_parameter(void);
#ifdef CONFIG_DEBUG_PREEMPT
extern void __this_cpu_preempt_check(const char *op);
#else
static inline void __this_cpu_preempt_check(const char *op) { }
#endif
#define __pcpu_size_call_return(stem, variable) \
({ typeof(variable) pscr_ret__; \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: pscr_ret__ = stem##1(variable);break; \
case 2: pscr_ret__ = stem##2(variable);break; \
case 4: pscr_ret__ = stem##4(variable);break; \
case 8: pscr_ret__ = stem##8(variable);break; \
default: \
__bad_size_call_parameter();break; \
} \
pscr_ret__; \
})
#define __pcpu_size_call_return2(stem, variable, ...) \
({ \
typeof(variable) pscr2_ret__; \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pscr2_ret__; \
})
/*
* Special handling for cmpxchg_double. cmpxchg_double is passed two
* percpu variables. The first has to be aligned to a double word
* boundary and the second has to follow directly thereafter.
* We enforce this on all architectures even if they don't support
* a double cmpxchg instruction, since it's a cheap requirement, and it
* avoids breaking the requirement for architectures with the instruction.
*/
#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
({ \
bool pdcrb_ret__; \
__verify_pcpu_ptr(&pcp1); \
BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
VM_BUG_ON((unsigned long)(&pcp2) != \
(unsigned long)(&pcp1) + sizeof(pcp1)); \
switch(sizeof(pcp1)) { \
case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
default: \
__bad_size_call_parameter(); break; \
} \
pdcrb_ret__; \
})
#define __pcpu_size_call(stem, variable, ...) \
do { \
__verify_pcpu_ptr(&(variable)); \
switch(sizeof(variable)) { \
case 1: stem##1(variable, __VA_ARGS__);break; \
case 2: stem##2(variable, __VA_ARGS__);break; \
case 4: stem##4(variable, __VA_ARGS__);break; \
case 8: stem##8(variable, __VA_ARGS__);break; \
default: \
__bad_size_call_parameter();break; \
} \
} while (0)
/*
* this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
*
* Optimized manipulation for memory allocated through the per cpu
* allocator or for addresses of per cpu variables.
*
* These operation guarantee exclusivity of access for other operations
* on the *same* processor. The assumption is that per cpu data is only
* accessed by a single processor instance (the current one).
*
* The first group is used for accesses that must be done in a
* preemption safe way since we know that the context is not preempt
* safe. Interrupts may occur. If the interrupt modifies the variable
* too then RMW actions will not be reliable.
*
* The arch code can provide optimized functions in two ways:
*
* 1. Override the function completely. F.e. define this_cpu_add().
* The arch must then ensure that the various scalar format passed
* are handled correctly.
*
* 2. Provide functions for certain scalar sizes. F.e. provide
* this_cpu_add_2() to provide per cpu atomic operations for 2 byte
* sized RMW actions. If arch code does not provide operations for
* a scalar size then the fallback in the generic code will be
* used.
*/
#define _this_cpu_generic_read(pcp) \
({ typeof(pcp) ret__; \
preempt_disable(); \
ret__ = *this_cpu_ptr(&(pcp)); \
preempt_enable(); \
ret__; \
})
#ifndef this_cpu_read
# ifndef this_cpu_read_1
# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
# endif
# ifndef this_cpu_read_2
# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
# endif
# ifndef this_cpu_read_4
# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
# endif
# ifndef this_cpu_read_8
# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
# endif
# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
#endif
#define _this_cpu_generic_to_op(pcp, val, op) \
do { \
unsigned long flags; \
raw_local_irq_save(flags); \
*raw_cpu_ptr(&(pcp)) op val; \
raw_local_irq_restore(flags); \
} while (0)
#ifndef this_cpu_write
# ifndef this_cpu_write_1
# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
# endif
# ifndef this_cpu_write_2
# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
# endif
# ifndef this_cpu_write_4
# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
# endif
# ifndef this_cpu_write_8
# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
# endif
# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
#endif
#ifndef this_cpu_add
# ifndef this_cpu_add_1
# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
# endif
# ifndef this_cpu_add_2
# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
# endif
# ifndef this_cpu_add_4
# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
# endif
# ifndef this_cpu_add_8
# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
# endif
# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
#endif
#ifndef this_cpu_sub
# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val))
#endif
#ifndef this_cpu_inc
# define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
#endif
#ifndef this_cpu_dec
# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
#endif
#ifndef this_cpu_and
# ifndef this_cpu_and_1
# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
# endif
# ifndef this_cpu_and_2
# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
# endif
# ifndef this_cpu_and_4
# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
# endif
# ifndef this_cpu_and_8
# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
# endif
# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
#endif
#ifndef this_cpu_or
# ifndef this_cpu_or_1
# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
# endif
# ifndef this_cpu_or_2
# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
# endif
# ifndef this_cpu_or_4
# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
# endif
# ifndef this_cpu_or_8
# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
# endif
# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
#endif
#define _this_cpu_generic_add_return(pcp, val) \
({ \
typeof(pcp) ret__; \
unsigned long flags; \
raw_local_irq_save(flags); \
raw_cpu_add(pcp, val); \
ret__ = raw_cpu_read(pcp); \
raw_local_irq_restore(flags); \
ret__; \
})
#ifndef this_cpu_add_return
# ifndef this_cpu_add_return_1
# define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
# endif
# ifndef this_cpu_add_return_2
# define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
# endif
# ifndef this_cpu_add_return_4
# define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
# endif
# ifndef this_cpu_add_return_8
# define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
# endif
# define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
#endif
#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
#define _this_cpu_generic_xchg(pcp, nval) \
({ typeof(pcp) ret__; \
unsigned long flags; \
raw_local_irq_save(flags); \
ret__ = raw_cpu_read(pcp); \
raw_cpu_write(pcp, nval); \
raw_local_irq_restore(flags); \
ret__; \
})
#ifndef this_cpu_xchg
# ifndef this_cpu_xchg_1
# define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
# endif
# ifndef this_cpu_xchg_2
# define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
# endif
# ifndef this_cpu_xchg_4
# define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
# endif
# ifndef this_cpu_xchg_8
# define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
# endif
# define this_cpu_xchg(pcp, nval) \
__pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
#endif
#define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
({ \
typeof(pcp) ret__; \
unsigned long flags; \
raw_local_irq_save(flags); \
ret__ = raw_cpu_read(pcp); \
if (ret__ == (oval)) \
raw_cpu_write(pcp, nval); \
raw_local_irq_restore(flags); \
ret__; \
})
#ifndef this_cpu_cmpxchg
# ifndef this_cpu_cmpxchg_1
# define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# ifndef this_cpu_cmpxchg_2
# define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# ifndef this_cpu_cmpxchg_4
# define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# ifndef this_cpu_cmpxchg_8
# define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# define this_cpu_cmpxchg(pcp, oval, nval) \
__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
#endif
/*
* cmpxchg_double replaces two adjacent scalars at once. The first
* two parameters are per cpu variables which have to be of the same
* size. A truth value is returned to indicate success or failure
* (since a double register result is difficult to handle). There is
* very limited hardware support for these operations, so only certain
* sizes may work.
*/
#define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({ \
int ret__; \
unsigned long flags; \
raw_local_irq_save(flags); \
ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
oval1, oval2, nval1, nval2); \
raw_local_irq_restore(flags); \
ret__; \
})
#ifndef this_cpu_cmpxchg_double
# ifndef this_cpu_cmpxchg_double_1
# define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# ifndef this_cpu_cmpxchg_double_2
# define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# ifndef this_cpu_cmpxchg_double_4
# define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# ifndef this_cpu_cmpxchg_double_8
# define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
#endif
/*
* Generic percpu operations for contexts where we do not want to do
* any checks for preemptiosn.
*
* If there is no other protection through preempt disable and/or
* disabling interupts then one of these RMW operations can show unexpected
* behavior because the execution thread was rescheduled on another processor
* or an interrupt occurred and the same percpu variable was modified from
* the interrupt context.
*/
#ifndef raw_cpu_read
# ifndef raw_cpu_read_1
# define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
# endif
# ifndef raw_cpu_read_2
# define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
# endif
# ifndef raw_cpu_read_4
# define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
# endif
# ifndef raw_cpu_read_8
# define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
# endif
# define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
#endif
#define raw_cpu_generic_to_op(pcp, val, op) \
do { \
*raw_cpu_ptr(&(pcp)) op val; \
} while (0)
#ifndef raw_cpu_write
# ifndef raw_cpu_write_1
# define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
# endif
# ifndef raw_cpu_write_2
# define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
# endif
# ifndef raw_cpu_write_4
# define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
# endif
# ifndef raw_cpu_write_8
# define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
# endif
# define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
#endif
#ifndef raw_cpu_add
# ifndef raw_cpu_add_1
# define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
# endif
# ifndef raw_cpu_add_2
# define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
# endif
# ifndef raw_cpu_add_4
# define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
# endif
# ifndef raw_cpu_add_8
# define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
# endif
# define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
#endif
#ifndef raw_cpu_sub
# define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val))
#endif
#ifndef raw_cpu_inc
# define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1)
#endif
#ifndef raw_cpu_dec
# define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1)
#endif
#ifndef raw_cpu_and
# ifndef raw_cpu_and_1
# define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
# endif
# ifndef raw_cpu_and_2
# define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
# endif
# ifndef raw_cpu_and_4
# define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
# endif
# ifndef raw_cpu_and_8
# define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
# endif
# define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
#endif
#ifndef raw_cpu_or
# ifndef raw_cpu_or_1
# define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
# endif
# ifndef raw_cpu_or_2
# define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
# endif
# ifndef raw_cpu_or_4
# define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
# endif
# ifndef raw_cpu_or_8
# define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
# endif
# define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
#endif
#define raw_cpu_generic_add_return(pcp, val) \
({ \
raw_cpu_add(pcp, val); \
raw_cpu_read(pcp); \
})
#ifndef raw_cpu_add_return
# ifndef raw_cpu_add_return_1
# define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
# endif
# ifndef raw_cpu_add_return_2
# define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
# endif
# ifndef raw_cpu_add_return_4
# define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
# endif
# ifndef raw_cpu_add_return_8
# define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
# endif
# define raw_cpu_add_return(pcp, val) \
__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
#endif
#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
#define raw_cpu_generic_xchg(pcp, nval) \
({ typeof(pcp) ret__; \
ret__ = raw_cpu_read(pcp); \
raw_cpu_write(pcp, nval); \
ret__; \
})
#ifndef raw_cpu_xchg
# ifndef raw_cpu_xchg_1
# define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
# endif
# ifndef raw_cpu_xchg_2
# define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
# endif
# ifndef raw_cpu_xchg_4
# define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
# endif
# ifndef raw_cpu_xchg_8
# define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
# endif
# define raw_cpu_xchg(pcp, nval) \
__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
#endif
#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
({ \
typeof(pcp) ret__; \
ret__ = raw_cpu_read(pcp); \
if (ret__ == (oval)) \
raw_cpu_write(pcp, nval); \
ret__; \
})
#ifndef raw_cpu_cmpxchg
# ifndef raw_cpu_cmpxchg_1
# define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# ifndef raw_cpu_cmpxchg_2
# define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# ifndef raw_cpu_cmpxchg_4
# define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# ifndef raw_cpu_cmpxchg_8
# define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
# endif
# define raw_cpu_cmpxchg(pcp, oval, nval) \
__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
#endif
#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
({ \
int __ret = 0; \
if (raw_cpu_read(pcp1) == (oval1) && \
raw_cpu_read(pcp2) == (oval2)) { \
raw_cpu_write(pcp1, (nval1)); \
raw_cpu_write(pcp2, (nval2)); \
__ret = 1; \
} \
(__ret); \
})
#ifndef raw_cpu_cmpxchg_double
# ifndef raw_cpu_cmpxchg_double_1
# define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# ifndef raw_cpu_cmpxchg_double_2
# define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# ifndef raw_cpu_cmpxchg_double_4
# define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# ifndef raw_cpu_cmpxchg_double_8
# define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
# endif
# define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
#endif
/*
* Generic percpu operations for context that are safe from preemption/interrupts.
*/
#ifndef __this_cpu_read
# define __this_cpu_read(pcp) \
(__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp)))
#endif
#ifndef __this_cpu_write
# define __this_cpu_write(pcp, val) \
do { __this_cpu_preempt_check("write"); \
__pcpu_size_call(raw_cpu_write_, (pcp), (val)); \
} while (0)
#endif
#ifndef __this_cpu_add
# define __this_cpu_add(pcp, val) \
do { __this_cpu_preempt_check("add"); \
__pcpu_size_call(raw_cpu_add_, (pcp), (val)); \
} while (0)
#endif
#ifndef __this_cpu_sub
# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
#endif
#ifndef __this_cpu_inc
# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
#endif
#ifndef __this_cpu_dec
# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
#endif
#ifndef __this_cpu_and
# define __this_cpu_and(pcp, val) \
do { __this_cpu_preempt_check("and"); \
__pcpu_size_call(raw_cpu_and_, (pcp), (val)); \
} while (0)
#endif
#ifndef __this_cpu_or
# define __this_cpu_or(pcp, val) \
do { __this_cpu_preempt_check("or"); \
__pcpu_size_call(raw_cpu_or_, (pcp), (val)); \
} while (0)
#endif
#ifndef __this_cpu_add_return
# define __this_cpu_add_return(pcp, val) \
(__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val))
#endif
#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
#ifndef __this_cpu_xchg
# define __this_cpu_xchg(pcp, nval) \
(__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval))
#endif
#ifndef __this_cpu_cmpxchg
# define __this_cpu_cmpxchg(pcp, oval, nval) \
(__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval))
#endif
#ifndef __this_cpu_cmpxchg_double
# define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
(__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))
#endif
#endif /* __LINUX_PERCPU_H */ #endif /* __LINUX_PERCPU_H */

8
kernel/cgroup.c
View file

@ -1638,7 +1638,7 @@ static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
exit_root_id: exit_root_id:
cgroup_exit_root_id(root); cgroup_exit_root_id(root);
cancel_ref: cancel_ref:
percpu_ref_cancel_init(&root_cgrp->self.refcnt); percpu_ref_exit(&root_cgrp->self.refcnt);
out: out:
free_cgrp_cset_links(&tmp_links); free_cgrp_cset_links(&tmp_links);
return ret; return ret;
@ -4133,6 +4133,8 @@ static void css_free_work_fn(struct work_struct *work)
container_of(work, struct cgroup_subsys_state, destroy_work); container_of(work, struct cgroup_subsys_state, destroy_work);
struct cgroup *cgrp = css->cgroup; struct cgroup *cgrp = css->cgroup;
percpu_ref_exit(&css->refcnt);
if (css->ss) { if (css->ss) {
/* css free path */ /* css free path */
if (css->parent) if (css->parent)
@ -4330,7 +4332,7 @@ static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss)
err_free_id: err_free_id:
cgroup_idr_remove(&ss->css_idr, css->id); cgroup_idr_remove(&ss->css_idr, css->id);
err_free_percpu_ref: err_free_percpu_ref:
percpu_ref_cancel_init(&css->refcnt); percpu_ref_exit(&css->refcnt);
err_free_css: err_free_css:
call_rcu(&css->rcu_head, css_free_rcu_fn); call_rcu(&css->rcu_head, css_free_rcu_fn);
return err; return err;
@ -4441,7 +4443,7 @@ static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
out_free_id: out_free_id:
cgroup_idr_remove(&root->cgroup_idr, cgrp->id); cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
out_cancel_ref: out_cancel_ref:
percpu_ref_cancel_init(&cgrp->self.refcnt); percpu_ref_exit(&cgrp->self.refcnt);
out_free_cgrp: out_free_cgrp:
kfree(cgrp); kfree(cgrp);
out_unlock: out_unlock:

19
kernel/workqueue.c
View file

@ -2020,13 +2020,8 @@ __acquires(&pool->lock)
lockdep_copy_map(&lockdep_map, &work->lockdep_map); lockdep_copy_map(&lockdep_map, &work->lockdep_map);
#endif #endif
/* /* ensure we're on the correct CPU */
* Ensure we're on the correct CPU. DISASSOCIATED test is WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
* necessary to avoid spurious warnings from rescuers servicing the
* unbound or a disassociated pool.
*/
WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
!(pool->flags & POOL_DISASSOCIATED) &&
raw_smp_processor_id() != pool->cpu); raw_smp_processor_id() != pool->cpu);
/* /*
@ -4542,6 +4537,7 @@ static void rebind_workers(struct worker_pool *pool)
pool->attrs->cpumask) < 0); pool->attrs->cpumask) < 0);
spin_lock_irq(&pool->lock); spin_lock_irq(&pool->lock);
pool->flags &= ~POOL_DISASSOCIATED;
for_each_pool_worker(worker, pool) { for_each_pool_worker(worker, pool) {
unsigned int worker_flags = worker->flags; unsigned int worker_flags = worker->flags;
@ -4643,15 +4639,10 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb,
for_each_pool(pool, pi) { for_each_pool(pool, pi) {
mutex_lock(&pool->attach_mutex); mutex_lock(&pool->attach_mutex);
if (pool->cpu == cpu) { if (pool->cpu == cpu)
spin_lock_irq(&pool->lock);
pool->flags &= ~POOL_DISASSOCIATED;
spin_unlock_irq(&pool->lock);
rebind_workers(pool); rebind_workers(pool);
} else if (pool->cpu < 0) { else if (pool->cpu < 0)
restore_unbound_workers_cpumask(pool, cpu); restore_unbound_workers_cpumask(pool, cpu);
}
mutex_unlock(&pool->attach_mutex); mutex_unlock(&pool->attach_mutex);
} }

86
lib/percpu-refcount.c
View file

@ -31,6 +31,11 @@
#define PCPU_COUNT_BIAS (1U << 31) #define PCPU_COUNT_BIAS (1U << 31)
static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
{
return (unsigned __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
}
/** /**
* percpu_ref_init - initialize a percpu refcount * percpu_ref_init - initialize a percpu refcount
* @ref: percpu_ref to initialize * @ref: percpu_ref to initialize
@ -46,8 +51,8 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)
{ {
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS); atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
ref->pcpu_count = alloc_percpu(unsigned); ref->pcpu_count_ptr = (unsigned long)alloc_percpu(unsigned);
if (!ref->pcpu_count) if (!ref->pcpu_count_ptr)
return -ENOMEM; return -ENOMEM;
ref->release = release; ref->release = release;
@ -56,53 +61,71 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)
EXPORT_SYMBOL_GPL(percpu_ref_init); EXPORT_SYMBOL_GPL(percpu_ref_init);
/** /**
* percpu_ref_cancel_init - cancel percpu_ref_init() * percpu_ref_reinit - re-initialize a percpu refcount
* @ref: percpu_ref to cancel init for * @ref: perpcu_ref to re-initialize
* *
* Once a percpu_ref is initialized, its destruction is initiated by * Re-initialize @ref so that it's in the same state as when it finished
* percpu_ref_kill() and completes asynchronously, which can be painful to * percpu_ref_init(). @ref must have been initialized successfully, killed
* do when destroying a half-constructed object in init failure path. * and reached 0 but not exited.
* *
* This function destroys @ref without invoking @ref->release and the * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while
* memory area containing it can be freed immediately on return. To * this function is in progress.
* prevent accidental misuse, it's required that @ref has finished
* percpu_ref_init(), whether successful or not, but never used.
*
* The weird name and usage restriction are to prevent people from using
* this function by mistake for normal shutdown instead of
* percpu_ref_kill().
*/ */
void percpu_ref_cancel_init(struct percpu_ref *ref) void percpu_ref_reinit(struct percpu_ref *ref)
{ {
unsigned __percpu *pcpu_count = ref->pcpu_count; unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
int cpu; int cpu;
WARN_ON_ONCE(atomic_read(&ref->count) != 1 + PCPU_COUNT_BIAS); BUG_ON(!pcpu_count);
WARN_ON(!percpu_ref_is_zero(ref));
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
/*
* Restore per-cpu operation. smp_store_release() is paired with
* smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees
* that the zeroing is visible to all percpu accesses which can see
* the following PCPU_REF_DEAD clearing.
*/
for_each_possible_cpu(cpu)
*per_cpu_ptr(pcpu_count, cpu) = 0;
smp_store_release(&ref->pcpu_count_ptr,
ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
}
EXPORT_SYMBOL_GPL(percpu_ref_reinit);
/**
* percpu_ref_exit - undo percpu_ref_init()
* @ref: percpu_ref to exit
*
* This function exits @ref. The caller is responsible for ensuring that
* @ref is no longer in active use. The usual places to invoke this
* function from are the @ref->release() callback or in init failure path
* where percpu_ref_init() succeeded but other parts of the initialization
* of the embedding object failed.
*/
void percpu_ref_exit(struct percpu_ref *ref)
{
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
if (pcpu_count) { if (pcpu_count) {
for_each_possible_cpu(cpu) free_percpu(pcpu_count);
WARN_ON_ONCE(*per_cpu_ptr(pcpu_count, cpu)); ref->pcpu_count_ptr = PCPU_REF_DEAD;
free_percpu(ref->pcpu_count);
} }
} }
EXPORT_SYMBOL_GPL(percpu_ref_cancel_init); EXPORT_SYMBOL_GPL(percpu_ref_exit);
static void percpu_ref_kill_rcu(struct rcu_head *rcu) static void percpu_ref_kill_rcu(struct rcu_head *rcu)
{ {
struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
unsigned __percpu *pcpu_count = ref->pcpu_count; unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
unsigned count = 0; unsigned count = 0;
int cpu; int cpu;
/* Mask out PCPU_REF_DEAD */
pcpu_count = (unsigned __percpu *)
(((unsigned long) pcpu_count) & ~PCPU_STATUS_MASK);
for_each_possible_cpu(cpu) for_each_possible_cpu(cpu)
count += *per_cpu_ptr(pcpu_count, cpu); count += *per_cpu_ptr(pcpu_count, cpu);
free_percpu(pcpu_count);
pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count); pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count);
/* /*
@ -152,11 +175,10 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
void percpu_ref_kill_and_confirm(struct percpu_ref *ref, void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
percpu_ref_func_t *confirm_kill) percpu_ref_func_t *confirm_kill)
{ {
WARN_ONCE(REF_STATUS(ref->pcpu_count) == PCPU_REF_DEAD, WARN_ONCE(ref->pcpu_count_ptr & PCPU_REF_DEAD,
"percpu_ref_kill() called more than once!\n"); "percpu_ref_kill() called more than once!\n");
ref->pcpu_count = (unsigned __percpu *) ref->pcpu_count_ptr |= PCPU_REF_DEAD;
(((unsigned long) ref->pcpu_count)|PCPU_REF_DEAD);
ref->confirm_kill = confirm_kill; ref->confirm_kill = confirm_kill;
call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu); call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu);

3
mm/percpu.c
View file

@ -720,8 +720,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
if (unlikely(align < 2)) if (unlikely(align < 2))
align = 2; align = 2;
if (unlikely(size & 1)) size = ALIGN(size, 2);
size++;
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) { if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
WARN(true, "illegal size (%zu) or align (%zu) for " WARN(true, "illegal size (%zu) or align (%zu) for "