Merge branch 'for-3.17' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu into for-3.17/core
Merge the percpu_ref changes from Tejun, he says they are stable now.
This commit is contained in:
Jens Axboe
commit
17737d3b59
11 changed files with 838 additions and 826 deletions
|
|
@ -52,10 +52,9 @@
|
|||
* Compared to the generic __my_cpu_offset version, the following
|
||||
* saves one instruction and avoids clobbering a temp register.
|
||||
*/
|
||||
#define raw_cpu_ptr(ptr) \
|
||||
#define arch_raw_cpu_ptr(ptr) \
|
||||
({ \
|
||||
unsigned long tcp_ptr__; \
|
||||
__verify_pcpu_ptr(ptr); \
|
||||
asm volatile("add " __percpu_arg(1) ", %0" \
|
||||
: "=r" (tcp_ptr__) \
|
||||
: "m" (this_cpu_off), "0" (ptr)); \
|
||||
|
|
|
|||
|
|
@ -825,7 +825,7 @@ int core_tpg_add_lun(
|
|||
|
||||
ret = core_dev_export(dev, tpg, lun);
|
||||
if (ret < 0) {
|
||||
percpu_ref_cancel_init(&lun->lun_ref);
|
||||
percpu_ref_exit(&lun->lun_ref);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -880,5 +880,7 @@ int core_tpg_post_dellun(
|
|||
lun->lun_status = TRANSPORT_LUN_STATUS_FREE;
|
||||
spin_unlock(&tpg->tpg_lun_lock);
|
||||
|
||||
percpu_ref_exit(&lun->lun_ref);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
6
fs/aio.c
6
fs/aio.c
|
|
@ -506,6 +506,8 @@ static void free_ioctx(struct work_struct *work)
|
|||
|
||||
aio_free_ring(ctx);
|
||||
free_percpu(ctx->cpu);
|
||||
percpu_ref_exit(&ctx->reqs);
|
||||
percpu_ref_exit(&ctx->users);
|
||||
kmem_cache_free(kioctx_cachep, ctx);
|
||||
}
|
||||
|
||||
|
|
@ -715,8 +717,8 @@ static struct kioctx *ioctx_alloc(unsigned nr_events)
|
|||
err:
|
||||
mutex_unlock(&ctx->ring_lock);
|
||||
free_percpu(ctx->cpu);
|
||||
free_percpu(ctx->reqs.pcpu_count);
|
||||
free_percpu(ctx->users.pcpu_count);
|
||||
percpu_ref_exit(&ctx->reqs);
|
||||
percpu_ref_exit(&ctx->users);
|
||||
kmem_cache_free(kioctx_cachep, ctx);
|
||||
pr_debug("error allocating ioctx %d\n", err);
|
||||
return ERR_PTR(err);
|
||||
|
|
|
|||
|
|
@ -36,55 +36,17 @@ extern unsigned long __per_cpu_offset[NR_CPUS];
|
|||
#endif
|
||||
|
||||
/*
|
||||
* Add a offset to a pointer but keep the pointer as is.
|
||||
*
|
||||
* Only S390 provides its own means of moving the pointer.
|
||||
* Arch may define arch_raw_cpu_ptr() to provide more efficient address
|
||||
* translations for raw_cpu_ptr().
|
||||
*/
|
||||
#ifndef SHIFT_PERCPU_PTR
|
||||
/* Weird cast keeps both GCC and sparse happy. */
|
||||
#define SHIFT_PERCPU_PTR(__p, __offset) ({ \
|
||||
__verify_pcpu_ptr((__p)); \
|
||||
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset)); \
|
||||
})
|
||||
#ifndef arch_raw_cpu_ptr
|
||||
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* A percpu variable may point to a discarded regions. The following are
|
||||
* established ways to produce a usable pointer from the percpu variable
|
||||
* offset.
|
||||
*/
|
||||
#define per_cpu(var, cpu) \
|
||||
(*SHIFT_PERCPU_PTR(&(var), per_cpu_offset(cpu)))
|
||||
|
||||
#ifndef raw_cpu_ptr
|
||||
#define raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
|
||||
#endif
|
||||
#ifdef CONFIG_DEBUG_PREEMPT
|
||||
#define this_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, my_cpu_offset)
|
||||
#else
|
||||
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
|
||||
#endif
|
||||
|
||||
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
|
||||
#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
|
||||
|
||||
#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
|
||||
extern void setup_per_cpu_areas(void);
|
||||
#endif
|
||||
|
||||
#else /* ! SMP */
|
||||
|
||||
#define VERIFY_PERCPU_PTR(__p) ({ \
|
||||
__verify_pcpu_ptr((__p)); \
|
||||
(typeof(*(__p)) __kernel __force *)(__p); \
|
||||
})
|
||||
|
||||
#define per_cpu(var, cpu) (*((void)(cpu), VERIFY_PERCPU_PTR(&(var))))
|
||||
#define __get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var)))
|
||||
#define __raw_get_cpu_var(var) (*VERIFY_PERCPU_PTR(&(var)))
|
||||
#define this_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
|
||||
#define raw_cpu_ptr(ptr) this_cpu_ptr(ptr)
|
||||
|
||||
#endif /* SMP */
|
||||
|
||||
#ifndef PER_CPU_BASE_SECTION
|
||||
|
|
@ -95,25 +57,6 @@ extern void setup_per_cpu_areas(void);
|
|||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
|
||||
#ifdef MODULE
|
||||
#define PER_CPU_SHARED_ALIGNED_SECTION ""
|
||||
#define PER_CPU_ALIGNED_SECTION ""
|
||||
#else
|
||||
#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
|
||||
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
|
||||
#endif
|
||||
#define PER_CPU_FIRST_SECTION "..first"
|
||||
|
||||
#else
|
||||
|
||||
#define PER_CPU_SHARED_ALIGNED_SECTION ""
|
||||
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
|
||||
#define PER_CPU_FIRST_SECTION ""
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef PER_CPU_ATTRIBUTES
|
||||
#define PER_CPU_ATTRIBUTES
|
||||
#endif
|
||||
|
|
@ -122,7 +65,356 @@ extern void setup_per_cpu_areas(void);
|
|||
#define PER_CPU_DEF_ATTRIBUTES
|
||||
#endif
|
||||
|
||||
/* Keep until we have removed all uses of __this_cpu_ptr */
|
||||
#define __this_cpu_ptr raw_cpu_ptr
|
||||
#define raw_cpu_generic_to_op(pcp, val, op) \
|
||||
do { \
|
||||
*raw_cpu_ptr(&(pcp)) op val; \
|
||||
} while (0)
|
||||
|
||||
#define raw_cpu_generic_add_return(pcp, val) \
|
||||
({ \
|
||||
raw_cpu_add(pcp, val); \
|
||||
raw_cpu_read(pcp); \
|
||||
})
|
||||
|
||||
#define raw_cpu_generic_xchg(pcp, nval) \
|
||||
({ \
|
||||
typeof(pcp) __ret; \
|
||||
__ret = raw_cpu_read(pcp); \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
|
||||
({ \
|
||||
typeof(pcp) __ret; \
|
||||
__ret = raw_cpu_read(pcp); \
|
||||
if (__ret == (oval)) \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
({ \
|
||||
int __ret = 0; \
|
||||
if (raw_cpu_read(pcp1) == (oval1) && \
|
||||
raw_cpu_read(pcp2) == (oval2)) { \
|
||||
raw_cpu_write(pcp1, nval1); \
|
||||
raw_cpu_write(pcp2, nval2); \
|
||||
__ret = 1; \
|
||||
} \
|
||||
(__ret); \
|
||||
})
|
||||
|
||||
#define this_cpu_generic_read(pcp) \
|
||||
({ \
|
||||
typeof(pcp) __ret; \
|
||||
preempt_disable(); \
|
||||
__ret = *this_cpu_ptr(&(pcp)); \
|
||||
preempt_enable(); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define this_cpu_generic_to_op(pcp, val, op) \
|
||||
do { \
|
||||
unsigned long __flags; \
|
||||
raw_local_irq_save(__flags); \
|
||||
*raw_cpu_ptr(&(pcp)) op val; \
|
||||
raw_local_irq_restore(__flags); \
|
||||
} while (0)
|
||||
|
||||
#define this_cpu_generic_add_return(pcp, val) \
|
||||
({ \
|
||||
typeof(pcp) __ret; \
|
||||
unsigned long __flags; \
|
||||
raw_local_irq_save(__flags); \
|
||||
raw_cpu_add(pcp, val); \
|
||||
__ret = raw_cpu_read(pcp); \
|
||||
raw_local_irq_restore(__flags); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define this_cpu_generic_xchg(pcp, nval) \
|
||||
({ \
|
||||
typeof(pcp) __ret; \
|
||||
unsigned long __flags; \
|
||||
raw_local_irq_save(__flags); \
|
||||
__ret = raw_cpu_read(pcp); \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
raw_local_irq_restore(__flags); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
|
||||
({ \
|
||||
typeof(pcp) __ret; \
|
||||
unsigned long __flags; \
|
||||
raw_local_irq_save(__flags); \
|
||||
__ret = raw_cpu_read(pcp); \
|
||||
if (__ret == (oval)) \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
raw_local_irq_restore(__flags); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
({ \
|
||||
int __ret; \
|
||||
unsigned long __flags; \
|
||||
raw_local_irq_save(__flags); \
|
||||
__ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
|
||||
oval1, oval2, nval1, nval2); \
|
||||
raw_local_irq_restore(__flags); \
|
||||
__ret; \
|
||||
})
|
||||
|
||||
#ifndef raw_cpu_read_1
|
||||
#define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
#endif
|
||||
#ifndef raw_cpu_read_2
|
||||
#define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
#endif
|
||||
#ifndef raw_cpu_read_4
|
||||
#define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
#endif
|
||||
#ifndef raw_cpu_read_8
|
||||
#define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_write_1
|
||||
#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
#ifndef raw_cpu_write_2
|
||||
#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
#ifndef raw_cpu_write_4
|
||||
#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
#ifndef raw_cpu_write_8
|
||||
#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_add_1
|
||||
#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
#ifndef raw_cpu_add_2
|
||||
#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
#ifndef raw_cpu_add_4
|
||||
#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
#ifndef raw_cpu_add_8
|
||||
#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_and_1
|
||||
#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
#ifndef raw_cpu_and_2
|
||||
#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
#ifndef raw_cpu_and_4
|
||||
#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
#ifndef raw_cpu_and_8
|
||||
#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_or_1
|
||||
#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
#ifndef raw_cpu_or_2
|
||||
#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
#ifndef raw_cpu_or_4
|
||||
#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
#ifndef raw_cpu_or_8
|
||||
#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_add_return_1
|
||||
#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
#ifndef raw_cpu_add_return_2
|
||||
#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
#ifndef raw_cpu_add_return_4
|
||||
#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
#ifndef raw_cpu_add_return_8
|
||||
#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_xchg_1
|
||||
#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
#ifndef raw_cpu_xchg_2
|
||||
#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
#ifndef raw_cpu_xchg_4
|
||||
#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
#ifndef raw_cpu_xchg_8
|
||||
#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_cmpxchg_1
|
||||
#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
|
||||
raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
#ifndef raw_cpu_cmpxchg_2
|
||||
#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
|
||||
raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
#ifndef raw_cpu_cmpxchg_4
|
||||
#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
|
||||
raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
#ifndef raw_cpu_cmpxchg_8
|
||||
#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
|
||||
raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_cmpxchg_double_1
|
||||
#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
#ifndef raw_cpu_cmpxchg_double_2
|
||||
#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
#ifndef raw_cpu_cmpxchg_double_4
|
||||
#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
#ifndef raw_cpu_cmpxchg_double_8
|
||||
#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_read_1
|
||||
#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
|
||||
#endif
|
||||
#ifndef this_cpu_read_2
|
||||
#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
|
||||
#endif
|
||||
#ifndef this_cpu_read_4
|
||||
#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
|
||||
#endif
|
||||
#ifndef this_cpu_read_8
|
||||
#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_write_1
|
||||
#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
#ifndef this_cpu_write_2
|
||||
#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
#ifndef this_cpu_write_4
|
||||
#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
#ifndef this_cpu_write_8
|
||||
#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_add_1
|
||||
#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
#ifndef this_cpu_add_2
|
||||
#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
#ifndef this_cpu_add_4
|
||||
#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
#ifndef this_cpu_add_8
|
||||
#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_and_1
|
||||
#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
#ifndef this_cpu_and_2
|
||||
#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
#ifndef this_cpu_and_4
|
||||
#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
#ifndef this_cpu_and_8
|
||||
#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_or_1
|
||||
#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
#ifndef this_cpu_or_2
|
||||
#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
#ifndef this_cpu_or_4
|
||||
#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
#ifndef this_cpu_or_8
|
||||
#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_add_return_1
|
||||
#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
#ifndef this_cpu_add_return_2
|
||||
#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
#ifndef this_cpu_add_return_4
|
||||
#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
#ifndef this_cpu_add_return_8
|
||||
#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_xchg_1
|
||||
#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
#ifndef this_cpu_xchg_2
|
||||
#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
#ifndef this_cpu_xchg_4
|
||||
#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
#ifndef this_cpu_xchg_8
|
||||
#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_cmpxchg_1
|
||||
#define this_cpu_cmpxchg_1(pcp, oval, nval) \
|
||||
this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
#ifndef this_cpu_cmpxchg_2
|
||||
#define this_cpu_cmpxchg_2(pcp, oval, nval) \
|
||||
this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
#ifndef this_cpu_cmpxchg_4
|
||||
#define this_cpu_cmpxchg_4(pcp, oval, nval) \
|
||||
this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
#ifndef this_cpu_cmpxchg_8
|
||||
#define this_cpu_cmpxchg_8(pcp, oval, nval) \
|
||||
this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_cmpxchg_double_1
|
||||
#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
#ifndef this_cpu_cmpxchg_double_2
|
||||
#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
#ifndef this_cpu_cmpxchg_double_4
|
||||
#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
#ifndef this_cpu_cmpxchg_double_8
|
||||
#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
#endif
|
||||
|
||||
#endif /* _ASM_GENERIC_PERCPU_H_ */
|
||||
|
|
|
|||
|
|
@ -1,6 +1,40 @@
|
|||
/*
|
||||
* linux/percpu-defs.h - basic definitions for percpu areas
|
||||
*
|
||||
* DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
|
||||
*
|
||||
* This file is separate from linux/percpu.h to avoid cyclic inclusion
|
||||
* dependency from arch header files. Only to be included from
|
||||
* asm/percpu.h.
|
||||
*
|
||||
* This file includes macros necessary to declare percpu sections and
|
||||
* variables, and definitions of percpu accessors and operations. It
|
||||
* should provide enough percpu features to arch header files even when
|
||||
* they can only include asm/percpu.h to avoid cyclic inclusion dependency.
|
||||
*/
|
||||
|
||||
#ifndef _LINUX_PERCPU_DEFS_H
|
||||
#define _LINUX_PERCPU_DEFS_H
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
|
||||
#ifdef MODULE
|
||||
#define PER_CPU_SHARED_ALIGNED_SECTION ""
|
||||
#define PER_CPU_ALIGNED_SECTION ""
|
||||
#else
|
||||
#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
|
||||
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
|
||||
#endif
|
||||
#define PER_CPU_FIRST_SECTION "..first"
|
||||
|
||||
#else
|
||||
|
||||
#define PER_CPU_SHARED_ALIGNED_SECTION ""
|
||||
#define PER_CPU_ALIGNED_SECTION "..shared_aligned"
|
||||
#define PER_CPU_FIRST_SECTION ""
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Base implementations of per-CPU variable declarations and definitions, where
|
||||
* the section in which the variable is to be placed is provided by the
|
||||
|
|
@ -18,19 +52,6 @@
|
|||
#define __PCPU_DUMMY_ATTRS \
|
||||
__attribute__((section(".discard"), unused))
|
||||
|
||||
/*
|
||||
* Macro which verifies @ptr is a percpu pointer without evaluating
|
||||
* @ptr. This is to be used in percpu accessors to verify that the
|
||||
* input parameter is a percpu pointer.
|
||||
*
|
||||
* + 0 is required in order to convert the pointer type from a
|
||||
* potential array type to a pointer to a single item of the array.
|
||||
*/
|
||||
#define __verify_pcpu_ptr(ptr) do { \
|
||||
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
|
||||
(void)__vpp_verify; \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* s390 and alpha modules require percpu variables to be defined as
|
||||
* weak to force the compiler to generate GOT based external
|
||||
|
|
@ -164,4 +185,337 @@
|
|||
#define EXPORT_PER_CPU_SYMBOL_GPL(var)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Accessors and operations.
|
||||
*/
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
/*
|
||||
* __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
|
||||
* @ptr and is invoked once before a percpu area is accessed by all
|
||||
* accessors and operations. This is performed in the generic part of
|
||||
* percpu and arch overrides don't need to worry about it; however, if an
|
||||
* arch wants to implement an arch-specific percpu accessor or operation,
|
||||
* it may use __verify_pcpu_ptr() to verify the parameters.
|
||||
*
|
||||
* + 0 is required in order to convert the pointer type from a
|
||||
* potential array type to a pointer to a single item of the array.
|
||||
*/
|
||||
#define __verify_pcpu_ptr(ptr) \
|
||||
do { \
|
||||
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
|
||||
(void)__vpp_verify; \
|
||||
} while (0)
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
|
||||
/*
|
||||
* Add an offset to a pointer but keep the pointer as-is. Use RELOC_HIDE()
|
||||
* to prevent the compiler from making incorrect assumptions about the
|
||||
* pointer value. The weird cast keeps both GCC and sparse happy.
|
||||
*/
|
||||
#define SHIFT_PERCPU_PTR(__p, __offset) \
|
||||
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
|
||||
|
||||
#define per_cpu_ptr(ptr, cpu) \
|
||||
({ \
|
||||
__verify_pcpu_ptr(ptr); \
|
||||
SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu))); \
|
||||
})
|
||||
|
||||
#define raw_cpu_ptr(ptr) \
|
||||
({ \
|
||||
__verify_pcpu_ptr(ptr); \
|
||||
arch_raw_cpu_ptr(ptr); \
|
||||
})
|
||||
|
||||
#ifdef CONFIG_DEBUG_PREEMPT
|
||||
#define this_cpu_ptr(ptr) \
|
||||
({ \
|
||||
__verify_pcpu_ptr(ptr); \
|
||||
SHIFT_PERCPU_PTR(ptr, my_cpu_offset); \
|
||||
})
|
||||
#else
|
||||
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
|
||||
#endif
|
||||
|
||||
#else /* CONFIG_SMP */
|
||||
|
||||
#define VERIFY_PERCPU_PTR(__p) \
|
||||
({ \
|
||||
__verify_pcpu_ptr(__p); \
|
||||
(typeof(*(__p)) __kernel __force *)(__p); \
|
||||
})
|
||||
|
||||
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
|
||||
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
|
||||
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
|
||||
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
#define per_cpu(var, cpu) (*per_cpu_ptr(&(var), cpu))
|
||||
#define __raw_get_cpu_var(var) (*raw_cpu_ptr(&(var)))
|
||||
#define __get_cpu_var(var) (*this_cpu_ptr(&(var)))
|
||||
|
||||
/* keep until we have removed all uses of __this_cpu_ptr */
|
||||
#define __this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
|
||||
|
||||
/*
|
||||
* Must be an lvalue. Since @var must be a simple identifier,
|
||||
* we force a syntax error here if it isn't.
|
||||
*/
|
||||
#define get_cpu_var(var) \
|
||||
(*({ \
|
||||
preempt_disable(); \
|
||||
this_cpu_ptr(&var); \
|
||||
}))
|
||||
|
||||
/*
|
||||
* The weird & is necessary because sparse considers (void)(var) to be
|
||||
* a direct dereference of percpu variable (var).
|
||||
*/
|
||||
#define put_cpu_var(var) \
|
||||
do { \
|
||||
(void)&(var); \
|
||||
preempt_enable(); \
|
||||
} while (0)
|
||||
|
||||
#define get_cpu_ptr(var) \
|
||||
({ \
|
||||
preempt_disable(); \
|
||||
this_cpu_ptr(var); \
|
||||
})
|
||||
|
||||
#define put_cpu_ptr(var) \
|
||||
do { \
|
||||
(void)(var); \
|
||||
preempt_enable(); \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* Branching function to split up a function into a set of functions that
|
||||
* are called for different scalar sizes of the objects handled.
|
||||
*/
|
||||
|
||||
extern void __bad_size_call_parameter(void);
|
||||
|
||||
#ifdef CONFIG_DEBUG_PREEMPT
|
||||
extern void __this_cpu_preempt_check(const char *op);
|
||||
#else
|
||||
static inline void __this_cpu_preempt_check(const char *op) { }
|
||||
#endif
|
||||
|
||||
#define __pcpu_size_call_return(stem, variable) \
|
||||
({ \
|
||||
typeof(variable) pscr_ret__; \
|
||||
__verify_pcpu_ptr(&(variable)); \
|
||||
switch(sizeof(variable)) { \
|
||||
case 1: pscr_ret__ = stem##1(variable); break; \
|
||||
case 2: pscr_ret__ = stem##2(variable); break; \
|
||||
case 4: pscr_ret__ = stem##4(variable); break; \
|
||||
case 8: pscr_ret__ = stem##8(variable); break; \
|
||||
default: \
|
||||
__bad_size_call_parameter(); break; \
|
||||
} \
|
||||
pscr_ret__; \
|
||||
})
|
||||
|
||||
#define __pcpu_size_call_return2(stem, variable, ...) \
|
||||
({ \
|
||||
typeof(variable) pscr2_ret__; \
|
||||
__verify_pcpu_ptr(&(variable)); \
|
||||
switch(sizeof(variable)) { \
|
||||
case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
|
||||
case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
|
||||
case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
|
||||
case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
|
||||
default: \
|
||||
__bad_size_call_parameter(); break; \
|
||||
} \
|
||||
pscr2_ret__; \
|
||||
})
|
||||
|
||||
/*
|
||||
* Special handling for cmpxchg_double. cmpxchg_double is passed two
|
||||
* percpu variables. The first has to be aligned to a double word
|
||||
* boundary and the second has to follow directly thereafter.
|
||||
* We enforce this on all architectures even if they don't support
|
||||
* a double cmpxchg instruction, since it's a cheap requirement, and it
|
||||
* avoids breaking the requirement for architectures with the instruction.
|
||||
*/
|
||||
#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
|
||||
({ \
|
||||
bool pdcrb_ret__; \
|
||||
__verify_pcpu_ptr(&(pcp1)); \
|
||||
BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
|
||||
VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \
|
||||
VM_BUG_ON((unsigned long)(&(pcp2)) != \
|
||||
(unsigned long)(&(pcp1)) + sizeof(pcp1)); \
|
||||
switch(sizeof(pcp1)) { \
|
||||
case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
default: \
|
||||
__bad_size_call_parameter(); break; \
|
||||
} \
|
||||
pdcrb_ret__; \
|
||||
})
|
||||
|
||||
#define __pcpu_size_call(stem, variable, ...) \
|
||||
do { \
|
||||
__verify_pcpu_ptr(&(variable)); \
|
||||
switch(sizeof(variable)) { \
|
||||
case 1: stem##1(variable, __VA_ARGS__);break; \
|
||||
case 2: stem##2(variable, __VA_ARGS__);break; \
|
||||
case 4: stem##4(variable, __VA_ARGS__);break; \
|
||||
case 8: stem##8(variable, __VA_ARGS__);break; \
|
||||
default: \
|
||||
__bad_size_call_parameter();break; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
|
||||
*
|
||||
* Optimized manipulation for memory allocated through the per cpu
|
||||
* allocator or for addresses of per cpu variables.
|
||||
*
|
||||
* These operation guarantee exclusivity of access for other operations
|
||||
* on the *same* processor. The assumption is that per cpu data is only
|
||||
* accessed by a single processor instance (the current one).
|
||||
*
|
||||
* The arch code can provide optimized implementation by defining macros
|
||||
* for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
|
||||
* cpu atomic operations for 2 byte sized RMW actions. If arch code does
|
||||
* not provide operations for a scalar size then the fallback in the
|
||||
* generic code will be used.
|
||||
*
|
||||
* cmpxchg_double replaces two adjacent scalars at once. The first two
|
||||
* parameters are per cpu variables which have to be of the same size. A
|
||||
* truth value is returned to indicate success or failure (since a double
|
||||
* register result is difficult to handle). There is very limited hardware
|
||||
* support for these operations, so only certain sizes may work.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Operations for contexts where we do not want to do any checks for
|
||||
* preemptions. Unless strictly necessary, always use [__]this_cpu_*()
|
||||
* instead.
|
||||
*
|
||||
* If there is no other protection through preempt disable and/or disabling
|
||||
* interupts then one of these RMW operations can show unexpected behavior
|
||||
* because the execution thread was rescheduled on another processor or an
|
||||
* interrupt occurred and the same percpu variable was modified from the
|
||||
* interrupt context.
|
||||
*/
|
||||
#define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp)
|
||||
#define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val)
|
||||
#define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val)
|
||||
#define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val)
|
||||
#define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val)
|
||||
#define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
|
||||
#define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
|
||||
#define raw_cpu_cmpxchg(pcp, oval, nval) \
|
||||
__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
|
||||
#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
|
||||
#define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val))
|
||||
#define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1)
|
||||
#define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1)
|
||||
#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
|
||||
#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
|
||||
#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
|
||||
|
||||
/*
|
||||
* Operations for contexts that are safe from preemption/interrupts. These
|
||||
* operations verify that preemption is disabled.
|
||||
*/
|
||||
#define __this_cpu_read(pcp) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("read"); \
|
||||
raw_cpu_read(pcp); \
|
||||
})
|
||||
|
||||
#define __this_cpu_write(pcp, val) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("write"); \
|
||||
raw_cpu_write(pcp, val); \
|
||||
})
|
||||
|
||||
#define __this_cpu_add(pcp, val) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("add"); \
|
||||
raw_cpu_add(pcp, val); \
|
||||
})
|
||||
|
||||
#define __this_cpu_and(pcp, val) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("and"); \
|
||||
raw_cpu_and(pcp, val); \
|
||||
})
|
||||
|
||||
#define __this_cpu_or(pcp, val) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("or"); \
|
||||
raw_cpu_or(pcp, val); \
|
||||
})
|
||||
|
||||
#define __this_cpu_add_return(pcp, val) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("add_return"); \
|
||||
raw_cpu_add_return(pcp, val); \
|
||||
})
|
||||
|
||||
#define __this_cpu_xchg(pcp, nval) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("xchg"); \
|
||||
raw_cpu_xchg(pcp, nval); \
|
||||
})
|
||||
|
||||
#define __this_cpu_cmpxchg(pcp, oval, nval) \
|
||||
({ \
|
||||
__this_cpu_preempt_check("cmpxchg"); \
|
||||
raw_cpu_cmpxchg(pcp, oval, nval); \
|
||||
})
|
||||
|
||||
#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
({ __this_cpu_preempt_check("cmpxchg_double"); \
|
||||
raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \
|
||||
})
|
||||
|
||||
#define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val))
|
||||
#define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1)
|
||||
#define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1)
|
||||
#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
|
||||
#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
|
||||
#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
|
||||
|
||||
/*
|
||||
* Operations with implied preemption protection. These operations can be
|
||||
* used without worrying about preemption. Note that interrupts may still
|
||||
* occur while an operation is in progress and if the interrupt modifies
|
||||
* the variable too then RMW actions may not be reliable.
|
||||
*/
|
||||
#define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, pcp)
|
||||
#define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, pcp, val)
|
||||
#define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, pcp, val)
|
||||
#define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, pcp, val)
|
||||
#define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, pcp, val)
|
||||
#define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
|
||||
#define this_cpu_xchg(pcp, nval) __pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
|
||||
#define this_cpu_cmpxchg(pcp, oval, nval) \
|
||||
__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
|
||||
#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
|
||||
#define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val))
|
||||
#define this_cpu_inc(pcp) this_cpu_add(pcp, 1)
|
||||
#define this_cpu_dec(pcp) this_cpu_sub(pcp, 1)
|
||||
#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
|
||||
#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
|
||||
#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
#endif /* _LINUX_PERCPU_DEFS_H */
|
||||
|
|
|
|||
|
|
@ -57,11 +57,9 @@ struct percpu_ref {
|
|||
atomic_t count;
|
||||
/*
|
||||
* The low bit of the pointer indicates whether the ref is in percpu
|
||||
* mode; if set, then get/put will manipulate the atomic_t (this is a
|
||||
* hack because we need to keep the pointer around for
|
||||
* percpu_ref_kill_rcu())
|
||||
* mode; if set, then get/put will manipulate the atomic_t.
|
||||
*/
|
||||
unsigned __percpu *pcpu_count;
|
||||
unsigned long pcpu_count_ptr;
|
||||
percpu_ref_func_t *release;
|
||||
percpu_ref_func_t *confirm_kill;
|
||||
struct rcu_head rcu;
|
||||
|
|
@ -69,7 +67,8 @@ struct percpu_ref {
|
|||
|
||||
int __must_check percpu_ref_init(struct percpu_ref *ref,
|
||||
percpu_ref_func_t *release);
|
||||
void percpu_ref_cancel_init(struct percpu_ref *ref);
|
||||
void percpu_ref_reinit(struct percpu_ref *ref);
|
||||
void percpu_ref_exit(struct percpu_ref *ref);
|
||||
void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
|
||||
percpu_ref_func_t *confirm_kill);
|
||||
|
||||
|
|
@ -88,12 +87,28 @@ static inline void percpu_ref_kill(struct percpu_ref *ref)
|
|||
return percpu_ref_kill_and_confirm(ref, NULL);
|
||||
}
|
||||
|
||||
#define PCPU_STATUS_BITS 2
|
||||
#define PCPU_STATUS_MASK ((1 << PCPU_STATUS_BITS) - 1)
|
||||
#define PCPU_REF_PTR 0
|
||||
#define PCPU_REF_DEAD 1
|
||||
|
||||
#define REF_STATUS(count) (((unsigned long) count) & PCPU_STATUS_MASK)
|
||||
/*
|
||||
* Internal helper. Don't use outside percpu-refcount proper. The
|
||||
* function doesn't return the pointer and let the caller test it for NULL
|
||||
* because doing so forces the compiler to generate two conditional
|
||||
* branches as it can't assume that @ref->pcpu_count is not NULL.
|
||||
*/
|
||||
static inline bool __pcpu_ref_alive(struct percpu_ref *ref,
|
||||
unsigned __percpu **pcpu_countp)
|
||||
{
|
||||
unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr);
|
||||
|
||||
/* paired with smp_store_release() in percpu_ref_reinit() */
|
||||
smp_read_barrier_depends();
|
||||
|
||||
if (unlikely(pcpu_ptr & PCPU_REF_DEAD))
|
||||
return false;
|
||||
|
||||
*pcpu_countp = (unsigned __percpu *)pcpu_ptr;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* percpu_ref_get - increment a percpu refcount
|
||||
|
|
@ -107,9 +122,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref)
|
|||
|
||||
rcu_read_lock_sched();
|
||||
|
||||
pcpu_count = ACCESS_ONCE(ref->pcpu_count);
|
||||
|
||||
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
|
||||
if (__pcpu_ref_alive(ref, &pcpu_count))
|
||||
this_cpu_inc(*pcpu_count);
|
||||
else
|
||||
atomic_inc(&ref->count);
|
||||
|
|
@ -133,9 +146,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref)
|
|||
|
||||
rcu_read_lock_sched();
|
||||
|
||||
pcpu_count = ACCESS_ONCE(ref->pcpu_count);
|
||||
|
||||
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
|
||||
if (__pcpu_ref_alive(ref, &pcpu_count)) {
|
||||
this_cpu_inc(*pcpu_count);
|
||||
ret = true;
|
||||
} else {
|
||||
|
|
@ -168,9 +179,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref)
|
|||
|
||||
rcu_read_lock_sched();
|
||||
|
||||
pcpu_count = ACCESS_ONCE(ref->pcpu_count);
|
||||
|
||||
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) {
|
||||
if (__pcpu_ref_alive(ref, &pcpu_count)) {
|
||||
this_cpu_inc(*pcpu_count);
|
||||
ret = true;
|
||||
}
|
||||
|
|
@ -193,9 +202,7 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
|
|||
|
||||
rcu_read_lock_sched();
|
||||
|
||||
pcpu_count = ACCESS_ONCE(ref->pcpu_count);
|
||||
|
||||
if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR))
|
||||
if (__pcpu_ref_alive(ref, &pcpu_count))
|
||||
this_cpu_dec(*pcpu_count);
|
||||
else if (unlikely(atomic_dec_and_test(&ref->count)))
|
||||
ref->release(ref);
|
||||
|
|
@ -203,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref)
|
|||
rcu_read_unlock_sched();
|
||||
}
|
||||
|
||||
/**
|
||||
* percpu_ref_is_zero - test whether a percpu refcount reached zero
|
||||
* @ref: percpu_ref to test
|
||||
*
|
||||
* Returns %true if @ref reached zero.
|
||||
*/
|
||||
static inline bool percpu_ref_is_zero(struct percpu_ref *ref)
|
||||
{
|
||||
unsigned __percpu *pcpu_count;
|
||||
|
||||
if (__pcpu_ref_alive(ref, &pcpu_count))
|
||||
return false;
|
||||
return !atomic_read(&ref->count);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -23,32 +23,6 @@
|
|||
PERCPU_MODULE_RESERVE)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Must be an lvalue. Since @var must be a simple identifier,
|
||||
* we force a syntax error here if it isn't.
|
||||
*/
|
||||
#define get_cpu_var(var) (*({ \
|
||||
preempt_disable(); \
|
||||
this_cpu_ptr(&var); }))
|
||||
|
||||
/*
|
||||
* The weird & is necessary because sparse considers (void)(var) to be
|
||||
* a direct dereference of percpu variable (var).
|
||||
*/
|
||||
#define put_cpu_var(var) do { \
|
||||
(void)&(var); \
|
||||
preempt_enable(); \
|
||||
} while (0)
|
||||
|
||||
#define get_cpu_ptr(var) ({ \
|
||||
preempt_disable(); \
|
||||
this_cpu_ptr(var); })
|
||||
|
||||
#define put_cpu_ptr(var) do { \
|
||||
(void)(var); \
|
||||
preempt_enable(); \
|
||||
} while (0)
|
||||
|
||||
/* minimum unit size, also is the maximum supported allocation size */
|
||||
#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10)
|
||||
|
||||
|
|
@ -140,17 +114,6 @@ extern int __init pcpu_page_first_chunk(size_t reserved_size,
|
|||
pcpu_fc_populate_pte_fn_t populate_pte_fn);
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Use this to get to a cpu's version of the per-cpu object
|
||||
* dynamically allocated. Non-atomic access to the current CPU's
|
||||
* version should probably be combined with get_cpu()/put_cpu().
|
||||
*/
|
||||
#ifdef CONFIG_SMP
|
||||
#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
|
||||
#else
|
||||
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR((ptr)); })
|
||||
#endif
|
||||
|
||||
extern void __percpu *__alloc_reserved_percpu(size_t size, size_t align);
|
||||
extern bool is_kernel_percpu_address(unsigned long addr);
|
||||
|
||||
|
|
@ -166,640 +129,4 @@ extern phys_addr_t per_cpu_ptr_to_phys(void *addr);
|
|||
#define alloc_percpu(type) \
|
||||
(typeof(type) __percpu *)__alloc_percpu(sizeof(type), __alignof__(type))
|
||||
|
||||
/*
|
||||
* Branching function to split up a function into a set of functions that
|
||||
* are called for different scalar sizes of the objects handled.
|
||||
*/
|
||||
|
||||
extern void __bad_size_call_parameter(void);
|
||||
|
||||
#ifdef CONFIG_DEBUG_PREEMPT
|
||||
extern void __this_cpu_preempt_check(const char *op);
|
||||
#else
|
||||
static inline void __this_cpu_preempt_check(const char *op) { }
|
||||
#endif
|
||||
|
||||
#define __pcpu_size_call_return(stem, variable) \
|
||||
({ typeof(variable) pscr_ret__; \
|
||||
__verify_pcpu_ptr(&(variable)); \
|
||||
switch(sizeof(variable)) { \
|
||||
case 1: pscr_ret__ = stem##1(variable);break; \
|
||||
case 2: pscr_ret__ = stem##2(variable);break; \
|
||||
case 4: pscr_ret__ = stem##4(variable);break; \
|
||||
case 8: pscr_ret__ = stem##8(variable);break; \
|
||||
default: \
|
||||
__bad_size_call_parameter();break; \
|
||||
} \
|
||||
pscr_ret__; \
|
||||
})
|
||||
|
||||
#define __pcpu_size_call_return2(stem, variable, ...) \
|
||||
({ \
|
||||
typeof(variable) pscr2_ret__; \
|
||||
__verify_pcpu_ptr(&(variable)); \
|
||||
switch(sizeof(variable)) { \
|
||||
case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \
|
||||
case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \
|
||||
case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \
|
||||
case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \
|
||||
default: \
|
||||
__bad_size_call_parameter(); break; \
|
||||
} \
|
||||
pscr2_ret__; \
|
||||
})
|
||||
|
||||
/*
|
||||
* Special handling for cmpxchg_double. cmpxchg_double is passed two
|
||||
* percpu variables. The first has to be aligned to a double word
|
||||
* boundary and the second has to follow directly thereafter.
|
||||
* We enforce this on all architectures even if they don't support
|
||||
* a double cmpxchg instruction, since it's a cheap requirement, and it
|
||||
* avoids breaking the requirement for architectures with the instruction.
|
||||
*/
|
||||
#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \
|
||||
({ \
|
||||
bool pdcrb_ret__; \
|
||||
__verify_pcpu_ptr(&pcp1); \
|
||||
BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \
|
||||
VM_BUG_ON((unsigned long)(&pcp1) % (2 * sizeof(pcp1))); \
|
||||
VM_BUG_ON((unsigned long)(&pcp2) != \
|
||||
(unsigned long)(&pcp1) + sizeof(pcp1)); \
|
||||
switch(sizeof(pcp1)) { \
|
||||
case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \
|
||||
default: \
|
||||
__bad_size_call_parameter(); break; \
|
||||
} \
|
||||
pdcrb_ret__; \
|
||||
})
|
||||
|
||||
#define __pcpu_size_call(stem, variable, ...) \
|
||||
do { \
|
||||
__verify_pcpu_ptr(&(variable)); \
|
||||
switch(sizeof(variable)) { \
|
||||
case 1: stem##1(variable, __VA_ARGS__);break; \
|
||||
case 2: stem##2(variable, __VA_ARGS__);break; \
|
||||
case 4: stem##4(variable, __VA_ARGS__);break; \
|
||||
case 8: stem##8(variable, __VA_ARGS__);break; \
|
||||
default: \
|
||||
__bad_size_call_parameter();break; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
|
||||
*
|
||||
* Optimized manipulation for memory allocated through the per cpu
|
||||
* allocator or for addresses of per cpu variables.
|
||||
*
|
||||
* These operation guarantee exclusivity of access for other operations
|
||||
* on the *same* processor. The assumption is that per cpu data is only
|
||||
* accessed by a single processor instance (the current one).
|
||||
*
|
||||
* The first group is used for accesses that must be done in a
|
||||
* preemption safe way since we know that the context is not preempt
|
||||
* safe. Interrupts may occur. If the interrupt modifies the variable
|
||||
* too then RMW actions will not be reliable.
|
||||
*
|
||||
* The arch code can provide optimized functions in two ways:
|
||||
*
|
||||
* 1. Override the function completely. F.e. define this_cpu_add().
|
||||
* The arch must then ensure that the various scalar format passed
|
||||
* are handled correctly.
|
||||
*
|
||||
* 2. Provide functions for certain scalar sizes. F.e. provide
|
||||
* this_cpu_add_2() to provide per cpu atomic operations for 2 byte
|
||||
* sized RMW actions. If arch code does not provide operations for
|
||||
* a scalar size then the fallback in the generic code will be
|
||||
* used.
|
||||
*/
|
||||
|
||||
#define _this_cpu_generic_read(pcp) \
|
||||
({ typeof(pcp) ret__; \
|
||||
preempt_disable(); \
|
||||
ret__ = *this_cpu_ptr(&(pcp)); \
|
||||
preempt_enable(); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef this_cpu_read
|
||||
# ifndef this_cpu_read_1
|
||||
# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
|
||||
# endif
|
||||
# ifndef this_cpu_read_2
|
||||
# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
|
||||
# endif
|
||||
# ifndef this_cpu_read_4
|
||||
# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
|
||||
# endif
|
||||
# ifndef this_cpu_read_8
|
||||
# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
|
||||
# endif
|
||||
# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
|
||||
#endif
|
||||
|
||||
#define _this_cpu_generic_to_op(pcp, val, op) \
|
||||
do { \
|
||||
unsigned long flags; \
|
||||
raw_local_irq_save(flags); \
|
||||
*raw_cpu_ptr(&(pcp)) op val; \
|
||||
raw_local_irq_restore(flags); \
|
||||
} while (0)
|
||||
|
||||
#ifndef this_cpu_write
|
||||
# ifndef this_cpu_write_1
|
||||
# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# ifndef this_cpu_write_2
|
||||
# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# ifndef this_cpu_write_4
|
||||
# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# ifndef this_cpu_write_8
|
||||
# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_add
|
||||
# ifndef this_cpu_add_1
|
||||
# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# ifndef this_cpu_add_2
|
||||
# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# ifndef this_cpu_add_4
|
||||
# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# ifndef this_cpu_add_8
|
||||
# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_sub
|
||||
# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(typeof(pcp))(val))
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_inc
|
||||
# define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_dec
|
||||
# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_and
|
||||
# ifndef this_cpu_and_1
|
||||
# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# ifndef this_cpu_and_2
|
||||
# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# ifndef this_cpu_and_4
|
||||
# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# ifndef this_cpu_and_8
|
||||
# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#ifndef this_cpu_or
|
||||
# ifndef this_cpu_or_1
|
||||
# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# ifndef this_cpu_or_2
|
||||
# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# ifndef this_cpu_or_4
|
||||
# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# ifndef this_cpu_or_8
|
||||
# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#define _this_cpu_generic_add_return(pcp, val) \
|
||||
({ \
|
||||
typeof(pcp) ret__; \
|
||||
unsigned long flags; \
|
||||
raw_local_irq_save(flags); \
|
||||
raw_cpu_add(pcp, val); \
|
||||
ret__ = raw_cpu_read(pcp); \
|
||||
raw_local_irq_restore(flags); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef this_cpu_add_return
|
||||
# ifndef this_cpu_add_return_1
|
||||
# define this_cpu_add_return_1(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# ifndef this_cpu_add_return_2
|
||||
# define this_cpu_add_return_2(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# ifndef this_cpu_add_return_4
|
||||
# define this_cpu_add_return_4(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# ifndef this_cpu_add_return_8
|
||||
# define this_cpu_add_return_8(pcp, val) _this_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# define this_cpu_add_return(pcp, val) __pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
|
||||
#endif
|
||||
|
||||
#define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val))
|
||||
#define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1)
|
||||
#define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1)
|
||||
|
||||
#define _this_cpu_generic_xchg(pcp, nval) \
|
||||
({ typeof(pcp) ret__; \
|
||||
unsigned long flags; \
|
||||
raw_local_irq_save(flags); \
|
||||
ret__ = raw_cpu_read(pcp); \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
raw_local_irq_restore(flags); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef this_cpu_xchg
|
||||
# ifndef this_cpu_xchg_1
|
||||
# define this_cpu_xchg_1(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# ifndef this_cpu_xchg_2
|
||||
# define this_cpu_xchg_2(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# ifndef this_cpu_xchg_4
|
||||
# define this_cpu_xchg_4(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# ifndef this_cpu_xchg_8
|
||||
# define this_cpu_xchg_8(pcp, nval) _this_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# define this_cpu_xchg(pcp, nval) \
|
||||
__pcpu_size_call_return2(this_cpu_xchg_, (pcp), nval)
|
||||
#endif
|
||||
|
||||
#define _this_cpu_generic_cmpxchg(pcp, oval, nval) \
|
||||
({ \
|
||||
typeof(pcp) ret__; \
|
||||
unsigned long flags; \
|
||||
raw_local_irq_save(flags); \
|
||||
ret__ = raw_cpu_read(pcp); \
|
||||
if (ret__ == (oval)) \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
raw_local_irq_restore(flags); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef this_cpu_cmpxchg
|
||||
# ifndef this_cpu_cmpxchg_1
|
||||
# define this_cpu_cmpxchg_1(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# ifndef this_cpu_cmpxchg_2
|
||||
# define this_cpu_cmpxchg_2(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# ifndef this_cpu_cmpxchg_4
|
||||
# define this_cpu_cmpxchg_4(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# ifndef this_cpu_cmpxchg_8
|
||||
# define this_cpu_cmpxchg_8(pcp, oval, nval) _this_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# define this_cpu_cmpxchg(pcp, oval, nval) \
|
||||
__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* cmpxchg_double replaces two adjacent scalars at once. The first
|
||||
* two parameters are per cpu variables which have to be of the same
|
||||
* size. A truth value is returned to indicate success or failure
|
||||
* (since a double register result is difficult to handle). There is
|
||||
* very limited hardware support for these operations, so only certain
|
||||
* sizes may work.
|
||||
*/
|
||||
#define _this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
({ \
|
||||
int ret__; \
|
||||
unsigned long flags; \
|
||||
raw_local_irq_save(flags); \
|
||||
ret__ = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
|
||||
oval1, oval2, nval1, nval2); \
|
||||
raw_local_irq_restore(flags); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef this_cpu_cmpxchg_double
|
||||
# ifndef this_cpu_cmpxchg_double_1
|
||||
# define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# ifndef this_cpu_cmpxchg_double_2
|
||||
# define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# ifndef this_cpu_cmpxchg_double_4
|
||||
# define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# ifndef this_cpu_cmpxchg_double_8
|
||||
# define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
_this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Generic percpu operations for contexts where we do not want to do
|
||||
* any checks for preemptiosn.
|
||||
*
|
||||
* If there is no other protection through preempt disable and/or
|
||||
* disabling interupts then one of these RMW operations can show unexpected
|
||||
* behavior because the execution thread was rescheduled on another processor
|
||||
* or an interrupt occurred and the same percpu variable was modified from
|
||||
* the interrupt context.
|
||||
*/
|
||||
#ifndef raw_cpu_read
|
||||
# ifndef raw_cpu_read_1
|
||||
# define raw_cpu_read_1(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
# endif
|
||||
# ifndef raw_cpu_read_2
|
||||
# define raw_cpu_read_2(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
# endif
|
||||
# ifndef raw_cpu_read_4
|
||||
# define raw_cpu_read_4(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
# endif
|
||||
# ifndef raw_cpu_read_8
|
||||
# define raw_cpu_read_8(pcp) (*raw_cpu_ptr(&(pcp)))
|
||||
# endif
|
||||
# define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, (pcp))
|
||||
#endif
|
||||
|
||||
#define raw_cpu_generic_to_op(pcp, val, op) \
|
||||
do { \
|
||||
*raw_cpu_ptr(&(pcp)) op val; \
|
||||
} while (0)
|
||||
|
||||
|
||||
#ifndef raw_cpu_write
|
||||
# ifndef raw_cpu_write_1
|
||||
# define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# ifndef raw_cpu_write_2
|
||||
# define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# ifndef raw_cpu_write_4
|
||||
# define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# ifndef raw_cpu_write_8
|
||||
# define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), =)
|
||||
# endif
|
||||
# define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_add
|
||||
# ifndef raw_cpu_add_1
|
||||
# define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# ifndef raw_cpu_add_2
|
||||
# define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# ifndef raw_cpu_add_4
|
||||
# define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# ifndef raw_cpu_add_8
|
||||
# define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), +=)
|
||||
# endif
|
||||
# define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_sub
|
||||
# define raw_cpu_sub(pcp, val) raw_cpu_add((pcp), -(val))
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_inc
|
||||
# define raw_cpu_inc(pcp) raw_cpu_add((pcp), 1)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_dec
|
||||
# define raw_cpu_dec(pcp) raw_cpu_sub((pcp), 1)
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_and
|
||||
# ifndef raw_cpu_and_1
|
||||
# define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# ifndef raw_cpu_and_2
|
||||
# define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# ifndef raw_cpu_and_4
|
||||
# define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# ifndef raw_cpu_and_8
|
||||
# define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), &=)
|
||||
# endif
|
||||
# define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#ifndef raw_cpu_or
|
||||
# ifndef raw_cpu_or_1
|
||||
# define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# ifndef raw_cpu_or_2
|
||||
# define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# ifndef raw_cpu_or_4
|
||||
# define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# ifndef raw_cpu_or_8
|
||||
# define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op((pcp), (val), |=)
|
||||
# endif
|
||||
# define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, (pcp), (val))
|
||||
#endif
|
||||
|
||||
#define raw_cpu_generic_add_return(pcp, val) \
|
||||
({ \
|
||||
raw_cpu_add(pcp, val); \
|
||||
raw_cpu_read(pcp); \
|
||||
})
|
||||
|
||||
#ifndef raw_cpu_add_return
|
||||
# ifndef raw_cpu_add_return_1
|
||||
# define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# ifndef raw_cpu_add_return_2
|
||||
# define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# ifndef raw_cpu_add_return_4
|
||||
# define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# ifndef raw_cpu_add_return_8
|
||||
# define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
|
||||
# endif
|
||||
# define raw_cpu_add_return(pcp, val) \
|
||||
__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
|
||||
#endif
|
||||
|
||||
#define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val))
|
||||
#define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1)
|
||||
#define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1)
|
||||
|
||||
#define raw_cpu_generic_xchg(pcp, nval) \
|
||||
({ typeof(pcp) ret__; \
|
||||
ret__ = raw_cpu_read(pcp); \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef raw_cpu_xchg
|
||||
# ifndef raw_cpu_xchg_1
|
||||
# define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# ifndef raw_cpu_xchg_2
|
||||
# define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# ifndef raw_cpu_xchg_4
|
||||
# define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# ifndef raw_cpu_xchg_8
|
||||
# define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
|
||||
# endif
|
||||
# define raw_cpu_xchg(pcp, nval) \
|
||||
__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval)
|
||||
#endif
|
||||
|
||||
#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
|
||||
({ \
|
||||
typeof(pcp) ret__; \
|
||||
ret__ = raw_cpu_read(pcp); \
|
||||
if (ret__ == (oval)) \
|
||||
raw_cpu_write(pcp, nval); \
|
||||
ret__; \
|
||||
})
|
||||
|
||||
#ifndef raw_cpu_cmpxchg
|
||||
# ifndef raw_cpu_cmpxchg_1
|
||||
# define raw_cpu_cmpxchg_1(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# ifndef raw_cpu_cmpxchg_2
|
||||
# define raw_cpu_cmpxchg_2(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# ifndef raw_cpu_cmpxchg_4
|
||||
# define raw_cpu_cmpxchg_4(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# ifndef raw_cpu_cmpxchg_8
|
||||
# define raw_cpu_cmpxchg_8(pcp, oval, nval) raw_cpu_generic_cmpxchg(pcp, oval, nval)
|
||||
# endif
|
||||
# define raw_cpu_cmpxchg(pcp, oval, nval) \
|
||||
__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)
|
||||
#endif
|
||||
|
||||
#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
({ \
|
||||
int __ret = 0; \
|
||||
if (raw_cpu_read(pcp1) == (oval1) && \
|
||||
raw_cpu_read(pcp2) == (oval2)) { \
|
||||
raw_cpu_write(pcp1, (nval1)); \
|
||||
raw_cpu_write(pcp2, (nval2)); \
|
||||
__ret = 1; \
|
||||
} \
|
||||
(__ret); \
|
||||
})
|
||||
|
||||
#ifndef raw_cpu_cmpxchg_double
|
||||
# ifndef raw_cpu_cmpxchg_double_1
|
||||
# define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# ifndef raw_cpu_cmpxchg_double_2
|
||||
# define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# ifndef raw_cpu_cmpxchg_double_4
|
||||
# define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# ifndef raw_cpu_cmpxchg_double_8
|
||||
# define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
|
||||
# endif
|
||||
# define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2))
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Generic percpu operations for context that are safe from preemption/interrupts.
|
||||
*/
|
||||
#ifndef __this_cpu_read
|
||||
# define __this_cpu_read(pcp) \
|
||||
(__this_cpu_preempt_check("read"),__pcpu_size_call_return(raw_cpu_read_, (pcp)))
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_write
|
||||
# define __this_cpu_write(pcp, val) \
|
||||
do { __this_cpu_preempt_check("write"); \
|
||||
__pcpu_size_call(raw_cpu_write_, (pcp), (val)); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_add
|
||||
# define __this_cpu_add(pcp, val) \
|
||||
do { __this_cpu_preempt_check("add"); \
|
||||
__pcpu_size_call(raw_cpu_add_, (pcp), (val)); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_sub
|
||||
# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(typeof(pcp))(val))
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_inc
|
||||
# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_dec
|
||||
# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_and
|
||||
# define __this_cpu_and(pcp, val) \
|
||||
do { __this_cpu_preempt_check("and"); \
|
||||
__pcpu_size_call(raw_cpu_and_, (pcp), (val)); \
|
||||
} while (0)
|
||||
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_or
|
||||
# define __this_cpu_or(pcp, val) \
|
||||
do { __this_cpu_preempt_check("or"); \
|
||||
__pcpu_size_call(raw_cpu_or_, (pcp), (val)); \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_add_return
|
||||
# define __this_cpu_add_return(pcp, val) \
|
||||
(__this_cpu_preempt_check("add_return"),__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val))
|
||||
#endif
|
||||
|
||||
#define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val))
|
||||
#define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1)
|
||||
#define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1)
|
||||
|
||||
#ifndef __this_cpu_xchg
|
||||
# define __this_cpu_xchg(pcp, nval) \
|
||||
(__this_cpu_preempt_check("xchg"),__pcpu_size_call_return2(raw_cpu_xchg_, (pcp), nval))
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_cmpxchg
|
||||
# define __this_cpu_cmpxchg(pcp, oval, nval) \
|
||||
(__this_cpu_preempt_check("cmpxchg"),__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval))
|
||||
#endif
|
||||
|
||||
#ifndef __this_cpu_cmpxchg_double
|
||||
# define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
|
||||
(__this_cpu_preempt_check("cmpxchg_double"),__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, (pcp1), (pcp2), (oval1), (oval2), (nval1), (nval2)))
|
||||
#endif
|
||||
|
||||
#endif /* __LINUX_PERCPU_H */
|
||||
|
|
|
|||
|
|
@ -1638,7 +1638,7 @@ static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask)
|
|||
exit_root_id:
|
||||
cgroup_exit_root_id(root);
|
||||
cancel_ref:
|
||||
percpu_ref_cancel_init(&root_cgrp->self.refcnt);
|
||||
percpu_ref_exit(&root_cgrp->self.refcnt);
|
||||
out:
|
||||
free_cgrp_cset_links(&tmp_links);
|
||||
return ret;
|
||||
|
|
@ -4133,6 +4133,8 @@ static void css_free_work_fn(struct work_struct *work)
|
|||
container_of(work, struct cgroup_subsys_state, destroy_work);
|
||||
struct cgroup *cgrp = css->cgroup;
|
||||
|
||||
percpu_ref_exit(&css->refcnt);
|
||||
|
||||
if (css->ss) {
|
||||
/* css free path */
|
||||
if (css->parent)
|
||||
|
|
@ -4330,7 +4332,7 @@ static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss)
|
|||
err_free_id:
|
||||
cgroup_idr_remove(&ss->css_idr, css->id);
|
||||
err_free_percpu_ref:
|
||||
percpu_ref_cancel_init(&css->refcnt);
|
||||
percpu_ref_exit(&css->refcnt);
|
||||
err_free_css:
|
||||
call_rcu(&css->rcu_head, css_free_rcu_fn);
|
||||
return err;
|
||||
|
|
@ -4441,7 +4443,7 @@ static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name,
|
|||
out_free_id:
|
||||
cgroup_idr_remove(&root->cgroup_idr, cgrp->id);
|
||||
out_cancel_ref:
|
||||
percpu_ref_cancel_init(&cgrp->self.refcnt);
|
||||
percpu_ref_exit(&cgrp->self.refcnt);
|
||||
out_free_cgrp:
|
||||
kfree(cgrp);
|
||||
out_unlock:
|
||||
|
|
|
|||
|
|
@ -2020,13 +2020,8 @@ __acquires(&pool->lock)
|
|||
|
||||
lockdep_copy_map(&lockdep_map, &work->lockdep_map);
|
||||
#endif
|
||||
/*
|
||||
* Ensure we're on the correct CPU. DISASSOCIATED test is
|
||||
* necessary to avoid spurious warnings from rescuers servicing the
|
||||
* unbound or a disassociated pool.
|
||||
*/
|
||||
WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
|
||||
!(pool->flags & POOL_DISASSOCIATED) &&
|
||||
/* ensure we're on the correct CPU */
|
||||
WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
|
||||
raw_smp_processor_id() != pool->cpu);
|
||||
|
||||
/*
|
||||
|
|
@ -4542,6 +4537,7 @@ static void rebind_workers(struct worker_pool *pool)
|
|||
pool->attrs->cpumask) < 0);
|
||||
|
||||
spin_lock_irq(&pool->lock);
|
||||
pool->flags &= ~POOL_DISASSOCIATED;
|
||||
|
||||
for_each_pool_worker(worker, pool) {
|
||||
unsigned int worker_flags = worker->flags;
|
||||
|
|
@ -4643,15 +4639,10 @@ static int workqueue_cpu_up_callback(struct notifier_block *nfb,
|
|||
for_each_pool(pool, pi) {
|
||||
mutex_lock(&pool->attach_mutex);
|
||||
|
||||
if (pool->cpu == cpu) {
|
||||
spin_lock_irq(&pool->lock);
|
||||
pool->flags &= ~POOL_DISASSOCIATED;
|
||||
spin_unlock_irq(&pool->lock);
|
||||
|
||||
if (pool->cpu == cpu)
|
||||
rebind_workers(pool);
|
||||
} else if (pool->cpu < 0) {
|
||||
else if (pool->cpu < 0)
|
||||
restore_unbound_workers_cpumask(pool, cpu);
|
||||
}
|
||||
|
||||
mutex_unlock(&pool->attach_mutex);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,6 +31,11 @@
|
|||
|
||||
#define PCPU_COUNT_BIAS (1U << 31)
|
||||
|
||||
static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref)
|
||||
{
|
||||
return (unsigned __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
|
||||
}
|
||||
|
||||
/**
|
||||
* percpu_ref_init - initialize a percpu refcount
|
||||
* @ref: percpu_ref to initialize
|
||||
|
|
@ -46,8 +51,8 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)
|
|||
{
|
||||
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
|
||||
|
||||
ref->pcpu_count = alloc_percpu(unsigned);
|
||||
if (!ref->pcpu_count)
|
||||
ref->pcpu_count_ptr = (unsigned long)alloc_percpu(unsigned);
|
||||
if (!ref->pcpu_count_ptr)
|
||||
return -ENOMEM;
|
||||
|
||||
ref->release = release;
|
||||
|
|
@ -56,53 +61,71 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release)
|
|||
EXPORT_SYMBOL_GPL(percpu_ref_init);
|
||||
|
||||
/**
|
||||
* percpu_ref_cancel_init - cancel percpu_ref_init()
|
||||
* @ref: percpu_ref to cancel init for
|
||||
* percpu_ref_reinit - re-initialize a percpu refcount
|
||||
* @ref: perpcu_ref to re-initialize
|
||||
*
|
||||
* Once a percpu_ref is initialized, its destruction is initiated by
|
||||
* percpu_ref_kill() and completes asynchronously, which can be painful to
|
||||
* do when destroying a half-constructed object in init failure path.
|
||||
* Re-initialize @ref so that it's in the same state as when it finished
|
||||
* percpu_ref_init(). @ref must have been initialized successfully, killed
|
||||
* and reached 0 but not exited.
|
||||
*
|
||||
* This function destroys @ref without invoking @ref->release and the
|
||||
* memory area containing it can be freed immediately on return. To
|
||||
* prevent accidental misuse, it's required that @ref has finished
|
||||
* percpu_ref_init(), whether successful or not, but never used.
|
||||
*
|
||||
* The weird name and usage restriction are to prevent people from using
|
||||
* this function by mistake for normal shutdown instead of
|
||||
* percpu_ref_kill().
|
||||
* Note that percpu_ref_tryget[_live]() are safe to perform on @ref while
|
||||
* this function is in progress.
|
||||
*/
|
||||
void percpu_ref_cancel_init(struct percpu_ref *ref)
|
||||
void percpu_ref_reinit(struct percpu_ref *ref)
|
||||
{
|
||||
unsigned __percpu *pcpu_count = ref->pcpu_count;
|
||||
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
|
||||
int cpu;
|
||||
|
||||
WARN_ON_ONCE(atomic_read(&ref->count) != 1 + PCPU_COUNT_BIAS);
|
||||
BUG_ON(!pcpu_count);
|
||||
WARN_ON(!percpu_ref_is_zero(ref));
|
||||
|
||||
atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS);
|
||||
|
||||
/*
|
||||
* Restore per-cpu operation. smp_store_release() is paired with
|
||||
* smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees
|
||||
* that the zeroing is visible to all percpu accesses which can see
|
||||
* the following PCPU_REF_DEAD clearing.
|
||||
*/
|
||||
for_each_possible_cpu(cpu)
|
||||
*per_cpu_ptr(pcpu_count, cpu) = 0;
|
||||
|
||||
smp_store_release(&ref->pcpu_count_ptr,
|
||||
ref->pcpu_count_ptr & ~PCPU_REF_DEAD);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(percpu_ref_reinit);
|
||||
|
||||
/**
|
||||
* percpu_ref_exit - undo percpu_ref_init()
|
||||
* @ref: percpu_ref to exit
|
||||
*
|
||||
* This function exits @ref. The caller is responsible for ensuring that
|
||||
* @ref is no longer in active use. The usual places to invoke this
|
||||
* function from are the @ref->release() callback or in init failure path
|
||||
* where percpu_ref_init() succeeded but other parts of the initialization
|
||||
* of the embedding object failed.
|
||||
*/
|
||||
void percpu_ref_exit(struct percpu_ref *ref)
|
||||
{
|
||||
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
|
||||
|
||||
if (pcpu_count) {
|
||||
for_each_possible_cpu(cpu)
|
||||
WARN_ON_ONCE(*per_cpu_ptr(pcpu_count, cpu));
|
||||
free_percpu(ref->pcpu_count);
|
||||
free_percpu(pcpu_count);
|
||||
ref->pcpu_count_ptr = PCPU_REF_DEAD;
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(percpu_ref_cancel_init);
|
||||
EXPORT_SYMBOL_GPL(percpu_ref_exit);
|
||||
|
||||
static void percpu_ref_kill_rcu(struct rcu_head *rcu)
|
||||
{
|
||||
struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu);
|
||||
unsigned __percpu *pcpu_count = ref->pcpu_count;
|
||||
unsigned __percpu *pcpu_count = pcpu_count_ptr(ref);
|
||||
unsigned count = 0;
|
||||
int cpu;
|
||||
|
||||
/* Mask out PCPU_REF_DEAD */
|
||||
pcpu_count = (unsigned __percpu *)
|
||||
(((unsigned long) pcpu_count) & ~PCPU_STATUS_MASK);
|
||||
|
||||
for_each_possible_cpu(cpu)
|
||||
count += *per_cpu_ptr(pcpu_count, cpu);
|
||||
|
||||
free_percpu(pcpu_count);
|
||||
|
||||
pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count);
|
||||
|
||||
/*
|
||||
|
|
@ -152,11 +175,10 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu)
|
|||
void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
|
||||
percpu_ref_func_t *confirm_kill)
|
||||
{
|
||||
WARN_ONCE(REF_STATUS(ref->pcpu_count) == PCPU_REF_DEAD,
|
||||
WARN_ONCE(ref->pcpu_count_ptr & PCPU_REF_DEAD,
|
||||
"percpu_ref_kill() called more than once!\n");
|
||||
|
||||
ref->pcpu_count = (unsigned __percpu *)
|
||||
(((unsigned long) ref->pcpu_count)|PCPU_REF_DEAD);
|
||||
ref->pcpu_count_ptr |= PCPU_REF_DEAD;
|
||||
ref->confirm_kill = confirm_kill;
|
||||
|
||||
call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu);
|
||||
|
|
|
|||
|
|
@ -720,8 +720,7 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
|
|||
if (unlikely(align < 2))
|
||||
align = 2;
|
||||
|
||||
if (unlikely(size & 1))
|
||||
size++;
|
||||
size = ALIGN(size, 2);
|
||||
|
||||
if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
|
||||
WARN(true, "illegal size (%zu) or align (%zu) for "
|
||||
|
|
|
|||
Loading…
Reference in a new issue