diff --git a/include/linux/once.h b/include/linux/once.h index 2a83b538dd6a..285f12cb40e6 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -4,21 +4,54 @@ #include #include -bool __get_random_once(void *buf, int nbytes, bool *done, - struct static_key *once_key); +bool __do_once_start(bool *done, unsigned long *flags); +void __do_once_done(bool *done, struct static_key *once_key, + unsigned long *flags); -#define get_random_once(buf, nbytes) \ - ({ \ - bool ___ret = false; \ - static bool ___done = false; \ - static struct static_key ___once_key = \ - STATIC_KEY_INIT_TRUE; \ - if (static_key_true(&___once_key)) \ - ___ret = __get_random_once((buf), \ - (nbytes), \ - &___done, \ - &___once_key); \ - ___ret; \ +/* Call a function exactly once. The idea of DO_ONCE() is to perform + * a function call such as initialization of random seeds, etc, only + * once, where DO_ONCE() can live in the fast-path. After @func has + * been called with the passed arguments, the static key will patch + * out the condition into a nop. DO_ONCE() guarantees type safety of + * arguments! + * + * Not that the following is not equivalent ... + * + * DO_ONCE(func, arg); + * DO_ONCE(func, arg); + * + * ... to this version: + * + * void foo(void) + * { + * DO_ONCE(func, arg); + * } + * + * foo(); + * foo(); + * + * In case the one-time invocation could be triggered from multiple + * places, then a common helper function must be defined, so that only + * a single static key will be placed there! + */ +#define DO_ONCE(func, ...) \ + ({ \ + bool ___ret = false; \ + static bool ___done = false; \ + static struct static_key ___once_key = STATIC_KEY_INIT_TRUE; \ + if (static_key_true(&___once_key)) { \ + unsigned long ___flags; \ + ___ret = __do_once_start(&___done, &___flags); \ + if (unlikely(___ret)) { \ + func(__VA_ARGS__); \ + __do_once_done(&___done, &___once_key, \ + &___flags); \ + } \ + } \ + ___ret; \ }) +#define get_random_once(buf, nbytes) \ + DO_ONCE(get_random_bytes, (buf), (nbytes)) + #endif /* _LINUX_ONCE_H */ diff --git a/lib/once.c b/lib/once.c index 2d5a7de17aba..05c8604627eb 100644 --- a/lib/once.c +++ b/lib/once.c @@ -3,52 +3,60 @@ #include #include -struct __random_once_work { +struct once_work { struct work_struct work; struct static_key *key; }; -static void __random_once_deferred(struct work_struct *w) +static void once_deferred(struct work_struct *w) { - struct __random_once_work *work; + struct once_work *work; - work = container_of(w, struct __random_once_work, work); + work = container_of(w, struct once_work, work); BUG_ON(!static_key_enabled(work->key)); static_key_slow_dec(work->key); kfree(work); } -static void __random_once_disable_jump(struct static_key *key) +static void once_disable_jump(struct static_key *key) { - struct __random_once_work *w; + struct once_work *w; w = kmalloc(sizeof(*w), GFP_ATOMIC); if (!w) return; - INIT_WORK(&w->work, __random_once_deferred); + INIT_WORK(&w->work, once_deferred); w->key = key; schedule_work(&w->work); } -bool __get_random_once(void *buf, int nbytes, bool *done, - struct static_key *once_key) -{ - static DEFINE_SPINLOCK(lock); - unsigned long flags; +static DEFINE_SPINLOCK(once_lock); - spin_lock_irqsave(&lock, flags); +bool __do_once_start(bool *done, unsigned long *flags) + __acquires(once_lock) +{ + spin_lock_irqsave(&once_lock, *flags); if (*done) { - spin_unlock_irqrestore(&lock, flags); + spin_unlock_irqrestore(&once_lock, *flags); + /* Keep sparse happy by restoring an even lock count on + * this lock. In case we return here, we don't call into + * __do_once_done but return early in the DO_ONCE() macro. + */ + __acquire(once_lock); return false; } - get_random_bytes(buf, nbytes); - *done = true; - spin_unlock_irqrestore(&lock, flags); - - __random_once_disable_jump(once_key); - return true; } -EXPORT_SYMBOL(__get_random_once); +EXPORT_SYMBOL(__do_once_start); + +void __do_once_done(bool *done, struct static_key *once_key, + unsigned long *flags) + __releases(once_lock) +{ + *done = true; + spin_unlock_irqrestore(&once_lock, *flags); + once_disable_jump(once_key); +} +EXPORT_SYMBOL(__do_once_done);