futex: Replace fshared and clockrt with combined flags
In the early days we passed the mmap sem around. That became the "int fshared" with the fast gup improvements. Then we added "int clockrt" in places. This patch unifies these options as "flags". [ tglx: Split out the stale fshared cleanup ] Signed-off-by: Darren Hart <dvhart@linux.intel.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: John Kacur <jkacur@redhat.com> Cc: Ingo Molnar <mingo@elte.hu> LKML-Reference: <1289250609-16304-1-git-send-email-dvhart@linux.intel.com> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
This commit is contained in:
Darren Hart
Thomas Gleixner
parent
ae791a2d2e
commit
b41277dc7a
1 changed files with 70 additions and 76 deletions
146
kernel/futex.c
146
kernel/futex.c
|
|
@ -68,6 +68,14 @@ int __read_mostly futex_cmpxchg_enabled;
|
|||
|
||||
#define FUTEX_HASHBITS (CONFIG_BASE_SMALL ? 4 : 8)
|
||||
|
||||
/*
|
||||
* Futex flags used to encode options to functions and preserve them across
|
||||
* restarts.
|
||||
*/
|
||||
#define FLAGS_SHARED 0x01
|
||||
#define FLAGS_CLOCKRT 0x02
|
||||
#define FLAGS_HAS_TIMEOUT 0x04
|
||||
|
||||
/*
|
||||
* Priority Inheritance state:
|
||||
*/
|
||||
|
|
@ -869,7 +877,8 @@ double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
|
|||
/*
|
||||
* Wake up waiters matching bitset queued on this futex (uaddr).
|
||||
*/
|
||||
static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset)
|
||||
static int
|
||||
futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
|
||||
{
|
||||
struct futex_hash_bucket *hb;
|
||||
struct futex_q *this, *next;
|
||||
|
|
@ -880,7 +889,7 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset)
|
|||
if (!bitset)
|
||||
return -EINVAL;
|
||||
|
||||
ret = get_futex_key(uaddr, fshared, &key);
|
||||
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
|
||||
|
|
@ -916,7 +925,7 @@ static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset)
|
|||
* to this virtual address:
|
||||
*/
|
||||
static int
|
||||
futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
|
||||
futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
|
||||
int nr_wake, int nr_wake2, int op)
|
||||
{
|
||||
union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
|
||||
|
|
@ -926,10 +935,10 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
|
|||
int ret, op_ret;
|
||||
|
||||
retry:
|
||||
ret = get_futex_key(uaddr1, fshared, &key1);
|
||||
ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
ret = get_futex_key(uaddr2, fshared, &key2);
|
||||
ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2);
|
||||
if (unlikely(ret != 0))
|
||||
goto out_put_key1;
|
||||
|
||||
|
|
@ -961,7 +970,7 @@ futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
|
|||
if (ret)
|
||||
goto out_put_keys;
|
||||
|
||||
if (!fshared)
|
||||
if (!(flags & FLAGS_SHARED))
|
||||
goto retry_private;
|
||||
|
||||
put_futex_key(&key2);
|
||||
|
|
@ -1132,13 +1141,13 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
|
|||
/**
|
||||
* futex_requeue() - Requeue waiters from uaddr1 to uaddr2
|
||||
* @uaddr1: source futex user address
|
||||
* @fshared: 0 for a PROCESS_PRIVATE futex, 1 for PROCESS_SHARED
|
||||
* @flags: futex flags (FLAGS_SHARED, etc.)
|
||||
* @uaddr2: target futex user address
|
||||
* @nr_wake: number of waiters to wake (must be 1 for requeue_pi)
|
||||
* @nr_requeue: number of waiters to requeue (0-INT_MAX)
|
||||
* @cmpval: @uaddr1 expected value (or %NULL)
|
||||
* @requeue_pi: if we are attempting to requeue from a non-pi futex to a
|
||||
* pi futex (pi to pi requeue is not supported)
|
||||
* pi futex (pi to pi requeue is not supported)
|
||||
*
|
||||
* Requeue waiters on uaddr1 to uaddr2. In the requeue_pi case, try to acquire
|
||||
* uaddr2 atomically on behalf of the top waiter.
|
||||
|
|
@ -1147,9 +1156,9 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
|
|||
* >=0 - on success, the number of tasks requeued or woken
|
||||
* <0 - on error
|
||||
*/
|
||||
static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
|
||||
int nr_wake, int nr_requeue, u32 *cmpval,
|
||||
int requeue_pi)
|
||||
static int futex_requeue(u32 __user *uaddr1, unsigned int flags,
|
||||
u32 __user *uaddr2, int nr_wake, int nr_requeue,
|
||||
u32 *cmpval, int requeue_pi)
|
||||
{
|
||||
union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
|
||||
int drop_count = 0, task_count = 0, ret;
|
||||
|
|
@ -1190,10 +1199,10 @@ static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
|
|||
pi_state = NULL;
|
||||
}
|
||||
|
||||
ret = get_futex_key(uaddr1, fshared, &key1);
|
||||
ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
ret = get_futex_key(uaddr2, fshared, &key2);
|
||||
ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2);
|
||||
if (unlikely(ret != 0))
|
||||
goto out_put_key1;
|
||||
|
||||
|
|
@ -1215,7 +1224,7 @@ static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
|
|||
if (ret)
|
||||
goto out_put_keys;
|
||||
|
||||
if (!fshared)
|
||||
if (!(flags & FLAGS_SHARED))
|
||||
goto retry_private;
|
||||
|
||||
put_futex_key(&key2);
|
||||
|
|
@ -1586,14 +1595,6 @@ static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
|
|||
goto retry;
|
||||
}
|
||||
|
||||
/*
|
||||
* In case we must use restart_block to restart a futex_wait,
|
||||
* we encode in the 'flags' shared capability
|
||||
*/
|
||||
#define FLAGS_SHARED 0x01
|
||||
#define FLAGS_CLOCKRT 0x02
|
||||
#define FLAGS_HAS_TIMEOUT 0x04
|
||||
|
||||
static long futex_wait_restart(struct restart_block *restart);
|
||||
|
||||
/**
|
||||
|
|
@ -1712,7 +1713,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
|
|||
* futex_wait_setup() - Prepare to wait on a futex
|
||||
* @uaddr: the futex userspace address
|
||||
* @val: the expected value
|
||||
* @fshared: whether the futex is shared (1) or not (0)
|
||||
* @flags: futex flags (FLAGS_SHARED, etc.)
|
||||
* @q: the associated futex_q
|
||||
* @hb: storage for hash_bucket pointer to be returned to caller
|
||||
*
|
||||
|
|
@ -1725,7 +1726,7 @@ static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
|
|||
* 0 - uaddr contains val and hb has been locked
|
||||
* <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlcoked
|
||||
*/
|
||||
static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared,
|
||||
static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
|
||||
struct futex_q *q, struct futex_hash_bucket **hb)
|
||||
{
|
||||
u32 uval;
|
||||
|
|
@ -1750,7 +1751,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared,
|
|||
*/
|
||||
retry:
|
||||
q->key = FUTEX_KEY_INIT;
|
||||
ret = get_futex_key(uaddr, fshared, &q->key);
|
||||
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key);
|
||||
if (unlikely(ret != 0))
|
||||
return ret;
|
||||
|
||||
|
|
@ -1766,7 +1767,7 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared,
|
|||
if (ret)
|
||||
goto out;
|
||||
|
||||
if (!fshared)
|
||||
if (!(flags & FLAGS_SHARED))
|
||||
goto retry_private;
|
||||
|
||||
put_futex_key(&q->key);
|
||||
|
|
@ -1784,8 +1785,8 @@ static int futex_wait_setup(u32 __user *uaddr, u32 val, int fshared,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int futex_wait(u32 __user *uaddr, int fshared,
|
||||
u32 val, ktime_t *abs_time, u32 bitset, int clockrt)
|
||||
static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
|
||||
ktime_t *abs_time, u32 bitset)
|
||||
{
|
||||
struct hrtimer_sleeper timeout, *to = NULL;
|
||||
struct restart_block *restart;
|
||||
|
|
@ -1804,8 +1805,9 @@ static int futex_wait(u32 __user *uaddr, int fshared,
|
|||
if (abs_time) {
|
||||
to = &timeout;
|
||||
|
||||
hrtimer_init_on_stack(&to->timer, clockrt ? CLOCK_REALTIME :
|
||||
CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
||||
hrtimer_init_on_stack(&to->timer, (flags & FLAGS_CLOCKRT) ?
|
||||
CLOCK_REALTIME : CLOCK_MONOTONIC,
|
||||
HRTIMER_MODE_ABS);
|
||||
hrtimer_init_sleeper(to, current);
|
||||
hrtimer_set_expires_range_ns(&to->timer, *abs_time,
|
||||
current->timer_slack_ns);
|
||||
|
|
@ -1816,7 +1818,7 @@ static int futex_wait(u32 __user *uaddr, int fshared,
|
|||
* Prepare to wait on uaddr. On success, holds hb lock and increments
|
||||
* q.key refs.
|
||||
*/
|
||||
ret = futex_wait_setup(uaddr, val, fshared, &q, &hb);
|
||||
ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
|
||||
if (ret)
|
||||
goto out;
|
||||
|
||||
|
|
@ -1849,12 +1851,7 @@ static int futex_wait(u32 __user *uaddr, int fshared,
|
|||
restart->futex.val = val;
|
||||
restart->futex.time = abs_time->tv64;
|
||||
restart->futex.bitset = bitset;
|
||||
restart->futex.flags = FLAGS_HAS_TIMEOUT;
|
||||
|
||||
if (fshared)
|
||||
restart->futex.flags |= FLAGS_SHARED;
|
||||
if (clockrt)
|
||||
restart->futex.flags |= FLAGS_CLOCKRT;
|
||||
restart->futex.flags = flags;
|
||||
|
||||
ret = -ERESTART_RESTARTBLOCK;
|
||||
|
||||
|
|
@ -1870,7 +1867,6 @@ static int futex_wait(u32 __user *uaddr, int fshared,
|
|||
static long futex_wait_restart(struct restart_block *restart)
|
||||
{
|
||||
u32 __user *uaddr = restart->futex.uaddr;
|
||||
int fshared = 0;
|
||||
ktime_t t, *tp = NULL;
|
||||
|
||||
if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {
|
||||
|
|
@ -1878,11 +1874,9 @@ static long futex_wait_restart(struct restart_block *restart)
|
|||
tp = &t;
|
||||
}
|
||||
restart->fn = do_no_restart_syscall;
|
||||
if (restart->futex.flags & FLAGS_SHARED)
|
||||
fshared = 1;
|
||||
return (long)futex_wait(uaddr, fshared, restart->futex.val, tp,
|
||||
restart->futex.bitset,
|
||||
restart->futex.flags & FLAGS_CLOCKRT);
|
||||
|
||||
return (long)futex_wait(uaddr, restart->futex.flags,
|
||||
restart->futex.val, tp, restart->futex.bitset);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -1892,8 +1886,8 @@ static long futex_wait_restart(struct restart_block *restart)
|
|||
* if there are waiters then it will block, it does PI, etc. (Due to
|
||||
* races the kernel might see a 0 value of the futex too.)
|
||||
*/
|
||||
static int futex_lock_pi(u32 __user *uaddr, int fshared,
|
||||
int detect, ktime_t *time, int trylock)
|
||||
static int futex_lock_pi(u32 __user *uaddr, unsigned int flags, int detect,
|
||||
ktime_t *time, int trylock)
|
||||
{
|
||||
struct hrtimer_sleeper timeout, *to = NULL;
|
||||
struct futex_hash_bucket *hb;
|
||||
|
|
@ -1916,7 +1910,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
|
|||
q.requeue_pi_key = NULL;
|
||||
retry:
|
||||
q.key = FUTEX_KEY_INIT;
|
||||
ret = get_futex_key(uaddr, fshared, &q.key);
|
||||
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
|
||||
|
|
@ -2005,7 +1999,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
|
|||
if (ret)
|
||||
goto out_put_key;
|
||||
|
||||
if (!fshared)
|
||||
if (!(flags & FLAGS_SHARED))
|
||||
goto retry_private;
|
||||
|
||||
put_futex_key(&q.key);
|
||||
|
|
@ -2017,7 +2011,7 @@ static int futex_lock_pi(u32 __user *uaddr, int fshared,
|
|||
* This is the in-kernel slowpath: we look up the PI state (if any),
|
||||
* and do the rt-mutex unlock.
|
||||
*/
|
||||
static int futex_unlock_pi(u32 __user *uaddr, int fshared)
|
||||
static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
|
||||
{
|
||||
struct futex_hash_bucket *hb;
|
||||
struct futex_q *this, *next;
|
||||
|
|
@ -2035,7 +2029,7 @@ static int futex_unlock_pi(u32 __user *uaddr, int fshared)
|
|||
if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current))
|
||||
return -EPERM;
|
||||
|
||||
ret = get_futex_key(uaddr, fshared, &key);
|
||||
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
|
||||
|
|
@ -2157,7 +2151,7 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
|
|||
/**
|
||||
* futex_wait_requeue_pi() - Wait on uaddr and take uaddr2
|
||||
* @uaddr: the futex we initially wait on (non-pi)
|
||||
* @fshared: whether the futexes are shared (1) or not (0). They must be
|
||||
* @flags: futex flags (FLAGS_SHARED, FLAGS_CLOCKRT, etc.), they must be
|
||||
* the same type, no requeueing from private to shared, etc.
|
||||
* @val: the expected value of uaddr
|
||||
* @abs_time: absolute timeout
|
||||
|
|
@ -2195,9 +2189,9 @@ int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
|
|||
* 0 - On success
|
||||
* <0 - On error
|
||||
*/
|
||||
static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
|
||||
static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
|
||||
u32 val, ktime_t *abs_time, u32 bitset,
|
||||
int clockrt, u32 __user *uaddr2)
|
||||
u32 __user *uaddr2)
|
||||
{
|
||||
struct hrtimer_sleeper timeout, *to = NULL;
|
||||
struct rt_mutex_waiter rt_waiter;
|
||||
|
|
@ -2212,8 +2206,9 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
|
|||
|
||||
if (abs_time) {
|
||||
to = &timeout;
|
||||
hrtimer_init_on_stack(&to->timer, clockrt ? CLOCK_REALTIME :
|
||||
CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
|
||||
hrtimer_init_on_stack(&to->timer, (flags & FLAGS_CLOCKRT) ?
|
||||
CLOCK_REALTIME : CLOCK_MONOTONIC,
|
||||
HRTIMER_MODE_ABS);
|
||||
hrtimer_init_sleeper(to, current);
|
||||
hrtimer_set_expires_range_ns(&to->timer, *abs_time,
|
||||
current->timer_slack_ns);
|
||||
|
|
@ -2227,7 +2222,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
|
|||
rt_waiter.task = NULL;
|
||||
|
||||
key2 = FUTEX_KEY_INIT;
|
||||
ret = get_futex_key(uaddr2, fshared, &key2);
|
||||
ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2);
|
||||
if (unlikely(ret != 0))
|
||||
goto out;
|
||||
|
||||
|
|
@ -2240,7 +2235,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, int fshared,
|
|||
* Prepare to wait on uaddr. On success, increments q.key (key1) ref
|
||||
* count.
|
||||
*/
|
||||
ret = futex_wait_setup(uaddr, val, fshared, &q, &hb);
|
||||
ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
|
||||
if (ret)
|
||||
goto out_key2;
|
||||
|
||||
|
|
@ -2547,58 +2542,57 @@ void exit_robust_list(struct task_struct *curr)
|
|||
long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
|
||||
u32 __user *uaddr2, u32 val2, u32 val3)
|
||||
{
|
||||
int clockrt, ret = -ENOSYS;
|
||||
int cmd = op & FUTEX_CMD_MASK;
|
||||
int fshared = 0;
|
||||
int ret = -ENOSYS, cmd = op & FUTEX_CMD_MASK;
|
||||
unsigned int flags = 0;
|
||||
|
||||
if (!(op & FUTEX_PRIVATE_FLAG))
|
||||
fshared = 1;
|
||||
flags |= FLAGS_SHARED;
|
||||
|
||||
clockrt = op & FUTEX_CLOCK_REALTIME;
|
||||
if (clockrt && cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
|
||||
return -ENOSYS;
|
||||
if (op & FUTEX_CLOCK_REALTIME) {
|
||||
flags |= FLAGS_CLOCKRT;
|
||||
if (cmd != FUTEX_WAIT_BITSET && cmd != FUTEX_WAIT_REQUEUE_PI)
|
||||
return -ENOSYS;
|
||||
}
|
||||
|
||||
switch (cmd) {
|
||||
case FUTEX_WAIT:
|
||||
val3 = FUTEX_BITSET_MATCH_ANY;
|
||||
case FUTEX_WAIT_BITSET:
|
||||
ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt);
|
||||
ret = futex_wait(uaddr, flags, val, timeout, val3);
|
||||
break;
|
||||
case FUTEX_WAKE:
|
||||
val3 = FUTEX_BITSET_MATCH_ANY;
|
||||
case FUTEX_WAKE_BITSET:
|
||||
ret = futex_wake(uaddr, fshared, val, val3);
|
||||
ret = futex_wake(uaddr, flags, val, val3);
|
||||
break;
|
||||
case FUTEX_REQUEUE:
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, NULL, 0);
|
||||
ret = futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);
|
||||
break;
|
||||
case FUTEX_CMP_REQUEUE:
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, &val3,
|
||||
0);
|
||||
ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);
|
||||
break;
|
||||
case FUTEX_WAKE_OP:
|
||||
ret = futex_wake_op(uaddr, fshared, uaddr2, val, val2, val3);
|
||||
ret = futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
|
||||
break;
|
||||
case FUTEX_LOCK_PI:
|
||||
if (futex_cmpxchg_enabled)
|
||||
ret = futex_lock_pi(uaddr, fshared, val, timeout, 0);
|
||||
ret = futex_lock_pi(uaddr, flags, val, timeout, 0);
|
||||
break;
|
||||
case FUTEX_UNLOCK_PI:
|
||||
if (futex_cmpxchg_enabled)
|
||||
ret = futex_unlock_pi(uaddr, fshared);
|
||||
ret = futex_unlock_pi(uaddr, flags);
|
||||
break;
|
||||
case FUTEX_TRYLOCK_PI:
|
||||
if (futex_cmpxchg_enabled)
|
||||
ret = futex_lock_pi(uaddr, fshared, 0, timeout, 1);
|
||||
ret = futex_lock_pi(uaddr, flags, 0, timeout, 1);
|
||||
break;
|
||||
case FUTEX_WAIT_REQUEUE_PI:
|
||||
val3 = FUTEX_BITSET_MATCH_ANY;
|
||||
ret = futex_wait_requeue_pi(uaddr, fshared, val, timeout, val3,
|
||||
clockrt, uaddr2);
|
||||
ret = futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
|
||||
uaddr2);
|
||||
break;
|
||||
case FUTEX_CMP_REQUEUE_PI:
|
||||
ret = futex_requeue(uaddr, fshared, uaddr2, val, val2, &val3,
|
||||
1);
|
||||
ret = futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
|
||||
break;
|
||||
default:
|
||||
ret = -ENOSYS;
|
||||
|
|
|
|||
Loading…
Reference in a new issue