sparc64: Make rwsems 64-bit.
Basically tip-off the powerpc code, use a 64-bit type and atomic64_t interfaces for the implementation. This gets us off of the by-hand asm code I wrote, which frankly I think probably ruins I-cache hit rates. The idea was the keep the call chains less deep, but anything taking the rw-semaphores probably is also calling other stuff and therefore already has allocated a stack-frame. So no real stack frame savings ever. Ben H. has posted patches to make powerpc use 64-bit too and with some abstractions we can probably use a shared header file somewhere. With suggestions from Sam Ravnborg. Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
David S. Miller
parent
b10f997bb0
commit
9b3bb86aca
4 changed files with 104 additions and 193 deletions
|
|
@ -1,12 +0,0 @@
|
|||
/* rwsem-const.h: RW semaphore counter constants. */
|
||||
#ifndef _SPARC64_RWSEM_CONST_H
|
||||
#define _SPARC64_RWSEM_CONST_H
|
||||
|
||||
#define RWSEM_UNLOCKED_VALUE 0x00000000
|
||||
#define RWSEM_ACTIVE_BIAS 0x00000001
|
||||
#define RWSEM_ACTIVE_MASK 0x0000ffff
|
||||
#define RWSEM_WAITING_BIAS (-0x00010000)
|
||||
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
|
||||
#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
|
||||
|
||||
#endif /* _SPARC64_RWSEM_CONST_H */
|
||||
|
|
@ -15,16 +15,21 @@
|
|||
|
||||
#include <linux/list.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <asm/rwsem-const.h>
|
||||
|
||||
struct rwsem_waiter;
|
||||
|
||||
struct rw_semaphore {
|
||||
signed int count;
|
||||
spinlock_t wait_lock;
|
||||
struct list_head wait_list;
|
||||
signed long count;
|
||||
#define RWSEM_UNLOCKED_VALUE 0x00000000L
|
||||
#define RWSEM_ACTIVE_BIAS 0x00000001L
|
||||
#define RWSEM_ACTIVE_MASK 0xffffffffL
|
||||
#define RWSEM_WAITING_BIAS (-RWSEM_ACTIVE_MASK-1)
|
||||
#define RWSEM_ACTIVE_READ_BIAS RWSEM_ACTIVE_BIAS
|
||||
#define RWSEM_ACTIVE_WRITE_BIAS (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
|
||||
spinlock_t wait_lock;
|
||||
struct list_head wait_list;
|
||||
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
||||
struct lockdep_map dep_map;
|
||||
struct lockdep_map dep_map;
|
||||
#endif
|
||||
};
|
||||
|
||||
|
|
@ -41,6 +46,11 @@ struct rw_semaphore {
|
|||
#define DECLARE_RWSEM(name) \
|
||||
struct rw_semaphore name = __RWSEM_INITIALIZER(name)
|
||||
|
||||
extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem);
|
||||
extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem);
|
||||
extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem);
|
||||
extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem);
|
||||
|
||||
extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
|
||||
struct lock_class_key *key);
|
||||
|
||||
|
|
@ -51,27 +61,103 @@ do { \
|
|||
__init_rwsem((sem), #sem, &__key); \
|
||||
} while (0)
|
||||
|
||||
extern void __down_read(struct rw_semaphore *sem);
|
||||
extern int __down_read_trylock(struct rw_semaphore *sem);
|
||||
extern void __down_write(struct rw_semaphore *sem);
|
||||
extern int __down_write_trylock(struct rw_semaphore *sem);
|
||||
extern void __up_read(struct rw_semaphore *sem);
|
||||
extern void __up_write(struct rw_semaphore *sem);
|
||||
extern void __downgrade_write(struct rw_semaphore *sem);
|
||||
/*
|
||||
* lock for reading
|
||||
*/
|
||||
static inline void __down_read(struct rw_semaphore *sem)
|
||||
{
|
||||
if (unlikely(atomic64_inc_return((atomic64_t *)(&sem->count)) <= 0L))
|
||||
rwsem_down_read_failed(sem);
|
||||
}
|
||||
|
||||
static inline int __down_read_trylock(struct rw_semaphore *sem)
|
||||
{
|
||||
long tmp;
|
||||
|
||||
while ((tmp = sem->count) >= 0L) {
|
||||
if (tmp == cmpxchg(&sem->count, tmp,
|
||||
tmp + RWSEM_ACTIVE_READ_BIAS)) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* lock for writing
|
||||
*/
|
||||
static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
|
||||
{
|
||||
__down_write(sem);
|
||||
long tmp;
|
||||
|
||||
tmp = atomic64_add_return(RWSEM_ACTIVE_WRITE_BIAS,
|
||||
(atomic64_t *)(&sem->count));
|
||||
if (unlikely(tmp != RWSEM_ACTIVE_WRITE_BIAS))
|
||||
rwsem_down_write_failed(sem);
|
||||
}
|
||||
|
||||
static inline int rwsem_atomic_update(int delta, struct rw_semaphore *sem)
|
||||
static inline void __down_write(struct rw_semaphore *sem)
|
||||
{
|
||||
return atomic_add_return(delta, (atomic_t *)(&sem->count));
|
||||
__down_write_nested(sem, 0);
|
||||
}
|
||||
|
||||
static inline void rwsem_atomic_add(int delta, struct rw_semaphore *sem)
|
||||
static inline int __down_write_trylock(struct rw_semaphore *sem)
|
||||
{
|
||||
atomic_add(delta, (atomic_t *)(&sem->count));
|
||||
long tmp;
|
||||
|
||||
tmp = cmpxchg(&sem->count, RWSEM_UNLOCKED_VALUE,
|
||||
RWSEM_ACTIVE_WRITE_BIAS);
|
||||
return tmp == RWSEM_UNLOCKED_VALUE;
|
||||
}
|
||||
|
||||
/*
|
||||
* unlock after reading
|
||||
*/
|
||||
static inline void __up_read(struct rw_semaphore *sem)
|
||||
{
|
||||
long tmp;
|
||||
|
||||
tmp = atomic64_dec_return((atomic64_t *)(&sem->count));
|
||||
if (unlikely(tmp < -1L && (tmp & RWSEM_ACTIVE_MASK) == 0L))
|
||||
rwsem_wake(sem);
|
||||
}
|
||||
|
||||
/*
|
||||
* unlock after writing
|
||||
*/
|
||||
static inline void __up_write(struct rw_semaphore *sem)
|
||||
{
|
||||
if (unlikely(atomic64_sub_return(RWSEM_ACTIVE_WRITE_BIAS,
|
||||
(atomic64_t *)(&sem->count)) < 0L))
|
||||
rwsem_wake(sem);
|
||||
}
|
||||
|
||||
/*
|
||||
* implement atomic add functionality
|
||||
*/
|
||||
static inline void rwsem_atomic_add(long delta, struct rw_semaphore *sem)
|
||||
{
|
||||
atomic64_add(delta, (atomic64_t *)(&sem->count));
|
||||
}
|
||||
|
||||
/*
|
||||
* downgrade write lock to read lock
|
||||
*/
|
||||
static inline void __downgrade_write(struct rw_semaphore *sem)
|
||||
{
|
||||
long tmp;
|
||||
|
||||
tmp = atomic64_add_return(-RWSEM_WAITING_BIAS, (atomic64_t *)(&sem->count));
|
||||
if (tmp < 0L)
|
||||
rwsem_downgrade_wake(sem);
|
||||
}
|
||||
|
||||
/*
|
||||
* implement exchange and add functionality
|
||||
*/
|
||||
static inline long rwsem_atomic_update(long delta, struct rw_semaphore *sem)
|
||||
{
|
||||
return atomic64_add_return(delta, (atomic64_t *)(&sem->count));
|
||||
}
|
||||
|
||||
static inline int rwsem_is_locked(struct rw_semaphore *sem)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o
|
|||
lib-$(CONFIG_SPARC32) += copy_user.o locks.o
|
||||
lib-y += atomic_$(BITS).o
|
||||
lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o
|
||||
lib-y += rwsem_$(BITS).o
|
||||
lib-$(CONFIG_SPARC32) += rwsem_32.o
|
||||
lib-$(CONFIG_SPARC32) += muldi3.o bitext.o cmpdi2.o
|
||||
|
||||
lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o
|
||||
|
|
|
|||
|
|
@ -1,163 +0,0 @@
|
|||
/* rwsem.S: RW semaphore assembler.
|
||||
*
|
||||
* Written by David S. Miller (davem@redhat.com), 2001.
|
||||
* Derived from asm-i386/rwsem.h
|
||||
*/
|
||||
|
||||
#include <asm/rwsem-const.h>
|
||||
|
||||
.section .sched.text, "ax"
|
||||
|
||||
.globl __down_read
|
||||
__down_read:
|
||||
1: lduw [%o0], %g1
|
||||
add %g1, 1, %g7
|
||||
cas [%o0], %g1, %g7
|
||||
cmp %g1, %g7
|
||||
bne,pn %icc, 1b
|
||||
add %g7, 1, %g7
|
||||
cmp %g7, 0
|
||||
bl,pn %icc, 3f
|
||||
nop
|
||||
2:
|
||||
retl
|
||||
nop
|
||||
3:
|
||||
save %sp, -192, %sp
|
||||
call rwsem_down_read_failed
|
||||
mov %i0, %o0
|
||||
ret
|
||||
restore
|
||||
.size __down_read, .-__down_read
|
||||
|
||||
.globl __down_read_trylock
|
||||
__down_read_trylock:
|
||||
1: lduw [%o0], %g1
|
||||
add %g1, 1, %g7
|
||||
cmp %g7, 0
|
||||
bl,pn %icc, 2f
|
||||
mov 0, %o1
|
||||
cas [%o0], %g1, %g7
|
||||
cmp %g1, %g7
|
||||
bne,pn %icc, 1b
|
||||
mov 1, %o1
|
||||
2: retl
|
||||
mov %o1, %o0
|
||||
.size __down_read_trylock, .-__down_read_trylock
|
||||
|
||||
.globl __down_write
|
||||
__down_write:
|
||||
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
|
||||
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
|
||||
1:
|
||||
lduw [%o0], %g3
|
||||
add %g3, %g1, %g7
|
||||
cas [%o0], %g3, %g7
|
||||
cmp %g3, %g7
|
||||
bne,pn %icc, 1b
|
||||
cmp %g7, 0
|
||||
bne,pn %icc, 3f
|
||||
nop
|
||||
2: retl
|
||||
nop
|
||||
3:
|
||||
save %sp, -192, %sp
|
||||
call rwsem_down_write_failed
|
||||
mov %i0, %o0
|
||||
ret
|
||||
restore
|
||||
.size __down_write, .-__down_write
|
||||
|
||||
.globl __down_write_trylock
|
||||
__down_write_trylock:
|
||||
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
|
||||
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
|
||||
1:
|
||||
lduw [%o0], %g3
|
||||
cmp %g3, 0
|
||||
bne,pn %icc, 2f
|
||||
mov 0, %o1
|
||||
add %g3, %g1, %g7
|
||||
cas [%o0], %g3, %g7
|
||||
cmp %g3, %g7
|
||||
bne,pn %icc, 1b
|
||||
mov 1, %o1
|
||||
2: retl
|
||||
mov %o1, %o0
|
||||
.size __down_write_trylock, .-__down_write_trylock
|
||||
|
||||
.globl __up_read
|
||||
__up_read:
|
||||
1:
|
||||
lduw [%o0], %g1
|
||||
sub %g1, 1, %g7
|
||||
cas [%o0], %g1, %g7
|
||||
cmp %g1, %g7
|
||||
bne,pn %icc, 1b
|
||||
cmp %g7, 0
|
||||
bl,pn %icc, 3f
|
||||
nop
|
||||
2: retl
|
||||
nop
|
||||
3: sethi %hi(RWSEM_ACTIVE_MASK), %g1
|
||||
sub %g7, 1, %g7
|
||||
or %g1, %lo(RWSEM_ACTIVE_MASK), %g1
|
||||
andcc %g7, %g1, %g0
|
||||
bne,pn %icc, 2b
|
||||
nop
|
||||
save %sp, -192, %sp
|
||||
call rwsem_wake
|
||||
mov %i0, %o0
|
||||
ret
|
||||
restore
|
||||
.size __up_read, .-__up_read
|
||||
|
||||
.globl __up_write
|
||||
__up_write:
|
||||
sethi %hi(RWSEM_ACTIVE_WRITE_BIAS), %g1
|
||||
or %g1, %lo(RWSEM_ACTIVE_WRITE_BIAS), %g1
|
||||
1:
|
||||
lduw [%o0], %g3
|
||||
sub %g3, %g1, %g7
|
||||
cas [%o0], %g3, %g7
|
||||
cmp %g3, %g7
|
||||
bne,pn %icc, 1b
|
||||
sub %g7, %g1, %g7
|
||||
cmp %g7, 0
|
||||
bl,pn %icc, 3f
|
||||
nop
|
||||
2:
|
||||
retl
|
||||
nop
|
||||
3:
|
||||
save %sp, -192, %sp
|
||||
call rwsem_wake
|
||||
mov %i0, %o0
|
||||
ret
|
||||
restore
|
||||
.size __up_write, .-__up_write
|
||||
|
||||
.globl __downgrade_write
|
||||
__downgrade_write:
|
||||
sethi %hi(RWSEM_WAITING_BIAS), %g1
|
||||
or %g1, %lo(RWSEM_WAITING_BIAS), %g1
|
||||
1:
|
||||
lduw [%o0], %g3
|
||||
sub %g3, %g1, %g7
|
||||
cas [%o0], %g3, %g7
|
||||
cmp %g3, %g7
|
||||
bne,pn %icc, 1b
|
||||
sub %g7, %g1, %g7
|
||||
cmp %g7, 0
|
||||
bl,pn %icc, 3f
|
||||
nop
|
||||
2:
|
||||
retl
|
||||
nop
|
||||
3:
|
||||
save %sp, -192, %sp
|
||||
call rwsem_downgrade_wake
|
||||
mov %i0, %o0
|
||||
ret
|
||||
restore
|
||||
.size __downgrade_write, .-__downgrade_write
|
||||
Loading…
Reference in a new issue