2c57a0e233
This patchset does rework to find_bit function family to achieve better performance, and decrease size of text. All rework is done in patch 1. Patches 2 and 3 are about code moving and renaming. It was boot-tested on x86_64 and MIPS (big-endian) machines. Performance tests were ran on userspace with code like this: /* addr[] is filled from /dev/urandom */ start = clock(); while (ret < nbits) ret = find_next_bit(addr, nbits, ret + 1); end = clock(); printf("%ld\t", (unsigned long) end - start); On Intel(R) Core(TM) i7-3770 CPU @ 3.40GHz measurements are: (for find_next_bit, nbits is 8M, for find_first_bit - 80K) find_next_bit: find_first_bit: new current new current 26932 43151 14777 14925 26947 43182 14521 15423 26507 43824 15053 14705 27329 43759 14473 14777 26895 43367 14847 15023 26990 43693 15103 15163 26775 43299 15067 15232 27282 42752 14544 15121 27504 43088 14644 14858 26761 43856 14699 15193 26692 43075 14781 14681 27137 42969 14451 15061 ... ... find_next_bit performance gain is 35-40%; find_first_bit - no measurable difference. On ARM machine, there is arch-specific implementation for find_bit. Thanks a lot to George Spelvin and Rasmus Villemoes for hints and helpful discussions. This patch (of 3): New implementations takes less space in source file (see diffstat) and in object. For me it's 710 vs 453 bytes of text. It also shows better performance. find_last_bit description fixed due to obvious typo. [akpm@linux-foundation.org: include linux/bitmap.h, per Rasmus] Signed-off-by: Yury Norov <yury.norov@gmail.com> Reviewed-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: George Spelvin <linux@horizon.com> Cc: Alexey Klimov <klimov.linux@gmail.com> Cc: David S. Miller <davem@davemloft.net> Cc: Daniel Borkmann <dborkman@redhat.com> Cc: Hannes Frederic Sowa <hannes@stressinduktion.org> Cc: Lai Jiangshan <laijs@cn.fujitsu.com> Cc: Mark Salter <msalter@redhat.com> Cc: AKASHI Takahiro <takahiro.akashi@linaro.org> Cc: Thomas Graf <tgraf@suug.ch> Cc: Valentin Rothberg <valentinrothberg@gmail.com> Cc: Chris Wilson <chris@chris-wilson.co.uk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
230 lines
5.6 KiB
C
230 lines
5.6 KiB
C
#ifndef _LINUX_BITOPS_H
|
|
#define _LINUX_BITOPS_H
|
|
#include <asm/types.h>
|
|
|
|
#ifdef __KERNEL__
|
|
#define BIT(nr) (1UL << (nr))
|
|
#define BIT_ULL(nr) (1ULL << (nr))
|
|
#define BIT_MASK(nr) (1UL << ((nr) % BITS_PER_LONG))
|
|
#define BIT_WORD(nr) ((nr) / BITS_PER_LONG)
|
|
#define BIT_ULL_MASK(nr) (1ULL << ((nr) % BITS_PER_LONG_LONG))
|
|
#define BIT_ULL_WORD(nr) ((nr) / BITS_PER_LONG_LONG)
|
|
#define BITS_PER_BYTE 8
|
|
#define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, BITS_PER_BYTE * sizeof(long))
|
|
#endif
|
|
|
|
/*
|
|
* Create a contiguous bitmask starting at bit position @l and ending at
|
|
* position @h. For example
|
|
* GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000.
|
|
*/
|
|
#define GENMASK(h, l) \
|
|
(((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h))))
|
|
|
|
#define GENMASK_ULL(h, l) \
|
|
(((~0ULL) << (l)) & (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h))))
|
|
|
|
extern unsigned int __sw_hweight8(unsigned int w);
|
|
extern unsigned int __sw_hweight16(unsigned int w);
|
|
extern unsigned int __sw_hweight32(unsigned int w);
|
|
extern unsigned long __sw_hweight64(__u64 w);
|
|
|
|
/*
|
|
* Include this here because some architectures need generic_ffs/fls in
|
|
* scope
|
|
*/
|
|
#include <asm/bitops.h>
|
|
|
|
#define for_each_set_bit(bit, addr, size) \
|
|
for ((bit) = find_first_bit((addr), (size)); \
|
|
(bit) < (size); \
|
|
(bit) = find_next_bit((addr), (size), (bit) + 1))
|
|
|
|
/* same as for_each_set_bit() but use bit as value to start with */
|
|
#define for_each_set_bit_from(bit, addr, size) \
|
|
for ((bit) = find_next_bit((addr), (size), (bit)); \
|
|
(bit) < (size); \
|
|
(bit) = find_next_bit((addr), (size), (bit) + 1))
|
|
|
|
#define for_each_clear_bit(bit, addr, size) \
|
|
for ((bit) = find_first_zero_bit((addr), (size)); \
|
|
(bit) < (size); \
|
|
(bit) = find_next_zero_bit((addr), (size), (bit) + 1))
|
|
|
|
/* same as for_each_clear_bit() but use bit as value to start with */
|
|
#define for_each_clear_bit_from(bit, addr, size) \
|
|
for ((bit) = find_next_zero_bit((addr), (size), (bit)); \
|
|
(bit) < (size); \
|
|
(bit) = find_next_zero_bit((addr), (size), (bit) + 1))
|
|
|
|
static __inline__ int get_bitmask_order(unsigned int count)
|
|
{
|
|
int order;
|
|
|
|
order = fls(count);
|
|
return order; /* We could be slightly more clever with -1 here... */
|
|
}
|
|
|
|
static __inline__ int get_count_order(unsigned int count)
|
|
{
|
|
int order;
|
|
|
|
order = fls(count) - 1;
|
|
if (count & (count - 1))
|
|
order++;
|
|
return order;
|
|
}
|
|
|
|
static inline unsigned long hweight_long(unsigned long w)
|
|
{
|
|
return sizeof(w) == 4 ? hweight32(w) : hweight64(w);
|
|
}
|
|
|
|
/**
|
|
* rol64 - rotate a 64-bit value left
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u64 rol64(__u64 word, unsigned int shift)
|
|
{
|
|
return (word << shift) | (word >> (64 - shift));
|
|
}
|
|
|
|
/**
|
|
* ror64 - rotate a 64-bit value right
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u64 ror64(__u64 word, unsigned int shift)
|
|
{
|
|
return (word >> shift) | (word << (64 - shift));
|
|
}
|
|
|
|
/**
|
|
* rol32 - rotate a 32-bit value left
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u32 rol32(__u32 word, unsigned int shift)
|
|
{
|
|
return (word << shift) | (word >> (32 - shift));
|
|
}
|
|
|
|
/**
|
|
* ror32 - rotate a 32-bit value right
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u32 ror32(__u32 word, unsigned int shift)
|
|
{
|
|
return (word >> shift) | (word << (32 - shift));
|
|
}
|
|
|
|
/**
|
|
* rol16 - rotate a 16-bit value left
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u16 rol16(__u16 word, unsigned int shift)
|
|
{
|
|
return (word << shift) | (word >> (16 - shift));
|
|
}
|
|
|
|
/**
|
|
* ror16 - rotate a 16-bit value right
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u16 ror16(__u16 word, unsigned int shift)
|
|
{
|
|
return (word >> shift) | (word << (16 - shift));
|
|
}
|
|
|
|
/**
|
|
* rol8 - rotate an 8-bit value left
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u8 rol8(__u8 word, unsigned int shift)
|
|
{
|
|
return (word << shift) | (word >> (8 - shift));
|
|
}
|
|
|
|
/**
|
|
* ror8 - rotate an 8-bit value right
|
|
* @word: value to rotate
|
|
* @shift: bits to roll
|
|
*/
|
|
static inline __u8 ror8(__u8 word, unsigned int shift)
|
|
{
|
|
return (word >> shift) | (word << (8 - shift));
|
|
}
|
|
|
|
/**
|
|
* sign_extend32 - sign extend a 32-bit value using specified bit as sign-bit
|
|
* @value: value to sign extend
|
|
* @index: 0 based bit index (0<=index<32) to sign bit
|
|
*/
|
|
static inline __s32 sign_extend32(__u32 value, int index)
|
|
{
|
|
__u8 shift = 31 - index;
|
|
return (__s32)(value << shift) >> shift;
|
|
}
|
|
|
|
static inline unsigned fls_long(unsigned long l)
|
|
{
|
|
if (sizeof(l) == 4)
|
|
return fls(l);
|
|
return fls64(l);
|
|
}
|
|
|
|
/**
|
|
* __ffs64 - find first set bit in a 64 bit word
|
|
* @word: The 64 bit word
|
|
*
|
|
* On 64 bit arches this is a synomyn for __ffs
|
|
* The result is not defined if no bits are set, so check that @word
|
|
* is non-zero before calling this.
|
|
*/
|
|
static inline unsigned long __ffs64(u64 word)
|
|
{
|
|
#if BITS_PER_LONG == 32
|
|
if (((u32)word) == 0UL)
|
|
return __ffs((u32)(word >> 32)) + 32;
|
|
#elif BITS_PER_LONG != 64
|
|
#error BITS_PER_LONG not 32 or 64
|
|
#endif
|
|
return __ffs((unsigned long)word);
|
|
}
|
|
|
|
#ifdef __KERNEL__
|
|
|
|
#ifndef set_mask_bits
|
|
#define set_mask_bits(ptr, _mask, _bits) \
|
|
({ \
|
|
const typeof(*ptr) mask = (_mask), bits = (_bits); \
|
|
typeof(*ptr) old, new; \
|
|
\
|
|
do { \
|
|
old = ACCESS_ONCE(*ptr); \
|
|
new = (old & ~mask) | bits; \
|
|
} while (cmpxchg(ptr, old, new) != old); \
|
|
\
|
|
new; \
|
|
})
|
|
#endif
|
|
|
|
#ifndef find_last_bit
|
|
/**
|
|
* find_last_bit - find the last set bit in a memory region
|
|
* @addr: The address to start the search at
|
|
* @size: The number of bits to search
|
|
*
|
|
* Returns the bit number of the last set bit, or size.
|
|
*/
|
|
extern unsigned long find_last_bit(const unsigned long *addr,
|
|
unsigned long size);
|
|
#endif
|
|
|
|
#endif /* __KERNEL__ */
|
|
#endif
|