a998d43423
Now the helper function from filter.c for negative offsets is exported, it can be used it in the jit to handle negative offsets. First modify the asm load helper functions to handle: - know positive offsets - know negative offsets - any offset then the compiler can be modified to explicitly use these helper when appropriate. This fixes the case of a negative X register and allows to lift the restriction that bpf programs with negative offsets can't be jited. Signed-of-by: Jan Seiffert <kaffeemonster@googlemail.com> Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
200 lines
4.1 KiB
ArmAsm
200 lines
4.1 KiB
ArmAsm
/* bpf_jit.S : BPF JIT helper functions
|
|
*
|
|
* Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
|
|
*
|
|
* This program is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU General Public License
|
|
* as published by the Free Software Foundation; version 2
|
|
* of the License.
|
|
*/
|
|
#include <linux/linkage.h>
|
|
#include <asm/dwarf2.h>
|
|
|
|
/*
|
|
* Calling convention :
|
|
* rdi : skb pointer
|
|
* esi : offset of byte(s) to fetch in skb (can be scratched)
|
|
* r8 : copy of skb->data
|
|
* r9d : hlen = skb->len - skb->data_len
|
|
*/
|
|
#define SKBDATA %r8
|
|
#define SKF_MAX_NEG_OFF $(-0x200000) /* SKF_LL_OFF from filter.h */
|
|
|
|
sk_load_word:
|
|
.globl sk_load_word
|
|
|
|
test %esi,%esi
|
|
js bpf_slow_path_word_neg
|
|
|
|
sk_load_word_positive_offset:
|
|
.globl sk_load_word_positive_offset
|
|
|
|
mov %r9d,%eax # hlen
|
|
sub %esi,%eax # hlen - offset
|
|
cmp $3,%eax
|
|
jle bpf_slow_path_word
|
|
mov (SKBDATA,%rsi),%eax
|
|
bswap %eax /* ntohl() */
|
|
ret
|
|
|
|
sk_load_half:
|
|
.globl sk_load_half
|
|
|
|
test %esi,%esi
|
|
js bpf_slow_path_half_neg
|
|
|
|
sk_load_half_positive_offset:
|
|
.globl sk_load_half_positive_offset
|
|
|
|
mov %r9d,%eax
|
|
sub %esi,%eax # hlen - offset
|
|
cmp $1,%eax
|
|
jle bpf_slow_path_half
|
|
movzwl (SKBDATA,%rsi),%eax
|
|
rol $8,%ax # ntohs()
|
|
ret
|
|
|
|
sk_load_byte:
|
|
.globl sk_load_byte
|
|
|
|
test %esi,%esi
|
|
js bpf_slow_path_byte_neg
|
|
|
|
sk_load_byte_positive_offset:
|
|
.globl sk_load_byte_positive_offset
|
|
|
|
cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte */
|
|
jle bpf_slow_path_byte
|
|
movzbl (SKBDATA,%rsi),%eax
|
|
ret
|
|
|
|
/**
|
|
* sk_load_byte_msh - BPF_S_LDX_B_MSH helper
|
|
*
|
|
* Implements BPF_S_LDX_B_MSH : ldxb 4*([offset]&0xf)
|
|
* Must preserve A accumulator (%eax)
|
|
* Inputs : %esi is the offset value
|
|
*/
|
|
sk_load_byte_msh:
|
|
.globl sk_load_byte_msh
|
|
test %esi,%esi
|
|
js bpf_slow_path_byte_msh_neg
|
|
|
|
sk_load_byte_msh_positive_offset:
|
|
.globl sk_load_byte_msh_positive_offset
|
|
cmp %esi,%r9d /* if (offset >= hlen) goto bpf_slow_path_byte_msh */
|
|
jle bpf_slow_path_byte_msh
|
|
movzbl (SKBDATA,%rsi),%ebx
|
|
and $15,%bl
|
|
shl $2,%bl
|
|
ret
|
|
|
|
/* rsi contains offset and can be scratched */
|
|
#define bpf_slow_path_common(LEN) \
|
|
push %rdi; /* save skb */ \
|
|
push %r9; \
|
|
push SKBDATA; \
|
|
/* rsi already has offset */ \
|
|
mov $LEN,%ecx; /* len */ \
|
|
lea -12(%rbp),%rdx; \
|
|
call skb_copy_bits; \
|
|
test %eax,%eax; \
|
|
pop SKBDATA; \
|
|
pop %r9; \
|
|
pop %rdi
|
|
|
|
|
|
bpf_slow_path_word:
|
|
bpf_slow_path_common(4)
|
|
js bpf_error
|
|
mov -12(%rbp),%eax
|
|
bswap %eax
|
|
ret
|
|
|
|
bpf_slow_path_half:
|
|
bpf_slow_path_common(2)
|
|
js bpf_error
|
|
mov -12(%rbp),%ax
|
|
rol $8,%ax
|
|
movzwl %ax,%eax
|
|
ret
|
|
|
|
bpf_slow_path_byte:
|
|
bpf_slow_path_common(1)
|
|
js bpf_error
|
|
movzbl -12(%rbp),%eax
|
|
ret
|
|
|
|
bpf_slow_path_byte_msh:
|
|
xchg %eax,%ebx /* dont lose A , X is about to be scratched */
|
|
bpf_slow_path_common(1)
|
|
js bpf_error
|
|
movzbl -12(%rbp),%eax
|
|
and $15,%al
|
|
shl $2,%al
|
|
xchg %eax,%ebx
|
|
ret
|
|
|
|
#define sk_negative_common(SIZE) \
|
|
push %rdi; /* save skb */ \
|
|
push %r9; \
|
|
push SKBDATA; \
|
|
/* rsi already has offset */ \
|
|
mov $SIZE,%ecx; /* size */ \
|
|
call bpf_internal_load_pointer_neg_helper; \
|
|
test %rax,%rax; \
|
|
pop SKBDATA; \
|
|
pop %r9; \
|
|
pop %rdi; \
|
|
jz bpf_error
|
|
|
|
|
|
bpf_slow_path_word_neg:
|
|
cmp SKF_MAX_NEG_OFF, %esi /* test range */
|
|
jl bpf_error /* offset lower -> error */
|
|
sk_load_word_negative_offset:
|
|
.globl sk_load_word_negative_offset
|
|
sk_negative_common(4)
|
|
mov (%rax), %eax
|
|
bswap %eax
|
|
ret
|
|
|
|
bpf_slow_path_half_neg:
|
|
cmp SKF_MAX_NEG_OFF, %esi
|
|
jl bpf_error
|
|
sk_load_half_negative_offset:
|
|
.globl sk_load_half_negative_offset
|
|
sk_negative_common(2)
|
|
mov (%rax),%ax
|
|
rol $8,%ax
|
|
movzwl %ax,%eax
|
|
ret
|
|
|
|
bpf_slow_path_byte_neg:
|
|
cmp SKF_MAX_NEG_OFF, %esi
|
|
jl bpf_error
|
|
sk_load_byte_negative_offset:
|
|
.globl sk_load_byte_negative_offset
|
|
sk_negative_common(1)
|
|
movzbl (%rax), %eax
|
|
ret
|
|
|
|
bpf_slow_path_byte_msh_neg:
|
|
cmp SKF_MAX_NEG_OFF, %esi
|
|
jl bpf_error
|
|
sk_load_byte_msh_negative_offset:
|
|
.globl sk_load_byte_msh_negative_offset
|
|
xchg %eax,%ebx /* dont lose A , X is about to be scratched */
|
|
sk_negative_common(1)
|
|
movzbl (%rax),%eax
|
|
and $15,%al
|
|
shl $2,%al
|
|
xchg %eax,%ebx
|
|
ret
|
|
|
|
bpf_error:
|
|
# force a return 0 from jit handler
|
|
xor %eax,%eax
|
|
mov -8(%rbp),%rbx
|
|
leaveq
|
|
ret
|