[MIPS] R4000/R4400 daddiu erratum workaround
This complements the generic R4000/R4400 errata workaround code and adds bits for the daddiu problem. In most places it just modifies handwritten assembly code so that the assembler is allowed to use a temporary register as daddiu may now be treated as a macro that expands to a sequence of li and daddu. It is the AT register or, where AT is unavailable or used explicitly for another purpose, an explicitly-named register is selected, using the .set at=<reg> feature added recently to gas. This feature is only used if CONFIG_CPU_DADDI_WORKAROUNDS has been set, so if the workaround remains disabled, the required version of binutils stays unchanged. Similarly, daddiu instructions put in branch delay slots in noreorder fragments are now taken out of them and the assembler is allowed to reorder them itself as possible (which it does making the whole idea of scheduling them into delay slots manually questionable). Also in the very few places where such a simple conversion was not possible, a handcoded longer sequence is implemented. Other than that there are changes to code responsible for building the TLB fault and page clear/copy handlers to avoid daddiu as appropriate. These are only effective if the erratum is verified to be present at the run time. Finally there is a trivial update to __delay(), because it uses daddiu in a branch delay slot. Signed-off-by: Maciej W. Rozycki <macro@linux-mips.org> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
This commit is contained in:
Maciej W. Rozycki
Ralf Baechle
parent
20d60d9973
commit
619b6e18fc
11 changed files with 227 additions and 83 deletions
|
|
@ -6,7 +6,7 @@
|
|||
* Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
|
||||
* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2001 MIPS Technologies, Inc.
|
||||
* Copyright (C) 2002 Maciej W. Rozycki
|
||||
* Copyright (C) 2002, 2007 Maciej W. Rozycki
|
||||
*/
|
||||
#include <linux/init.h>
|
||||
|
||||
|
|
@ -471,7 +471,13 @@ NESTED(nmi_handler, PT_SIZE, sp)
|
|||
jr k0
|
||||
rfe
|
||||
#else
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
LONG_ADDIU k0, 4 /* stall on $k0 */
|
||||
#else
|
||||
.set at=v1
|
||||
LONG_ADDIU k0, 4
|
||||
.set noat
|
||||
#endif
|
||||
MTC0 k0, CP0_EPC
|
||||
/* I hope three instructions between MTC0 and ERET are enough... */
|
||||
ori k1, _THREAD_MASK
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
*
|
||||
* Copyright (C) 1998, 1999 Ralf Baechle
|
||||
* Copyright (C) 1999 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*/
|
||||
#include <linux/errno.h>
|
||||
#include <asm/asm.h>
|
||||
|
|
@ -52,9 +53,12 @@
|
|||
#define UNIT(unit) ((unit)*NBYTES)
|
||||
|
||||
#define ADDC(sum,reg) \
|
||||
.set push; \
|
||||
.set noat; \
|
||||
ADD sum, reg; \
|
||||
sltu v1, sum, reg; \
|
||||
ADD sum, v1
|
||||
ADD sum, v1; \
|
||||
.set pop
|
||||
|
||||
#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
|
||||
LOAD _t0, (offset + UNIT(0))(src); \
|
||||
|
|
@ -178,8 +182,10 @@ move_128bytes:
|
|||
CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
|
||||
CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
|
||||
LONG_SUBU t8, t8, 0x01
|
||||
bnez t8, move_128bytes
|
||||
.set reorder /* DADDI_WAR */
|
||||
PTR_ADDU src, src, 0x80
|
||||
bnez t8, move_128bytes
|
||||
.set noreorder
|
||||
|
||||
1:
|
||||
beqz t2, 1f
|
||||
|
|
@ -208,8 +214,10 @@ end_words:
|
|||
lw t0, (src)
|
||||
LONG_SUBU t8, t8, 0x1
|
||||
ADDC(sum, t0)
|
||||
bnez t8, end_words
|
||||
.set reorder /* DADDI_WAR */
|
||||
PTR_ADDU src, src, 0x4
|
||||
bnez t8, end_words
|
||||
.set noreorder
|
||||
|
||||
/* unknown src alignment and < 8 bytes to go */
|
||||
small_csumcpy:
|
||||
|
|
@ -246,6 +254,8 @@ small_csumcpy:
|
|||
1: ADDC(sum, t1)
|
||||
|
||||
/* fold checksum */
|
||||
.set push
|
||||
.set noat
|
||||
#ifdef USE_DOUBLE
|
||||
dsll32 v1, sum, 0
|
||||
daddu sum, v1
|
||||
|
|
@ -266,6 +276,7 @@ small_csumcpy:
|
|||
srl sum, sum, 8
|
||||
or sum, v1
|
||||
andi sum, 0xffff
|
||||
.set pop
|
||||
1:
|
||||
.set reorder
|
||||
/* Add the passed partial csum. */
|
||||
|
|
@ -373,7 +384,11 @@ small_csumcpy:
|
|||
|
||||
#define ADDRMASK (NBYTES-1)
|
||||
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
.set noat
|
||||
#else
|
||||
.set at=v1
|
||||
#endif
|
||||
|
||||
LEAF(__csum_partial_copy_user)
|
||||
PTR_ADDU AT, src, len /* See (1) above. */
|
||||
|
|
@ -441,8 +456,10 @@ EXC( STORE t6, UNIT(6)(dst), s_exc)
|
|||
ADDC(sum, t6)
|
||||
EXC( STORE t7, UNIT(7)(dst), s_exc)
|
||||
ADDC(sum, t7)
|
||||
bgez len, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 8*NBYTES
|
||||
bgez len, 1b
|
||||
.set noreorder
|
||||
ADD len, 8*NBYTES # revert len (see above)
|
||||
|
||||
/*
|
||||
|
|
@ -471,8 +488,10 @@ EXC( STORE t2, UNIT(2)(dst), s_exc)
|
|||
ADDC(sum, t2)
|
||||
EXC( STORE t3, UNIT(3)(dst), s_exc)
|
||||
ADDC(sum, t3)
|
||||
beqz len, done
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 4*NBYTES
|
||||
beqz len, done
|
||||
.set noreorder
|
||||
less_than_4units:
|
||||
/*
|
||||
* rem = len % NBYTES
|
||||
|
|
@ -485,8 +504,10 @@ EXC( LOAD t0, 0(src), l_exc)
|
|||
SUB len, len, NBYTES
|
||||
EXC( STORE t0, 0(dst), s_exc)
|
||||
ADDC(sum, t0)
|
||||
bne rem, len, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, NBYTES
|
||||
bne rem, len, 1b
|
||||
.set noreorder
|
||||
|
||||
/*
|
||||
* src and dst are aligned, need to copy rem bytes (rem < NBYTES)
|
||||
|
|
@ -572,8 +593,10 @@ EXC( STORE t2, UNIT(2)(dst), s_exc)
|
|||
ADDC(sum, t2)
|
||||
EXC( STORE t3, UNIT(3)(dst), s_exc)
|
||||
ADDC(sum, t3)
|
||||
bne len, rem, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 4*NBYTES
|
||||
bne len, rem, 1b
|
||||
.set noreorder
|
||||
|
||||
cleanup_src_unaligned:
|
||||
beqz len, done
|
||||
|
|
@ -587,8 +610,10 @@ EXC( LDREST t0, REST(0)(src), l_exc_copy)
|
|||
SUB len, len, NBYTES
|
||||
EXC( STORE t0, 0(dst), s_exc)
|
||||
ADDC(sum, t0)
|
||||
bne len, rem, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, NBYTES
|
||||
bne len, rem, 1b
|
||||
.set noreorder
|
||||
|
||||
copy_bytes_checklen:
|
||||
beqz len, done
|
||||
|
|
@ -631,6 +656,8 @@ copy_bytes_done:
|
|||
ADDC(sum, t2)
|
||||
done:
|
||||
/* fold checksum */
|
||||
.set push
|
||||
.set noat
|
||||
#ifdef USE_DOUBLE
|
||||
dsll32 v1, sum, 0
|
||||
daddu sum, v1
|
||||
|
|
@ -651,6 +678,7 @@ done:
|
|||
srl sum, sum, 8
|
||||
or sum, v1
|
||||
andi sum, 0xffff
|
||||
.set pop
|
||||
1:
|
||||
.set reorder
|
||||
ADDC(sum, psum)
|
||||
|
|
@ -678,8 +706,10 @@ EXC( lbu t1, 0(src), l_exc)
|
|||
SLLV t1, t1, t2
|
||||
addu t2, SHIFT_INC
|
||||
ADDC(sum, t1)
|
||||
bne src, t0, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 1
|
||||
bne src, t0, 1b
|
||||
.set noreorder
|
||||
l_exc:
|
||||
LOAD t0, TI_TASK($28)
|
||||
nop
|
||||
|
|
@ -697,12 +727,22 @@ l_exc:
|
|||
* Clear len bytes starting at dst. Can't call __bzero because it
|
||||
* might modify len. An inefficient loop for these rare times...
|
||||
*/
|
||||
beqz len, done
|
||||
.set reorder /* DADDI_WAR */
|
||||
SUB src, len, 1
|
||||
beqz len, done
|
||||
.set noreorder
|
||||
1: sb zero, 0(dst)
|
||||
ADD dst, dst, 1
|
||||
.set push
|
||||
.set noat
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
bnez src, 1b
|
||||
SUB src, src, 1
|
||||
#else
|
||||
li v1, 1
|
||||
bnez src, 1b
|
||||
SUB src, src, v1
|
||||
#endif
|
||||
li v1, -EFAULT
|
||||
b done
|
||||
sw v1, (errptr)
|
||||
|
|
@ -712,4 +752,5 @@ s_exc:
|
|||
li v1, -EFAULT
|
||||
jr ra
|
||||
sw v1, (errptr)
|
||||
.set pop
|
||||
END(__csum_partial_copy_user)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
* Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2002 Broadcom, Inc.
|
||||
* memcpy/copy_user author: Mark Vandevoorde
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*
|
||||
* Mnemonic names for arguments to memcpy/__copy_user
|
||||
*/
|
||||
|
|
@ -175,7 +176,11 @@
|
|||
|
||||
.text
|
||||
.set noreorder
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
.set noat
|
||||
#else
|
||||
.set at=v1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* A combined memcpy/__copy_user
|
||||
|
|
@ -268,8 +273,10 @@ EXC( LOAD t3, UNIT(3)(src), l_exc_copy)
|
|||
STORE t1, UNIT(1)(dst)
|
||||
STORE t2, UNIT(2)(dst)
|
||||
STORE t3, UNIT(3)(dst)
|
||||
beqz len, done
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 4*NBYTES
|
||||
beqz len, done
|
||||
.set noreorder
|
||||
less_than_4units:
|
||||
/*
|
||||
* rem = len % NBYTES
|
||||
|
|
@ -281,8 +288,10 @@ EXC( LOAD t0, 0(src), l_exc)
|
|||
ADD src, src, NBYTES
|
||||
SUB len, len, NBYTES
|
||||
STORE t0, 0(dst)
|
||||
bne rem, len, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, NBYTES
|
||||
bne rem, len, 1b
|
||||
.set noreorder
|
||||
|
||||
/*
|
||||
* src and dst are aligned, need to copy rem bytes (rem < NBYTES)
|
||||
|
|
@ -361,8 +370,10 @@ EXC( LDREST t3, REST(3)(src), l_exc_copy)
|
|||
STORE t2, UNIT(2)(dst)
|
||||
STORE t3, UNIT(3)(dst)
|
||||
PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed)
|
||||
bne len, rem, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 4*NBYTES
|
||||
bne len, rem, 1b
|
||||
.set noreorder
|
||||
|
||||
cleanup_src_unaligned:
|
||||
beqz len, done
|
||||
|
|
@ -375,8 +386,10 @@ EXC( LDREST t0, REST(0)(src), l_exc_copy)
|
|||
ADD src, src, NBYTES
|
||||
SUB len, len, NBYTES
|
||||
STORE t0, 0(dst)
|
||||
bne len, rem, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, NBYTES
|
||||
bne len, rem, 1b
|
||||
.set noreorder
|
||||
|
||||
copy_bytes_checklen:
|
||||
beqz len, done
|
||||
|
|
@ -424,8 +437,10 @@ l_exc_copy:
|
|||
EXC( lb t1, 0(src), l_exc)
|
||||
ADD src, src, 1
|
||||
sb t1, 0(dst) # can't fault -- we're copy_from_user
|
||||
bne src, t0, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 1
|
||||
bne src, t0, 1b
|
||||
.set noreorder
|
||||
l_exc:
|
||||
LOAD t0, TI_TASK($28)
|
||||
nop
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
* Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2002 Broadcom, Inc.
|
||||
* memcpy/copy_user author: Mark Vandevoorde
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*
|
||||
* Mnemonic names for arguments to memcpy/__copy_user
|
||||
*/
|
||||
|
|
@ -175,7 +176,11 @@
|
|||
|
||||
.text
|
||||
.set noreorder
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
.set noat
|
||||
#else
|
||||
.set at=v1
|
||||
#endif
|
||||
|
||||
/*
|
||||
* A combined memcpy/__copy_user
|
||||
|
|
@ -271,8 +276,10 @@ EXC( STORE t0, UNIT(0)(dst), s_exc_p4u)
|
|||
EXC( STORE t1, UNIT(1)(dst), s_exc_p3u)
|
||||
EXC( STORE t2, UNIT(2)(dst), s_exc_p2u)
|
||||
EXC( STORE t3, UNIT(3)(dst), s_exc_p1u)
|
||||
beqz len, done
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 4*NBYTES
|
||||
beqz len, done
|
||||
.set noreorder
|
||||
less_than_4units:
|
||||
/*
|
||||
* rem = len % NBYTES
|
||||
|
|
@ -284,8 +291,10 @@ EXC( LOAD t0, 0(src), l_exc)
|
|||
ADD src, src, NBYTES
|
||||
SUB len, len, NBYTES
|
||||
EXC( STORE t0, 0(dst), s_exc_p1u)
|
||||
bne rem, len, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, NBYTES
|
||||
bne rem, len, 1b
|
||||
.set noreorder
|
||||
|
||||
/*
|
||||
* src and dst are aligned, need to copy rem bytes (rem < NBYTES)
|
||||
|
|
@ -364,8 +373,10 @@ EXC( STORE t1, UNIT(1)(dst), s_exc_p3u)
|
|||
EXC( STORE t2, UNIT(2)(dst), s_exc_p2u)
|
||||
EXC( STORE t3, UNIT(3)(dst), s_exc_p1u)
|
||||
PREF( 1, 9*32(dst) ) # 1 is PREF_STORE (not streamed)
|
||||
bne len, rem, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 4*NBYTES
|
||||
bne len, rem, 1b
|
||||
.set noreorder
|
||||
|
||||
cleanup_src_unaligned:
|
||||
beqz len, done
|
||||
|
|
@ -378,8 +389,10 @@ EXC( LDREST t0, REST(0)(src), l_exc_copy)
|
|||
ADD src, src, NBYTES
|
||||
SUB len, len, NBYTES
|
||||
EXC( STORE t0, 0(dst), s_exc_p1u)
|
||||
bne len, rem, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, NBYTES
|
||||
bne len, rem, 1b
|
||||
.set noreorder
|
||||
|
||||
copy_bytes_checklen:
|
||||
beqz len, done
|
||||
|
|
@ -427,8 +440,10 @@ l_exc_copy:
|
|||
EXC( lb t1, 0(src), l_exc)
|
||||
ADD src, src, 1
|
||||
sb t1, 0(dst) # can't fault -- we're copy_from_user
|
||||
bne src, t0, 1b
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD dst, dst, 1
|
||||
bne src, t0, 1b
|
||||
.set noreorder
|
||||
l_exc:
|
||||
LOAD t0, TI_TASK($28)
|
||||
nop
|
||||
|
|
@ -446,20 +461,33 @@ l_exc:
|
|||
* Clear len bytes starting at dst. Can't call __bzero because it
|
||||
* might modify len. An inefficient loop for these rare times...
|
||||
*/
|
||||
beqz len, done
|
||||
.set reorder /* DADDI_WAR */
|
||||
SUB src, len, 1
|
||||
beqz len, done
|
||||
.set noreorder
|
||||
1: sb zero, 0(dst)
|
||||
ADD dst, dst, 1
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
bnez src, 1b
|
||||
SUB src, src, 1
|
||||
#else
|
||||
.set push
|
||||
.set noat
|
||||
li v1, 1
|
||||
bnez src, 1b
|
||||
SUB src, src, v1
|
||||
.set pop
|
||||
#endif
|
||||
jr ra
|
||||
nop
|
||||
|
||||
|
||||
#define SEXC(n) \
|
||||
.set reorder; /* DADDI_WAR */ \
|
||||
s_exc_p ## n ## u: \
|
||||
ADD len, len, n*NBYTES; \
|
||||
jr ra; \
|
||||
ADD len, len, n*NBYTES
|
||||
.set noreorder
|
||||
|
||||
SEXC(8)
|
||||
SEXC(7)
|
||||
|
|
@ -471,8 +499,10 @@ SEXC(2)
|
|||
SEXC(1)
|
||||
|
||||
s_exc_p1:
|
||||
jr ra
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD len, len, 1
|
||||
jr ra
|
||||
.set noreorder
|
||||
s_exc:
|
||||
jr ra
|
||||
nop
|
||||
|
|
@ -502,8 +532,10 @@ r_end_bytes:
|
|||
SUB a2, a2, 0x1
|
||||
sb t0, -1(a0)
|
||||
SUB a1, a1, 0x1
|
||||
bnez a2, r_end_bytes
|
||||
.set reorder /* DADDI_WAR */
|
||||
SUB a0, a0, 0x1
|
||||
bnez a2, r_end_bytes
|
||||
.set noreorder
|
||||
|
||||
r_out:
|
||||
jr ra
|
||||
|
|
@ -514,8 +546,10 @@ r_end_bytes_up:
|
|||
SUB a2, a2, 0x1
|
||||
sb t0, (a0)
|
||||
ADD a1, a1, 0x1
|
||||
bnez a2, r_end_bytes_up
|
||||
.set reorder /* DADDI_WAR */
|
||||
ADD a0, a0, 0x1
|
||||
bnez a2, r_end_bytes_up
|
||||
.set noreorder
|
||||
|
||||
jr ra
|
||||
move a2, zero
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*
|
||||
* Copyright (C) 1998, 1999, 2000 by Ralf Baechle
|
||||
* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*/
|
||||
#include <asm/asm.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
|
|
@ -74,8 +75,16 @@ FEXPORT(__bzero)
|
|||
bnez t0, small_memset
|
||||
andi t0, a0, LONGMASK /* aligned? */
|
||||
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
beqz t0, 1f
|
||||
PTR_SUBU t0, LONGSIZE /* alignment in bytes */
|
||||
#else
|
||||
.set noat
|
||||
li AT, LONGSIZE
|
||||
beqz t0, 1f
|
||||
PTR_SUBU t0, AT /* alignment in bytes */
|
||||
.set at
|
||||
#endif
|
||||
|
||||
#ifdef __MIPSEB__
|
||||
EX(LONG_S_L, a1, (a0), first_fixup) /* make word/dword aligned */
|
||||
|
|
@ -106,7 +115,7 @@ memset_partial:
|
|||
.set noat
|
||||
LONG_SRL AT, t0, 1
|
||||
PTR_SUBU t1, AT
|
||||
.set noat
|
||||
.set at
|
||||
#endif
|
||||
jr t1
|
||||
PTR_ADDU a0, t0 /* dest ptr */
|
||||
|
|
|
|||
|
|
@ -41,9 +41,9 @@ FEXPORT(__strncpy_from_user_nocheck_asm)
|
|||
beqz t0, 2f
|
||||
sb t0, (a0)
|
||||
PTR_ADDIU v0, 1
|
||||
bne v0, a2, 1b
|
||||
PTR_ADDIU a0, 1
|
||||
.set reorder
|
||||
PTR_ADDIU a0, 1
|
||||
bne v0, a2, 1b
|
||||
2: PTR_ADDU t0, a1, v0
|
||||
xor t0, a1
|
||||
bltz t0, fault
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
* for more details.
|
||||
*
|
||||
* Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org)
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*/
|
||||
#include <linux/init.h>
|
||||
#include <linux/kernel.h>
|
||||
|
|
@ -12,6 +13,7 @@
|
|||
#include <linux/module.h>
|
||||
#include <linux/proc_fs.h>
|
||||
|
||||
#include <asm/bugs.h>
|
||||
#include <asm/cacheops.h>
|
||||
#include <asm/inst.h>
|
||||
#include <asm/io.h>
|
||||
|
|
@ -255,64 +257,58 @@ static inline void build_store_reg(int reg)
|
|||
__build_store_reg(reg);
|
||||
}
|
||||
|
||||
static inline void build_addiu_a2_a0(unsigned long offset)
|
||||
static inline void build_addiu_rt_rs(unsigned int rt, unsigned int rs,
|
||||
unsigned long offset)
|
||||
{
|
||||
union mips_instruction mi;
|
||||
|
||||
BUG_ON(offset > 0x7fff);
|
||||
|
||||
mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
|
||||
mi.i_format.rs = 4; /* $a0 */
|
||||
mi.i_format.rt = 6; /* $a2 */
|
||||
if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) {
|
||||
mi.i_format.opcode = addiu_op;
|
||||
mi.i_format.rs = 0; /* $zero */
|
||||
mi.i_format.rt = 25; /* $t9 */
|
||||
mi.i_format.simmediate = offset;
|
||||
|
||||
emit_instruction(mi);
|
||||
|
||||
mi.r_format.opcode = spec_op;
|
||||
mi.r_format.rs = rs;
|
||||
mi.r_format.rt = 25; /* $t9 */
|
||||
mi.r_format.rd = rt;
|
||||
mi.r_format.re = 0;
|
||||
mi.r_format.func = daddu_op;
|
||||
} else {
|
||||
mi.i_format.opcode = cpu_has_64bit_gp_regs ?
|
||||
daddiu_op : addiu_op;
|
||||
mi.i_format.rs = rs;
|
||||
mi.i_format.rt = rt;
|
||||
mi.i_format.simmediate = offset;
|
||||
}
|
||||
emit_instruction(mi);
|
||||
}
|
||||
|
||||
static inline void build_addiu_a2_a0(unsigned long offset)
|
||||
{
|
||||
build_addiu_rt_rs(6, 4, offset); /* $a2, $a0, offset */
|
||||
}
|
||||
|
||||
static inline void build_addiu_a2(unsigned long offset)
|
||||
{
|
||||
union mips_instruction mi;
|
||||
|
||||
BUG_ON(offset > 0x7fff);
|
||||
|
||||
mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
|
||||
mi.i_format.rs = 6; /* $a2 */
|
||||
mi.i_format.rt = 6; /* $a2 */
|
||||
mi.i_format.simmediate = offset;
|
||||
|
||||
emit_instruction(mi);
|
||||
build_addiu_rt_rs(6, 6, offset); /* $a2, $a2, offset */
|
||||
}
|
||||
|
||||
static inline void build_addiu_a1(unsigned long offset)
|
||||
{
|
||||
union mips_instruction mi;
|
||||
|
||||
BUG_ON(offset > 0x7fff);
|
||||
|
||||
mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
|
||||
mi.i_format.rs = 5; /* $a1 */
|
||||
mi.i_format.rt = 5; /* $a1 */
|
||||
mi.i_format.simmediate = offset;
|
||||
build_addiu_rt_rs(5, 5, offset); /* $a1, $a1, offset */
|
||||
|
||||
load_offset -= offset;
|
||||
|
||||
emit_instruction(mi);
|
||||
}
|
||||
|
||||
static inline void build_addiu_a0(unsigned long offset)
|
||||
{
|
||||
union mips_instruction mi;
|
||||
|
||||
BUG_ON(offset > 0x7fff);
|
||||
|
||||
mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
|
||||
mi.i_format.rs = 4; /* $a0 */
|
||||
mi.i_format.rt = 4; /* $a0 */
|
||||
mi.i_format.simmediate = offset;
|
||||
build_addiu_rt_rs(4, 4, offset); /* $a0, $a0, offset */
|
||||
|
||||
store_offset -= offset;
|
||||
|
||||
emit_instruction(mi);
|
||||
}
|
||||
|
||||
static inline void build_bne(unsigned int *dest)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
* Synthesize TLB refill handlers at runtime.
|
||||
*
|
||||
* Copyright (C) 2004,2005,2006 by Thiemo Seufer
|
||||
* Copyright (C) 2005 Maciej W. Rozycki
|
||||
* Copyright (C) 2005, 2007 Maciej W. Rozycki
|
||||
* Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
|
||||
*
|
||||
* ... and the days got worse and worse and now you see
|
||||
|
|
@ -27,6 +27,7 @@
|
|||
#include <linux/string.h>
|
||||
#include <linux/init.h>
|
||||
|
||||
#include <asm/bugs.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/mmu_context.h>
|
||||
|
|
@ -293,7 +294,7 @@ static void __init build_insn(u32 **buf, enum opcode opc, ...)
|
|||
break;
|
||||
}
|
||||
|
||||
if (!ip)
|
||||
if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
|
||||
panic("Unsupported TLB synthesizer instruction %d", opc);
|
||||
|
||||
op = ip->match;
|
||||
|
|
@ -525,23 +526,33 @@ L_LA(_r3000_write_probe_fail)
|
|||
#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
|
||||
#define i_ehb(buf) i_sll(buf, 0, 0, 3)
|
||||
|
||||
#ifdef CONFIG_64BIT
|
||||
static __init int __maybe_unused in_compat_space_p(long addr)
|
||||
{
|
||||
/* Is this address in 32bit compat space? */
|
||||
#ifdef CONFIG_64BIT
|
||||
return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
|
||||
#else
|
||||
return 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
static __init int __maybe_unused rel_highest(long val)
|
||||
{
|
||||
#ifdef CONFIG_64BIT
|
||||
return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static __init int __maybe_unused rel_higher(long val)
|
||||
{
|
||||
#ifdef CONFIG_64BIT
|
||||
return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
|
||||
}
|
||||
#else
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static __init int rel_hi(long val)
|
||||
{
|
||||
|
|
@ -555,7 +566,6 @@ static __init int rel_lo(long val)
|
|||
|
||||
static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr)
|
||||
{
|
||||
#ifdef CONFIG_64BIT
|
||||
if (!in_compat_space_p(addr)) {
|
||||
i_lui(buf, rs, rel_highest(addr));
|
||||
if (rel_higher(addr))
|
||||
|
|
@ -567,16 +577,18 @@ static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr)
|
|||
} else
|
||||
i_dsll32(buf, rs, rs, 0);
|
||||
} else
|
||||
#endif
|
||||
i_lui(buf, rs, rel_hi(addr));
|
||||
}
|
||||
|
||||
static __init void __maybe_unused i_LA(u32 **buf, unsigned int rs,
|
||||
long addr)
|
||||
static __init void __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr)
|
||||
{
|
||||
i_LA_mostly(buf, rs, addr);
|
||||
if (rel_lo(addr))
|
||||
i_ADDIU(buf, rs, rs, rel_lo(addr));
|
||||
if (rel_lo(addr)) {
|
||||
if (!in_compat_space_p(addr))
|
||||
i_daddiu(buf, rs, rs, rel_lo(addr));
|
||||
else
|
||||
i_addiu(buf, rs, rs, rel_lo(addr));
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1085,6 +1097,9 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
|
|||
} else {
|
||||
i_LA_mostly(p, ptr, modd);
|
||||
il_b(p, r, label_vmalloc_done);
|
||||
if (in_compat_space_p(modd))
|
||||
i_addiu(p, ptr, ptr, rel_lo(modd));
|
||||
else
|
||||
i_daddiu(p, ptr, ptr, rel_lo(modd));
|
||||
}
|
||||
|
||||
|
|
@ -1106,6 +1121,9 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
|
|||
} else {
|
||||
i_LA_mostly(p, ptr, swpd);
|
||||
il_b(p, r, label_vmalloc_done);
|
||||
if (in_compat_space_p(swpd))
|
||||
i_addiu(p, ptr, ptr, rel_lo(swpd));
|
||||
else
|
||||
i_daddiu(p, ptr, ptr, rel_lo(swpd));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ static inline void __delay(unsigned long loops)
|
|||
" .set reorder \n"
|
||||
: "=r" (loops)
|
||||
: "0" (loops));
|
||||
else if (sizeof(long) == 8)
|
||||
else if (sizeof(long) == 8 && !DADDI_WAR)
|
||||
__asm__ __volatile__ (
|
||||
" .set noreorder \n"
|
||||
" .align 3 \n"
|
||||
|
|
@ -37,6 +37,15 @@ static inline void __delay(unsigned long loops)
|
|||
" .set reorder \n"
|
||||
: "=r" (loops)
|
||||
: "0" (loops));
|
||||
else if (sizeof(long) == 8 && DADDI_WAR)
|
||||
__asm__ __volatile__ (
|
||||
" .set noreorder \n"
|
||||
" .align 3 \n"
|
||||
"1: bnez %0, 1b \n"
|
||||
" dsubu %0, %2 \n"
|
||||
" .set reorder \n"
|
||||
: "=r" (loops)
|
||||
: "0" (loops), "r" (1));
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
* Copyright (C) 1994, 95, 96, 99, 2001 Ralf Baechle
|
||||
* Copyright (C) 1994, 1995, 1996 Paul M. Antoine.
|
||||
* Copyright (C) 1999 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*/
|
||||
#ifndef _ASM_STACKFRAME_H
|
||||
#define _ASM_STACKFRAME_H
|
||||
|
|
@ -145,8 +146,16 @@
|
|||
.set reorder
|
||||
/* Called from user mode, new stack. */
|
||||
get_saved_sp
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
8: move k0, sp
|
||||
PTR_SUBU sp, k1, PT_SIZE
|
||||
#else
|
||||
.set at=k0
|
||||
8: PTR_SUBU k1, PT_SIZE
|
||||
.set noat
|
||||
move k0, sp
|
||||
move sp, k1
|
||||
#endif
|
||||
LONG_S k0, PT_R29(sp)
|
||||
LONG_S $3, PT_R3(sp)
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*
|
||||
* Copyright (C) 1996, 1997, 1998, 1999, 2000, 03, 04 by Ralf Baechle
|
||||
* Copyright (C) 1999, 2000 Silicon Graphics, Inc.
|
||||
* Copyright (C) 2007 Maciej W. Rozycki
|
||||
*/
|
||||
#ifndef _ASM_UACCESS_H
|
||||
#define _ASM_UACCESS_H
|
||||
|
|
@ -387,6 +388,12 @@ extern void __put_user_unknown(void);
|
|||
"jal\t" #destination "\n\t"
|
||||
#endif
|
||||
|
||||
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
|
||||
#define DADDI_SCRATCH "$0"
|
||||
#else
|
||||
#define DADDI_SCRATCH "$3"
|
||||
#endif
|
||||
|
||||
extern size_t __copy_user(void *__to, const void *__from, size_t __n);
|
||||
|
||||
#define __invoke_copy_to_user(to, from, n) \
|
||||
|
|
@ -403,7 +410,7 @@ extern size_t __copy_user(void *__to, const void *__from, size_t __n);
|
|||
: "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
|
||||
: \
|
||||
: "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31", \
|
||||
"memory"); \
|
||||
DADDI_SCRATCH, "memory"); \
|
||||
__cu_len_r; \
|
||||
})
|
||||
|
||||
|
|
@ -512,7 +519,7 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
|
|||
: "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
|
||||
: \
|
||||
: "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31", \
|
||||
"memory"); \
|
||||
DADDI_SCRATCH, "memory"); \
|
||||
__cu_len_r; \
|
||||
})
|
||||
|
||||
|
|
@ -535,7 +542,7 @@ extern size_t __copy_user_inatomic(void *__to, const void *__from, size_t __n);
|
|||
: "+r" (__cu_to_r), "+r" (__cu_from_r), "+r" (__cu_len_r) \
|
||||
: \
|
||||
: "$8", "$9", "$10", "$11", "$12", "$15", "$24", "$31", \
|
||||
"memory"); \
|
||||
DADDI_SCRATCH, "memory"); \
|
||||
__cu_len_r; \
|
||||
})
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue