Commit 99d5040e authored by Max Filippov's avatar Max Filippov Committed by Chris Zankel
Browse files

xtensa: keep a3 and excsave1 on entry to exception handlers



Based on the SMP patch by Joe Taylor and subsequent fixes.
Preserve exception table pointer (normally stored in excsave1 SR) as it
cannot be easily restored in SMP environment.

Signed-off-by: default avatarMax Filippov <jcmvbkbc@gmail.com>
Signed-off-by: default avatarChris Zankel <chris@zankel.net>
parent 16c5becf
......@@ -82,6 +82,7 @@
#define PS_CALLINC_SHIFT 16
#define PS_CALLINC_MASK 0x00030000
#define PS_OWB_SHIFT 8
#define PS_OWB_WIDTH 4
#define PS_OWB_MASK 0x00000F00
#define PS_RING_SHIFT 6
#define PS_RING_MASK 0x000000C0
......
......@@ -146,9 +146,9 @@
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -171,7 +171,6 @@ ENTRY(fast_unaligned)
s32i a8, a2, PT_AREG8
rsr a0, depc
xsr a3, excsave1
s32i a0, a2, PT_AREG2
s32i a3, a2, PT_AREG3
......
......@@ -32,9 +32,9 @@
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -225,9 +225,9 @@ ENDPROC(coprocessor_restore)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -245,7 +245,6 @@ ENTRY(fast_coprocessor)
/* Save remaining registers a1-a3 and SAR */
xsr a3, excsave1
s32i a3, a2, PT_AREG3
rsr a3, sar
s32i a1, a2, PT_AREG1
......
......@@ -91,9 +91,9 @@
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original value in depc
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave1: a3
* excsave1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -109,9 +109,8 @@
ENTRY(user_exception)
/* Save a2, a3, and depc, restore excsave_1 and set SP. */
/* Save a1, a2, a3, and set SP. */
xsr a3, excsave1
rsr a0, depc
s32i a1, a2, PT_AREG1
s32i a0, a2, PT_AREG2
......@@ -237,9 +236,9 @@ ENDPROC(user_exception)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -255,9 +254,8 @@ ENDPROC(user_exception)
ENTRY(kernel_exception)
/* Save a0, a2, a3, DEPC and set SP. */
/* Save a1, a2, a3, and set SP. */
xsr a3, excsave1 # restore a3, excsave_1
rsr a0, depc # get a2
s32i a1, a2, PT_AREG1
s32i a0, a2, PT_AREG2
......@@ -408,7 +406,7 @@ common_exception:
* exception handler and call the exception handler.
*/
movi a4, exc_table
rsr a4, excsave1
mov a6, a1 # pass stack frame
mov a7, a0 # pass EXCCAUSE
addx4 a4, a0, a4
......@@ -832,9 +830,9 @@ ENDPROC(unrecoverable_exception)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -857,18 +855,16 @@ ENTRY(fast_alloca)
rsr a0, depc # get a2
s32i a4, a2, PT_AREG4 # save a4 and
s32i a3, a2, PT_AREG3
s32i a0, a2, PT_AREG2 # a2 to stack
/* Exit critical section. */
movi a0, 0
rsr a3, excsave1
s32i a0, a3, EXC_TABLE_FIXUP
/* Restore a3, excsave_1 */
xsr a3, excsave1 # make sure excsave_1 is valid for dbl.
rsr a4, epc1 # get exception address
s32i a3, a2, PT_AREG3 # save a3 to stack
#ifdef ALLOCA_EXCEPTION_IN_IRAM
#error iram not supported
......@@ -1007,9 +1003,9 @@ ENDPROC(fast_alloca)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*/
ENTRY(fast_syscall_kernel)
......@@ -1056,7 +1052,6 @@ ENTRY(fast_syscall_unrecoverable)
l32i a0, a2, PT_AREG0 # restore a0
xsr a2, depc # restore a2, depc
rsr a3, excsave1
wsr a0, excsave1
movi a0, unrecoverable_exception
......@@ -1078,10 +1073,10 @@ ENDPROC(fast_syscall_unrecoverable)
* a0: a2 (syscall-nr), original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in a0 and DEPC
* a3: dispatch table, original in excsave_1
* a3: a3
* a4..a15: unchanged
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -1114,8 +1109,6 @@ ENDPROC(fast_syscall_unrecoverable)
ENTRY(fast_syscall_xtensa)
xsr a3, excsave1 # restore a3, excsave1
s32i a7, a2, PT_AREG7 # we need an additional register
movi a7, 4 # sizeof(unsigned int)
access_ok a3, a7, a0, a2, .Leac # a0: scratch reg, a2: sp
......@@ -1178,9 +1171,9 @@ ENDPROC(fast_syscall_xtensa)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* Note: We assume the stack pointer is EXC_TABLE_KSTK in the fixup handler.
*/
......@@ -1189,15 +1182,16 @@ ENTRY(fast_syscall_spill_registers)
/* Register a FIXUP handler (pass current wb as a parameter) */
xsr a3, excsave1
movi a0, fast_syscall_spill_registers_fixup
s32i a0, a3, EXC_TABLE_FIXUP
rsr a0, windowbase
s32i a0, a3, EXC_TABLE_PARAM
xsr a3, excsave1 # restore a3 and excsave_1
/* Save a3 and SAR on stack. */
/* Save a3, a4 and SAR on stack. */
rsr a0, sar
xsr a3, excsave1 # restore a3 and excsave_1
s32i a3, a2, PT_AREG3
s32i a4, a2, PT_AREG4
s32i a0, a2, PT_AREG5 # store SAR to PT_AREG5
......@@ -1251,14 +1245,14 @@ fast_syscall_spill_registers_fixup:
* in WS, so that the exception handlers save them to the task stack.
*/
rsr a3, excsave1 # get spill-mask
xsr a3, excsave1 # get spill-mask
slli a2, a3, 1 # shift left by one
slli a3, a2, 32-WSBITS
src a2, a2, a3 # a1 = xxwww1yyxxxwww1yy......
wsr a2, windowstart # set corrected windowstart
movi a3, exc_table
rsr a3, excsave1
l32i a2, a3, EXC_TABLE_DOUBLE_SAVE # restore a2
l32i a3, a3, EXC_TABLE_PARAM # original WB (in user task)
......@@ -1295,7 +1289,7 @@ fast_syscall_spill_registers_fixup:
/* Jump to the exception handler. */
movi a3, exc_table
rsr a3, excsave1
rsr a0, exccause
addx4 a0, a0, a3 # find entry in table
l32i a0, a0, EXC_TABLE_FAST_USER # load handler
......@@ -1312,6 +1306,7 @@ fast_syscall_spill_registers_fixup_return:
xsr a3, excsave1
movi a2, fast_syscall_spill_registers_fixup
s32i a2, a3, EXC_TABLE_FIXUP
s32i a0, a3, EXC_TABLE_DOUBLE_SAVE
rsr a2, windowbase
s32i a2, a3, EXC_TABLE_PARAM
l32i a2, a3, EXC_TABLE_KSTK
......@@ -1323,11 +1318,6 @@ fast_syscall_spill_registers_fixup_return:
wsr a3, windowbase
rsync
/* Restore a3 and return. */
movi a3, exc_table
xsr a3, excsave1
rfde
......@@ -1514,9 +1504,8 @@ ENTRY(_spill_registers)
movi a0, 0
movi a3, exc_table
rsr a3, excsave1
l32i a1, a3, EXC_TABLE_KSTK
wsr a3, excsave1
movi a4, (1 << PS_WOE_BIT) | LOCKLEVEL
wsr a4, ps
......@@ -1560,9 +1549,9 @@ ENDPROC(fast_second_level_miss_double_kernel)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -1570,9 +1559,10 @@ ENDPROC(fast_second_level_miss_double_kernel)
ENTRY(fast_second_level_miss)
/* Save a1. Note: we don't expect a double exception. */
/* Save a1 and a3. Note: we don't expect a double exception. */
s32i a1, a2, PT_AREG1
s32i a3, a2, PT_AREG3
/* We need to map the page of PTEs for the user task. Find
* the pointer to that page. Also, it's possible for tsk->mm
......@@ -1594,9 +1584,6 @@ ENTRY(fast_second_level_miss)
l32i a0, a1, TASK_MM # tsk->mm
beqz a0, 9f
/* We deliberately destroy a3 that holds the exception table. */
8: rsr a3, excvaddr # fault address
_PGD_OFFSET(a0, a3, a1)
l32i a0, a0, 0 # read pmdval
......@@ -1647,7 +1634,7 @@ ENTRY(fast_second_level_miss)
/* Exit critical section. */
4: movi a3, exc_table # restore a3
4: rsr a3, excsave1
movi a0, 0
s32i a0, a3, EXC_TABLE_FIXUP
......@@ -1655,8 +1642,8 @@ ENTRY(fast_second_level_miss)
l32i a0, a2, PT_AREG0
l32i a1, a2, PT_AREG1
l32i a3, a2, PT_AREG3
l32i a2, a2, PT_DEPC
xsr a3, excsave1
bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
......@@ -1743,11 +1730,8 @@ ENTRY(fast_second_level_miss)
2: /* Invalid PGD, default exception handling */
movi a3, exc_table
rsr a1, depc
xsr a3, excsave1
s32i a1, a2, PT_AREG2
s32i a3, a2, PT_AREG3
mov a1, a2
rsr a2, ps
......@@ -1767,9 +1751,9 @@ ENDPROC(fast_second_level_miss)
* a0: trashed, original value saved on stack (PT_AREG0)
* a1: a1
* a2: new stack pointer, original in DEPC
* a3: dispatch table
* a3: a3
* depc: a2, original value saved on stack (PT_DEPC)
* excsave_1: a3
* excsave_1: dispatch table
*
* PT_DEPC >= VALID_DOUBLE_EXCEPTION_ADDRESS: double exception, DEPC
* < VALID_DOUBLE_EXCEPTION_ADDRESS: regular exception
......@@ -1777,17 +1761,17 @@ ENDPROC(fast_second_level_miss)
ENTRY(fast_store_prohibited)
/* Save a1 and a4. */
/* Save a1 and a3. */
s32i a1, a2, PT_AREG1
s32i a4, a2, PT_AREG4
s32i a3, a2, PT_AREG3
GET_CURRENT(a1,a2)
l32i a0, a1, TASK_MM # tsk->mm
beqz a0, 9f
8: rsr a1, excvaddr # fault address
_PGD_OFFSET(a0, a1, a4)
_PGD_OFFSET(a0, a1, a3)
l32i a0, a0, 0
beqz a0, 2f
......@@ -1796,39 +1780,37 @@ ENTRY(fast_store_prohibited)
* and is not PAGE_NONE. See pgtable.h for possible PTE layouts.
*/
_PTE_OFFSET(a0, a1, a4)
l32i a4, a0, 0 # read pteval
_PTE_OFFSET(a0, a1, a3)
l32i a3, a0, 0 # read pteval
movi a1, _PAGE_CA_INVALID
ball a4, a1, 2f
bbci.l a4, _PAGE_WRITABLE_BIT, 2f
ball a3, a1, 2f
bbci.l a3, _PAGE_WRITABLE_BIT, 2f
movi a1, _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_HW_WRITE
or a4, a4, a1
or a3, a3, a1
rsr a1, excvaddr
s32i a4, a0, 0
s32i a3, a0, 0
/* We need to flush the cache if we have page coloring. */
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
dhwb a0, 0
#endif
pdtlb a0, a1
wdtlb a4, a0
wdtlb a3, a0
/* Exit critical section. */
movi a0, 0
rsr a3, excsave1
s32i a0, a3, EXC_TABLE_FIXUP
/* Restore the working registers, and return. */
l32i a4, a2, PT_AREG4
l32i a3, a2, PT_AREG3
l32i a1, a2, PT_AREG1
l32i a0, a2, PT_AREG0
l32i a2, a2, PT_DEPC
/* Restore excsave1 and a3. */
xsr a3, excsave1
bgeui a2, VALID_DOUBLE_EXCEPTION_ADDRESS, 1f
rsr a2, depc
......@@ -1845,11 +1827,8 @@ ENTRY(fast_store_prohibited)
2: /* If there was a problem, handle fault in C */
rsr a4, depc # still holds a2
xsr a3, excsave1
s32i a4, a2, PT_AREG2
s32i a3, a2, PT_AREG3
l32i a4, a2, PT_AREG4
rsr a3, depc # still holds a2
s32i a3, a2, PT_AREG2
mov a1, a2
rsr a2, ps
......
......@@ -78,6 +78,7 @@ ENTRY(_UserExceptionVector)
s32i a0, a2, PT_DEPC # mark it as a regular exception
addx4 a0, a0, a3 # find entry in table
l32i a0, a0, EXC_TABLE_FAST_USER # load handler
xsr a3, excsave1 # restore a3 and dispatch table
jx a0
ENDPROC(_UserExceptionVector)
......@@ -104,6 +105,7 @@ ENTRY(_KernelExceptionVector)
s32i a0, a2, PT_DEPC # mark it as a regular exception
addx4 a0, a0, a3 # find entry in table
l32i a0, a0, EXC_TABLE_FAST_KERNEL # load handler address
xsr a3, excsave1 # restore a3 and dispatch table
jx a0
ENDPROC(_KernelExceptionVector)
......@@ -168,7 +170,7 @@ ENDPROC(_KernelExceptionVector)
*
* a0: DEPC
* a1: a1
* a2: trashed, original value in EXC_TABLE_DOUBLE_A2
* a2: trashed, original value in EXC_TABLE_DOUBLE_SAVE
* a3: exctable
* depc: a0
* excsave_1: a3
......@@ -204,47 +206,46 @@ ENDPROC(_KernelExceptionVector)
.section .DoubleExceptionVector.text, "ax"
.begin literal_prefix .DoubleExceptionVector
.globl _DoubleExceptionVector_WindowUnderflow
.globl _DoubleExceptionVector_WindowOverflow
ENTRY(_DoubleExceptionVector)
/* Deliberately destroy excsave (don't assume it's value was valid). */
wsr a3, excsave1 # save a3
xsr a3, excsave1
s32i a2, a3, EXC_TABLE_DOUBLE_SAVE
/* Check for kernel double exception (usually fatal). */
rsr a3, ps
_bbci.l a3, PS_UM_BIT, .Lksp
rsr a2, ps
_bbci.l a2, PS_UM_BIT, .Lksp
/* Check if we are currently handling a window exception. */
/* Note: We don't need to indicate that we enter a critical section. */
xsr a0, depc # get DEPC, save a0
movi a3, WINDOW_VECTORS_VADDR
_bltu a0, a3, .Lfixup
addi a3, a3, WINDOW_VECTORS_SIZE
_bgeu a0, a3, .Lfixup
movi a2, WINDOW_VECTORS_VADDR
_bltu a0, a2, .Lfixup
addi a2, a2, WINDOW_VECTORS_SIZE
_bgeu a0, a2, .Lfixup
/* Window overflow/underflow exception. Get stack pointer. */
mov a3, a2
/* This explicit literal and the following references to it are made
* in order to fit DoubleExceptionVector.literals into the available
* 16-byte gap before DoubleExceptionVector.text in the absence of
* link time relaxation. See kernel/vmlinux.lds.S
*/
.literal .Lexc_table, exc_table
l32r a2, .Lexc_table
l32i a2, a2, EXC_TABLE_KSTK
l32i a2, a3, EXC_TABLE_KSTK
/* Check for overflow/underflow exception, jump if overflow. */
_bbci.l a0, 6, .Lovfl
/* a0: depc, a1: a1, a2: kstk, a3: a2, depc: a0, excsave: a3 */
_bbci.l a0, 6, _DoubleExceptionVector_WindowOverflow
/* Restart window underflow exception.
/*
* Restart window underflow exception.
* Currently:
* depc = orig a0,
* a0 = orig DEPC,
* a2 = new sp based on KSTK from exc_table
* a3 = excsave_1
* excsave_1 = orig a3
*
* We return to the instruction in user space that caused the window
* underflow exception. Therefore, we change window base to the value
* before we entered the window underflow exception and prepare the
......@@ -252,10 +253,11 @@ ENTRY(_DoubleExceptionVector)
* by changing depc (in a0).
* Note: We can trash the current window frame (a0...a3) and depc!
*/
_DoubleExceptionVector_WindowUnderflow:
xsr a3, excsave1
wsr a2, depc # save stack pointer temporarily
rsr a0, ps
extui a0, a0, PS_OWB_SHIFT, 4
extui a0, a0, PS_OWB_SHIFT, PS_OWB_WIDTH
wsr a0, windowbase
rsync
......@@ -263,28 +265,57 @@ ENTRY(_DoubleExceptionVector)
xsr a2, depc # save a2 and get stack pointer
s32i a0, a2, PT_AREG0
wsr a3, excsave1 # save a3
l32r a3, .Lexc_table
xsr a3, excsave1
rsr a0, exccause
s32i a0, a2, PT_DEPC # mark it as a regular exception
addx4 a0, a0, a3
xsr a3, excsave1
l32i a0, a0, EXC_TABLE_FAST_USER
jx a0
.Lfixup:/* Check for a fixup handler or if we were in a critical section. */
/*
* We only allow the ITLB miss exception if we are in kernel space.
* All other exceptions are unexpected and thus unrecoverable!
*/
#ifdef CONFIG_MMU
.extern fast_second_level_miss_double_kernel
.Lksp: /* a0: a0, a1: a1, a2: a2, a3: trashed, depc: depc, excsave: a3 */
rsr a3, exccause
beqi a3, EXCCAUSE_ITLB_MISS, 1f
addi a3, a3, -EXCCAUSE_DTLB_MISS
bnez a3, .Lunrecoverable
1: movi a3, fast_second_level_miss_double_kernel
jx a3
#else
.equ .Lksp, .Lunrecoverable
#endif
/* Critical! We can't handle this situation. PANIC! */
/* a0: depc, a1: a1, a2: a2, a3: trashed, depc: a0, excsave1: a3 */
.extern unrecoverable_exception
l32r a3, .Lexc_table
s32i a2, a3, EXC_TABLE_DOUBLE_SAVE # temporary variable
.Lunrecoverable_fixup:
l32i a2, a3, EXC_TABLE_DOUBLE_SAVE
xsr a0, depc
.Lunrecoverable:
rsr a3, excsave1
wsr a0, excsave1
movi a0, unrecoverable_exception
callx0 a0
.Lfixup:/* Check for a fixup handler or if we were in a critical section. */
/* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave1: a3 */
/* Enter critical section. */
l32i a2, a3, EXC_TABLE_FIXUP
s32i a3, a3, EXC_TABLE_FIXUP
beq a2, a3, .Lunrecoverable_fixup # critical!
beq a2, a3, .Lunrecoverable_fixup # critical section
beqz a2, .Ldflt # no handler was registered
/* a0: depc, a1: a1, a2: trash, a3: exctable, depc: a0, excsave: a3 */
......@@ -293,58 +324,145 @@ ENTRY(_DoubleExceptionVector)
.Ldflt: /* Get stack pointer. */
l32i a3, a3, EXC_TABLE_DOUBLE_SAVE
addi a2, a3, -PT_USER_SIZE
.Lovfl: /* Jump to default handlers. */