/*	$NetBSD: vectors.S,v 1.29 2022/06/26 11:14:36 jmcneill Exp $	*/

#include <aarch64/asm.h>
#include <aarch64/locore.h>

#include "assym.h"

#include "opt_compat_netbsd32.h"
#include "opt_cpuoptions.h"
#include "opt_ddb.h"
#include "opt_dtrace.h"

RCSID("$NetBSD: vectors.S,v 1.29 2022/06/26 11:14:36 jmcneill Exp $")

	ARMV8_DEFINE_OPTIONS

#ifdef KDTRACE_HOOKS
/*
 * dtrace needs to emulate  stp x29,x30,[sp,#-FRAMESIZE]!   where
 * FRAMESIZE can be as large as 512, so create a 512-byte buffer
 * between the interrupted code's frame and our struct trapframe.
 */
#define	TRAP_FRAMESIZE	(TF_SIZE + 512)
#else
#define	TRAP_FRAMESIZE	TF_SIZE
#endif

/*
 * Template for the handler functions.
 */
.macro	vector_func, func, el, intr_p, label, tpidr
	.align 7	/* cacheline-aligned */

ENTRY_NBTI(\func)

	.if \el == 1
	/* need to allocate stack on el1 */
	sub	sp, sp, #TRAP_FRAMESIZE
	.endif

	stp	x0, x1, [sp, #TF_X0]
	stp	x2, x3, [sp, #TF_X2]
	stp	x4, x5, [sp, #TF_X4]
	stp	x6, x7, [sp, #TF_X6]
	stp	x8, x9, [sp, #TF_X8]
	stp	x10, x11, [sp, #TF_X10]
	stp	x12, x13, [sp, #TF_X12]
	stp	x14, x15, [sp, #TF_X14]
	stp	x16, x17, [sp, #TF_X16]
	str	x18, [sp, #TF_X18]
	stp	x19, x20, [sp, #TF_X19]
	stp	x21, x22, [sp, #TF_X21]
	stp	x23, x24, [sp, #TF_X23]
	stp	x25, x26, [sp, #TF_X25]
	stp	x27, x28, [sp, #TF_X27]
	stp	x29, x30, [sp, #TF_X29]

	/* get sp and elr */
	.if \el == 0
	mrs	x20, sp_el0
	.else
	/* sp was already adjusted, so adjust x20 back */
	add	x20, sp, #TRAP_FRAMESIZE
	.endif
	mrs	x21, elr_el1

	/* store sp and elr */
	.if TF_SP + 8 == TF_PC
	stp	x20, x21, [sp, #TF_SP]
	.else
	str	x20, [sp, #TF_SP]
	str	x21, [sp, #TF_PC]
	.endif

	mrs	x22, spsr_el1
	str	x22, [sp, #TF_SPSR]

	.if \intr_p == 1
	mov	x23, #-1
	mov	x24, xzr
	.else
	mrs	x23, esr_el1
	mrs	x24, far_el1
	.endif

	.if TF_ESR + 8 == TF_FAR
	stp	x23, x24, [sp, #TF_ESR]
	.else
	str	x23, [sp, #TF_ESR]
	str	x24, [sp, #TF_FAR]
	.endif

	.if \el == 0
	/* curlwp->l_private = tpidr{,ro}_el0 */
	mrs	x1, tpidr_el1		/* x1 = curlwp */
	mrs	x0, tpidr\tpidr\()_el0
	str	x0, [x1, #L_PRIVATE]	/* curlwp->l_private = tpidr{,ro}_el0 */

#ifdef ARMV83_PAC
	/* Switch to the kern PAC key. */
	adrl	x4, _C_LABEL(aarch64_pac_enabled)
	ldr	w4, [x4]
	cbz	w4, 1f
	ldp	x5, x6, [x1, #L_MD_IA_KERN]
	msr	APIAKeyLo_EL1, x5
	msr	APIAKeyHi_EL1, x6
	isb
1:
#endif
	.endif

	adr	x30, el\el\()_trap_exit	/* el[01]_trap_exit */
	mov	x0, sp
#ifdef DDB
	mov	x29, sp			/* for backtrace */
#endif
	b	\label
END(\func)
.endm

/*
 * The vector_entry macro must be small enough to fit 0x80 bytes! We just jump
 * into the proper function, so this constraint is always respected.
 */
.macro	vector_entry, func
	.align 7	/* aligned 0x80 */
	b	\func
.endm

/*
 * The functions.
 */
vector_func	el1t_sync_handler,  1, 0, trap_el1t_sync
vector_func	el1t_irq_handler,   1, 1, trap_el1t_irq
vector_func	el1t_fiq_handler,   1, 1, trap_el1t_fiq
vector_func	el1t_error_handler, 1, 0, trap_el1t_error

vector_func	el1h_sync_handler,  1, 0, trap_el1h_sync
vector_func	el1h_intr_handler,  1, 1, cpu_irq
vector_func	el1h_fiq_handler,   1, 1, cpu_fiq
vector_func	el1h_error_handler, 1, 0, trap_el1h_error

vector_func	el0_sync_handler,  0, 0, trap_el0_sync
vector_func	el0_intr_handler,  0, 1, cpu_irq
vector_func	el0_fiq_handler,   0, 1, cpu_fiq
vector_func	el0_error_handler, 0, 0, trap_el0_error

vector_func	el0_32sync_handler,  0, 0, trap_el0_32sync, ro
vector_func	el0_32intr_handler,  0, 1, cpu_irq, ro
vector_func	el0_32fiq_handler,   0, 1, cpu_fiq, ro
vector_func	el0_32error_handler, 0, 0, trap_el0_32error, ro

/*
 * The vector table. Must be aligned to 2048.
 */
	.align 11
ENTRY_NBTI(el1_vectors)
	/*
	 * Exception taken from current Exception Level with SP_EL0.
	 * (These shouldn't happen)
	 */
	vector_entry	el1t_sync_handler
	vector_entry	el1t_irq_handler
	vector_entry	el1t_fiq_handler
	vector_entry	el1t_error_handler

	/*
	 * Exception taken from current Exception Level with SP_EL1.
	 * There are entries for exceptions caused in EL1 (kernel exceptions).
	 */
	vector_entry	el1h_sync_handler
	vector_entry	el1h_intr_handler
	vector_entry	el1h_fiq_handler
	vector_entry	el1h_error_handler

	/*
	 * Exception taken from lower Exception Level which is using AArch64.
	 * There are entries for exceptions caused in EL0 (native user exceptions).
	 */
	vector_entry	el0_sync_handler
	vector_entry	el0_intr_handler
	vector_entry	el0_fiq_handler
	vector_entry	el0_error_handler

	/*
	 * Exception taken from lower Exception Level which is using AArch32.
	 * There are entries for exceptions caused in EL0 (compat user exceptions).
	 */
	vector_entry	el0_32sync_handler
	vector_entry	el0_32intr_handler
	vector_entry	el0_32fiq_handler
	vector_entry	el0_32error_handler
END(el1_vectors)

	.macro unwind_x0_x2
	ldp	x0, x1, [sp, #TF_X0]
	ldr	x2, [sp, #TF_X2]
	.endm

	.macro unwind_x3_x30
	ldp	x3, x4, [sp, #TF_X3]
	ldp	x5, x6, [sp, #TF_X5]
	ldp	x7, x8, [sp, #TF_X7]
	ldp	x9, x10, [sp, #TF_X9]
	ldp	x11, x12, [sp, #TF_X11]
	ldp	x13, x14, [sp, #TF_X13]
	ldp	x15, x16, [sp, #TF_X15]
	ldp	x17, x18, [sp, #TF_X17]
	ldp	x19, x20, [sp, #TF_X19]
	ldp	x21, x22, [sp, #TF_X21]
	ldp	x23, x24, [sp, #TF_X23]
	ldp	x25, x26, [sp, #TF_X25]
	ldp	x27, x28, [sp, #TF_X27]
	ldp	x29, x30, [sp, #TF_X29]
	.endm

/*
 * EL1 exception return for trap and interrupt.
 */
#ifdef DDB
ENTRY_NP(el1_trap)
	nop				/* dummy for DDB backtrace (for lr-4) */
#endif
ENTRY_NP(el1_trap_exit)
	DISABLE_INTERRUPT		/* make sure I|F marked */

	unwind_x3_x30

#if TF_PC + 8 == TF_SPSR
	ldp	x0, x1, [sp, #TF_PC]
#else
	ldr	x0, [sp, #TF_PC]
	ldr	x1, [sp, #TF_SPSR]
#endif
	msr	elr_el1, x0		/* exception pc */
	msr	spsr_el1, x1		/* exception pstate */

	/*
	 * cpu_jump_onfault() modify tf->tf_sp, therefore
	 * we need to restore sp from trapframe,
	 * and unwind x0-x2 without sp.
	 */
	mov	x0, sp
	ldr	x1, [x0, #TF_SP]
	mov	sp, x1
	ldp	x1, x2, [x0, #TF_X1]
	ldr	x0, [x0, #TF_X0]

	ERET
END(el1_trap_exit)
#ifdef DDB
END(el1_trap)
#endif

/*
 * EL0 exception return for trap, interrupt and syscall with
 * possible AST processing.
 */
#ifdef DDB
ENTRY_NP(el0_trap)
	nop				/* dummy for DDB backtrace (for lr-4) */
#endif
ENTRY_NP(el0_trap_exit)

	adr	lr, 1f			/* return address from trap_doast */
1:
	/* while (curlwp->l_md.md_astpending != 0) { */
	DISABLE_INTERRUPT		/* make sure I|F marked */
	mrs	x9, tpidr_el1
	ldr	w8, [x9, #L_MD_ASTPENDING]
	cbz	w8, 9f

	/* curlwp->l_md.md_astpending = 0; */
	str	wzr, [x9, #L_MD_ASTPENDING]

	/*  trap_doast(tf); */
	ENABLE_INTERRUPT
	mov	x0, sp
	b	_C_LABEL(trap_doast)	/* tail call (return to 1b) */
	/* } */
9:

	/* x9 is tpidr_el1 */
	ldr	x23, [x9, #L_MD_CPACR]
	msr	cpacr_el1, x23		/* FP unit EL0 handover */
	isb				/* necessary? */

	ldr	x0, [x9, #L_PRIVATE]	/* tpidr_el0 = curlwp->l_private */
	msr	tpidr_el0, x0
#ifdef COMPAT_NETBSD32
	msr	tpidrro_el0, x0
#endif

#ifdef ARMV83_PAC
	/* Switch to the user PAC key. */
	adrl	x4, _C_LABEL(aarch64_pac_enabled)
	ldr	w4, [x4]
	cbz	w4, 1f
	ldp	x5, x6, [x9, #L_MD_IA_USER]
	msr	APIAKeyLo_EL1, x5
	msr	APIAKeyHi_EL1, x6
	isb
1:
#endif

	unwind_x3_x30

#if TF_PC + 8 == TF_SPSR
	ldp	x0, x1, [sp, #TF_PC]
#else
	ldr	x0, [sp, #TF_PC]
	ldr	x1, [sp, #TF_SPSR]
#endif
	ldr	x2, [sp, #TF_SP]
	msr	elr_el1, x0		/* exception pc */
	msr	spsr_el1, x1		/* exception pstate */
	msr	sp_el0, x2		/* restore EL0 stack */

	/* if the process is traced, enable MDSCR_EL1.SS */
	tbz	x1, #SPSR_SS_SHIFT, 1f
	mrs	x0, mdscr_el1
	orr	x0, x0, #MDSCR_SS
#ifdef DDB
	bic	x0, x0, #MDSCR_KDE
#endif
	msr	mdscr_el1, x0
1:
	unwind_x0_x2

	/* leave sp at l_md.md_utf, return back to EL0 user process */
	ERET
END(el0_trap_exit)
#ifdef DDB
END(el0_trap)
#endif