/*	$NetBSD: exception_vector.S,v 1.53 2021/07/26 21:43:11 andvar Exp $	*/

/*-
 * Copyright (c) 2002, 2019 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "opt_cputype.h"
#include "opt_ddb.h"
#include "opt_ptrace.h"

#include "assym.h"

#include <sh3/param.h>
#include <sh3/locore.h>
#include <sh3/exception.h>
#include <sh3/ubcreg.h>
#include <sh3/pte.h>
#include <sh3/mmu_sh3.h>
#include <sh3/mmu_sh4.h>

/* 
 * Align vectors more strictly here (where we don't really care) so
 * that .align 5 (i.e. 32B cache line) before data block does the
 * right thing w.r.t. final destinations after vectors are copied.
 */
#define _ALIGN_TEXT	.align 5
#include <sh3/asm.h>

__KERNEL_RCSID(0, "$NetBSD: exception_vector.S,v 1.53 2021/07/26 21:43:11 andvar Exp $")


/*
 * Exception vectors.
 * The following routines are copied to vector addresses.
 *	sh_vector_generic:	VBR + 0x100
 *	sh_vector_tlbmiss:	VBR + 0x400
 *	sh_vector_interrupt:	VBR + 0x600
 */

#define VECTOR_END_MARKER(sym)			\
		.globl	_C_LABEL(sym);		\
	_C_LABEL(sym):


/*
 * LINTSTUB: Var: char sh_vector_generic[1];
 *
 * void sh_vector_generic(void);
 *	Copied to VBR+0x100.  This code should be position independent
 *	and maximum 786 bytes long (== 0x400 - 0x100).
 */
NENTRY(sh_vector_generic)
	__EXCEPTION_ENTRY
	/* Identify exception cause */
	MOV	(EXPEVT, r0)
	mov.l	@r0, r0
	mov.l	r0, @(TF_EXPEVT, r14)	/* tf->tf_expevt = EXPEVT */
	/* Get curlwp */
	mov.l	.Lg_curlwp, r1
	mov.l	@r1, r4			/* 1st arg */
	/* Get TEA */
	MOV	(TEA, r1)
	mov.l	@r1, r6			/* 3rd arg */
	/* Check TLB exception or not */
	mov.l	.Lg_TLB_PROT_ST, r1
	cmp/hi	r1, r0
	bt/s	1f
	 mov	r4, r8	/* preserve curlwp across call */

	/* tlb_exception(curlwp, tf, TEA); */
	__INTR_MASK_EXCEPTION_UNBLOCK(r0, r1, r3)
	mov.l	.Lg_tlb_exception, r0
	jsr	@r0
	 mov	r14, r5			/* 2nd arg */

	/* Check for ASTs on exit to user mode. */
	__INTR_MASK(r0, r1)
	mov.l	.Lg_ast, r0
	mov	r8, r4
	jsr	@r0
	 mov	r14, r5
	bra	.Lg_return_from_exception
	 nop

1:	/* general_exception(curlwp, tf, TEA); */
#if defined(PTRACE_HOOKS) || defined(DDB)
	mov	#0, r2
	MOV	(BBRA, r1)
	mov.l	r2, @(TF_UBC, r14)	/* clear tf->tf_ubc */
	mov.w	r2, @r1			/* disable UBC channel A */
#endif
	__INTR_MASK_EXCEPTION_UNBLOCK(r0, r1, r3)
	mov.l	.Lg_general_exception, r0
	jsr	@r0
	 mov	r14, r5			/* 2nd arg */

	/* Check for ASTs on exit to user mode. */
	__INTR_MASK(r0, r1)
	mov.l	.Lg_ast, r0
	mov	r8, r4
	jsr	@r0
	 mov	r14, r5

#if defined(PTRACE_HOOKS) || defined(DDB)
	mov.l	@(TF_UBC, r14), r2
	tst	r2, r2			! single-step == 0?
	bt	.Lg_return_from_exception

	!! We are returning from DDB to do single step.  Channel A in
	!! UBC is already rigged, we just need to enable it.
	MOV	(BBRA, r3)
	MOV	(BARA, r5)
	__EXCEPTION_BLOCK(r0, r1)
	mov.l	@(TF_SPC, r14), r4
	mov.l	r4, @r5			! BARA = tf->tf_spc
	mov.w	r2, @r3			! BBRA = tf->tf_ubc
#endif /* PTRACE || DDB */
.Lg_return_from_exception:
	__EXCEPTION_RETURN

	.align	5
REG_SYMBOL(EXPEVT)
.Lg_curlwp:		.long	_C_LABEL(curlwp)
REG_SYMBOL(TEA)
.Lg_TLB_PROT_ST:	.long	EXPEVT_TLB_PROT_ST
.Lg_tlb_exception:	.long	_C_LABEL(tlb_exception)
.Lg_general_exception:	.long	_C_LABEL(general_exception)
.Lg_ast:		.long	_C_LABEL(ast)
REG_SYMBOL(BBRA)
REG_SYMBOL(BARA)

/* LINTSTUB: Var: char sh_vector_generic_end[1]; */
VECTOR_END_MARKER(sh_vector_generic_end)
	SET_ENTRY_SIZE(sh_vector_generic)


#ifdef SH3
/*
 * LINTSTUB: Var: char sh3_vector_tlbmiss[1];
 *
 * TLB miss vector.  We run through the fast path first, checking if
 * there's a valid mapping in curlwp or kernel pmap.  We do fast path
 * with exceptions disabled, so no P3 addresses please (including no
 * kernel stack, as we cannot wire TLB entries on sh3).  We can only
 * use BANK1 registers, and of those r6 and r7 are already taken.
 *
 * If we don't find a valid mapping in the fast path, we do context
 * save and call tlb exception handler.
 *
 * Copied to VBR+0x400.  This code should be position independent
 * and maximum 512 bytes long (== 0x600 - 0x400).
 */
NENTRY(sh3_vector_tlbmiss)
	mov	#(SH3_PTEH & 0xff), r4
	mov.l	.L3_VPN_cleanup, r0
	mov.l	@r4, r5
	and	r0, r5		! trim vpn to 4K page boundary
	!! For the duration of fast path we keep
	!! r4: SH3_PTEH - other PTE regs are addressable as @(offset, r4)
	!! r5: { VPN, ASID } that caused the miss

	cmp/pz	r5		! user space address?
	bt/s	.L3_user_va
	 mov	r5, r2		! copy of vpn to compute indices into ptd/ptp

	!! kernel space address, use pmap_kernel(), adjust vpn for indexing
	!! see __pmap_kpte_lookup
.L3_kernel_va:
	mov.l	.L3_VM_MIN_KERNEL_ADDRESS, r0
	mov.l	.L3_kernptd,  r1 ! pmap_kernel()->pm_ptp
	bra	.L3_fetch_pte
	 sub	r0, r2		! vpn -= VM_MIN_KERNEL_ADDRESS

	!! user space address, use curlwp's pmap
.L3_user_va:
	mov.l	.L3_curptd,  r1	! curlwp->...->pm_ptp

	!! see __pmap_pte_lookup
.L3_fetch_pte:
	mov.l	@r1, r3		! fetch ptd

	!! r2: vpn, prepared for indexing into ptd
	!! r3: pt_entry_t **ptd => pt_entry_t *ptp => pt_entry_t pte
#ifdef DEBUG
	tst	r3, r3		! ptd == NULL  - cannot happen
	bt/s	.L3_call_tlb_exception
#endif
	 mov	#-22, r1	! __PMAP_PTP_SHIFT

	!! __PMAP_PTP_INDEX(vpn)
	mov	r2, r0
	shld	r1, r0		! vpn >> __PMAP_PTP_SHIFT
	mov.l	.L3_ptp_index_mask, r1
	and	r1, r0		! ... & (__PMAP_PTP_N - 1)
	shll2	r0		! array index -> array offset
	mov.l	@(r0, r3), r3	! ptp = ptd[idx]
	tst	r3, r3		! if (ptp == NULL)
	bt/s	.L3_call_tlb_exception
	 mov	#-(PGSHIFT - 2), r1

	!! __PMAP_PTP_OFSET(vpn) - except we pre-shift 2 bits left to
	!! get the array offset directly, as we know bits 10 and 11
	!! are zero (we cleaned them in r5 to get 4K aligned VPN)
	shld	r1, r2		! vpn >> (PGSHIFT - 2)
	mov.l	.L3_ptp_offset_mask, r0
	and	r2, r0		! ... & ((__PMAP_PTP_PG_N - 1) << 2)
	mov.l	@(r0, r3), r3	! pte = ptp[idx]


	!! r3: pte
	!! r4: SH3_PTEH
	!! r5: { VPN, ASID }

	mov.l	.L3_PG_V, r0
	tst	r0, r3		! if ((pte & PG_V) == 0)
	bt/s	.L3_call_tlb_exception
	 nop

	mov.l	.L3_PG_HW_BITS, r1
	cmp/pz	r5		! user space address?
	and	r1, r3		! pte &= PG_HW_BITS
	bf/s	.L3_load_kernel
	 mov.l	r3, @(0x04, r4)	! *SH3_PTEL = pte

	!! load mapping for a user space page
	!! we reload PTEH to enter VPN aligned to 4K page boundary
.L3_load_user:
	mov.l	r5, @r4		! *SH3_PTEH = { VPN, ASID }
	ldtlb			! needs 2 insns padding before RTE
	nop
	nop
	rte
	 nop

	!! load mapping for a kernel space page
	!! we need to temporary set ASID to 0
.L3_load_kernel:
	mov.l	.L3_clear_ASID, r1
	and	r5, r1		! *SH3_PTEH & ~SH3_PTEH_ASID_MASK
	mov.l	r1, @r4		! *SH3_PTEH = { VPN, ASID = 0 }
	ldtlb
	mov.l	r5, @r4		! restore ASID
	nop
	rte
	 nop


	!! if we haven't found a valid mapping in the fast path
	!!     tlb_exception(curlwp, trapframe, tea)
.L3_call_tlb_exception:
	__EXCEPTION_ENTRY
	mov.l	.L3_SH3_EXPEVT, r2
	mov.l	.L3_curlwp, r1
	mov	#(SH3_TEA & 0xff), r0
	mov.l	@r2, r2			! *SH3_EXPEVT
	mov.l	@r0, r6			! arg3: va = *SH3_TEA
	mov.l	@r1, r4			! arg1: curlwp
	__INTR_MASK_EXCEPTION_UNBLOCK(r0, r1, r3)
	mov.l	.L3_tlb_exception, r0
	mov	r4, r8			! save curlwp across the call
	mov.l	r2, @(TF_EXPEVT, r14)	! tf->tf_expevt = EXPEVT
	jsr	@r0
	 mov	r14, r5			! arg2: trapframe

	/* Check for ASTs on exit to user mode. */
	__INTR_MASK(r0, r1)
	mov.l	.L3_ast, r0
	mov	r8, r4			! arg1: curlwp
	jsr	@r0
	 mov	r14, r5			! arg2: trapframe
	__EXCEPTION_RETURN

	.align	4
.L3_VPN_cleanup:		.long	~0x00000c00
.L3_curptd:			.long	_C_LABEL(curptd)
.L3_kernptd:			.long	_C_LABEL(__pmap_kernel)
.L3_VM_MIN_KERNEL_ADDRESS:	.long	VM_MIN_KERNEL_ADDRESS
.L3_ptp_index_mask:		.long	0x1ff
.L3_ptp_offset_mask:		.long	0x3ff << 2
.L3_PG_HW_BITS:			.long	PG_HW_BITS
.L3_PG_V:			.long	PG_V
.L3_clear_ASID:			.long	~SH3_PTEH_ASID_MASK
.L3_SH3_EXPEVT:			.long	SH3_EXPEVT
.L3_curlwp:			.long	_C_LABEL(curlwp)
.L3_tlb_exception:		.long	_C_LABEL(tlb_exception)
.L3_ast:			.long	_C_LABEL(ast)

/* LINTSTUB: Var: char sh3_vector_tlbmiss_end[1]; */
VECTOR_END_MARKER(sh3_vector_tlbmiss_end)
	SET_ENTRY_SIZE(sh3_vector_tlbmiss)

#endif /* SH3 */


#ifdef SH4
/*
 * LINTSTUB: Var: char sh4_vector_tlbmiss[1];
 *
 * TLB miss vector.  We run through the fast path first, checking if
 * there's a valid mapping in curlwp or kernel pmap.  We do fast path
 * with exceptions disabled, so no P3 addresses please (though we can
 * use kernel stack if need be, as its TLB entries are wired).  We can
 * only use BANK1 registers, and of those r6 and r7 are already taken.
 *
 * If we don't find a valid mapping in the fast path, we do context
 * save and call tlb exception handler.
 *
 * Copied to VBR+0x400.  This code should be relocatable
 * and maximum 512 bytes long (== 0x600 - 0x400).
 */
NENTRY(sh4_vector_tlbmiss)
	mov.l	.L4_SH4_PTEH, r4
	mov.l	.L4_VPN_cleanup, r0
	mov.l	@r4, r5
	and	r0, r5		! trim vpn to 4K page boundary
	!! For the duration of fast path we keep
	!! r4: SH4_PTEH - other PTE regs are addressable as @(offset, r4)
	!! r5: { VPN, ASID } that caused the miss

	cmp/pz	r5		! user space address?
	bt/s	.L4_user_va
	 mov	r5, r2		! copy of vpn to compute indices into ptd/ptp

	!! kernel space address, use pmap_kernel(), adjust vpn for indexing
	!! see __pmap_kpte_lookup
.L4_kernel_va:
	mov.l	.L4_VM_MIN_KERNEL_ADDRESS, r0
	mov.l	.L4_kernptd,  r1 ! pmap_kernel()->pm_ptp
	bra	.L4_fetch_pte
	 sub	r0, r2		! vpn -= VM_MIN_KERNEL_ADDRESS

	!! user space address, use curlwp's pmap
.L4_user_va:
	mov.l	.L4_curptd,  r1	! curlwp->...->pm_ptp

	!! see __pmap_pte_lookup
.L4_fetch_pte:
	mov.l	@r1, r3		! fetch ptd

	!! r2: vpn, prepared for indexing into ptd
	!! r3: pt_entry_t **ptd => pt_entry_t *ptp => pt_entry_t pte
#ifdef DEBUG
	tst	r3, r3		! ptd == NULL  - cannot happen
	bt/s	.L4_call_tlb_exception
#endif
	 mov	#-22, r1	! __PMAP_PTP_SHIFT

	!! __PMAP_PTP_INDEX(vpn)
	mov	r2, r0
	shld	r1, r0		! vpn >> __PMAP_PTP_SHIFT
	mov.l	.L4_ptp_index_mask, r1
	and	r1, r0		! ... & (__PMAP_PTP_N - 1)
	shll2	r0		! array index -> array offset
	mov.l	@(r0, r3), r3	! ptp = ptd[idx]
	tst	r3, r3		! if (ptp == NULL)
	bt/s	.L4_call_tlb_exception
	 mov	#-(PGSHIFT - 2), r1

	!! __PMAP_PTP_OFSET(vpn) - except we pre-shift 2 bits left to
	!! get the array offset directly, as we know bits 10 and 11
	!! are zero (we cleaned them in r5 to get 4K aligned VPN)
	shld	r1, r2		! vpn >> (PGSHIFT - 2)
	mov.l	.L4_ptp_offset_mask, r0
	and	r2, r0		! ... & ((__PMAP_PTP_PG_N - 1) << 2)
	mov.l	@(r0, r3), r3	! pte = ptp[idx]


	!! r3: pte
	!! r4: SH4_PTEH
	!! r5: { VPN, ASID }

	mov.l	.L4_PG_V, r0
	tst	r0, r3		! if ((pte & PG_V) == 0)
	bt/s	.L4_call_tlb_exception
	 mov	r3, r0		! prepare PCMCIA SA bits for SH4_PTEA

	mov.l	.L4_PG_HW_BITS, r1
	shlr8	r0
	and	r1, r3		! pte &= PG_HW_BITS
	shlr	r0		! pte >> _PG_PCMCIA_SHIFT
	cmp/pz	r5		! user space address?
	and	#SH4_PTEA_SA_MASK, r0
	mov.l	r3, @(0x04, r4)	! *SH4_PTEL = pte
	bf/s	.L4_load_kernel
	 mov.l	r0, @(0x34, r4)	! *SH4_PTEA = PCMCIA space attrs

	!! load mapping for a user space page
	!! we reload PTEH to enter VPN aligned to 4K page boundary
.L4_load_user:
	mov.l	r5, @r4		! *SH4_PTEH = { VPN, ASID }
	ldtlb			! needs 1 insn padding before RTE
	nop
	rte
	 nop

	!! load mapping for a kernel space page
	!! we need to temporary set ASID to 0
.L4_load_kernel:
	mov.l	.L4_clear_ASID, r1
	and	r5, r1		! *SH4_PTEH & ~SH4_PTEH_ASID_MASK
	mov.l	r1, @r4		! *SH4_PTEH = { VPN, ASID = 0 }
	ldtlb
	mov.l	r5, @r4		! restore ASID
	rte
	 nop


	!! if we haven't found a valid mapping in the fast path
	!!     tlb_exception(curlwp, trapframe, tea)
.L4_call_tlb_exception:
	__EXCEPTION_ENTRY
	mov.l	.L4_SH4_PTEH, r0
	mov.l	.L4_curlwp, r1
	mov.l	@(0x24, r0), r2		! *SH4_EXPEVT
	mov.l	@(0x0c, r0), r6		! arg3: va = *SH4_TEA
	mov.l	@r1, r4			! arg1: curlwp
	__INTR_MASK_EXCEPTION_UNBLOCK(r0, r1, r3)
	mov.l	.L4_tlb_exception, r0
	mov	r4, r8			! save curlwp across the call
	mov.l	r2, @(TF_EXPEVT, r14)	! tf->tf_expevt = EXPEVT
	jsr	@r0
	 mov	r14, r5			! arg2: trapframe

	/* Check for ASTs on exit to user mode. */
	__INTR_MASK(r0, r1)
	mov.l	.L4_ast, r0
	mov	r8, r4			! arg1: curlwp
	jsr	@r0
	 mov	r14, r5			! arg2: trapframe
	__EXCEPTION_RETURN

	.align	5
.L4_SH4_PTEH:			.long	SH4_PTEH
.L4_VPN_cleanup:		.long	~0x00000c00
.L4_curptd:			.long	_C_LABEL(curptd)
.L4_kernptd:			.long	_C_LABEL(__pmap_kernel)
.L4_VM_MIN_KERNEL_ADDRESS:	.long	VM_MIN_KERNEL_ADDRESS
.L4_ptp_index_mask:		.long	0x1ff
.L4_ptp_offset_mask:		.long	0x3ff << 2
.L4_PG_HW_BITS:			.long	PG_HW_BITS
.L4_PG_V:			.long	PG_V
.L4_clear_ASID:			.long	~SH4_PTEH_ASID_MASK
.L4_curlwp:			.long	_C_LABEL(curlwp)
.L4_tlb_exception:		.long	_C_LABEL(tlb_exception)
.L4_ast:			.long	_C_LABEL(ast)

/* LINTSTUB: Var: char sh4_vector_tlbmiss_end[1]; */
VECTOR_END_MARKER(sh4_vector_tlbmiss_end)
	SET_ENTRY_SIZE(sh4_vector_tlbmiss)

#endif /* SH4 */


/*
 * LINTSTUB: Var: char sh_vector_interrupt[1];
 *
 * void sh_vector_interrupt(void);
 *	Copied to VBR+0x600.  This code should be position independent.
 */
NENTRY(sh_vector_interrupt)
	__EXCEPTION_ENTRY
	!! arguments for intc_intr(): for struct clockframe
	stc	ssr, r4
	stc	spc, r5
	stc	r0_bank, r6		! ssp
	__INTR_MASK_EXCEPTION_UNBLOCK(r0, r1, r3)

	mov.l	.Li_ci_idepth, r8	! callee-saved
	mov.l	.Li_intc_intr, r0
	mov.l	@r8, r9			! callee-saved
	mov	#0, r1
	add	#1, r9			! curcpu()->ci_idepth++
	mov.l	r1, @(TF_EXPEVT, r14)	! tf->tf_expevt = 0 (for debug)?
	jsr	@r0			! intc_intr(ssr, spc, ssp)
	 mov.l	r9, @r8

	cmp/pl	r9			! curcpu()->ci_idepth > 0
	add	#-1, r9			! curcpu()->ci_idepth--
	bt/s	.Li_return_to_kernel	! returning from a nested interrupt
	 mov.l	r9, @r8

	mov.l	@(TF_SSR, r14), r2
	mov.l	.Li_PSL_MD, r1
	tst	r1, r2			! tf->tf_ssr & PSL_MD == 0 ?
	bt	.Li_return_to_user

.Li_return_to_kernel:
	!! Check for interrupted kernel RAS when returning to kernel
	mov.l	@(TF_SPC, r14), r2
	mov.l	.Li_ras_start, r3
	cmp/hi	r3, r2			! spc > _lock_cas_ras_start ?
	bf	.Li_return_from_interrupt

	mov.l	.Li_ras_end, r1
	cmp/hs	r1, r2			! spc >= _lock_cas_ras_end ?
	bt	.Li_return_from_interrupt

	bra	.Li_return_from_interrupt
	 mov.l	r3, @(TF_SPC, r14)	! spc = _lock_cas_ras_start

.Li_return_to_user:
	/* Check for ASTs on exit to user mode. */
	mov.l	.Li_ast, r0
	mov.l	.Li_curlwp, r1
	mov	r14, r5		/* 2nd arg */
	jsr	@r0
	 mov.l	@r1, r4		/* 1st arg */

.Li_return_from_interrupt:
	__EXCEPTION_RETURN

	.align	5
.Li_ci_idepth:		.long	_C_LABEL(cpu_info_store) + CI_IDEPTH
.Li_intc_intr:		.long	_C_LABEL(intc_intr)
.Li_PSL_MD:		.long	0x40000000 /* PSL_MD */
.Li_ras_start:		.long	_C_LABEL(_lock_cas_ras_start)
.Li_ras_end:		.long	_C_LABEL(_lock_cas_ras_end)
.Li_ast:		.long	_C_LABEL(ast)
.Li_curlwp:		.long	_C_LABEL(curlwp)


/* LINTSTUB: Var: char sh_vector_interrupt_end[1]; */
VECTOR_END_MARKER(sh_vector_interrupt_end)
	SET_ENTRY_SIZE(sh_vector_interrupt)