/* $NetBSD: locore.s,v 1.434 2024/02/17 19:36:07 palle Exp $ */ /* * Copyright (c) 2006-2010 Matthew R. Green * Copyright (c) 1996-2002 Eduardo Horvath * Copyright (c) 1996 Paul Kranenburg * Copyright (c) 1996 * The President and Fellows of Harvard College. * All rights reserved. * Copyright (c) 1992, 1993 * The Regents of the University of California. * All rights reserved. * * This software was developed by the Computer Systems Engineering group * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and * contributed to Berkeley. * * All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Lawrence Berkeley Laboratory. * This product includes software developed by Harvard University. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the * distribution. * 3. All advertising materials mentioning features or use of this * software must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * This product includes software developed by Harvard University. * This product includes software developed by Paul Kranenburg. * 4. Neither the name of the University nor the names of its * contributors may be used to endorse or promote products derived * from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. * * @(#)locore.s 8.4 (Berkeley) 12/10/93 */ #undef PARANOID /* Extremely expensive consistency checks */ #undef NO_VCACHE /* Map w/D$ disabled */ #undef TRAPSTATS /* Count traps */ #undef TRAPS_USE_IG /* Use Interrupt Globals for all traps */ #define HWREF /* Track ref/mod bits in trap handlers */ #undef DCACHE_BUG /* Flush D$ around ASI_PHYS accesses */ #undef NO_TSB /* Don't use TSB */ #define BB_ERRATA_1 /* writes to TICK_CMPR may fail */ #undef TLB_FLUSH_LOWVA /* also flush 32-bit entries from the MMU */ #include "opt_ddb.h" #include "opt_kgdb.h" #include "opt_multiprocessor.h" #include "opt_compat_netbsd.h" #include "opt_compat_netbsd32.h" #include "opt_lockdebug.h" #include "assym.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SUN4V #include #endif #include #define BLOCK_SIZE SPARC64_BLOCK_SIZE #define BLOCK_ALIGN SPARC64_BLOCK_ALIGN #ifdef SUN4V #define SUN4V_N_REG_WINDOWS 8 /* As per UA2005 spec */ #define SUN4V_NWINDOWS (SUN4V_N_REG_WINDOWS-1) /* This is an index number, so subtract one */ #endif #include "ksyms.h" /* Misc. macros */ .macro GET_MAXCWP reg #ifdef SUN4V sethi %hi(cputyp), \reg ld [\reg + %lo(cputyp)], \reg cmp \reg, CPU_SUN4V bne,pt %icc, 2f nop /* sun4v */ ba 3f mov SUN4V_NWINDOWS, \reg 2: #endif /* sun4u */ rdpr %ver, \reg and \reg, CWP, \reg 3: .endm .macro SET_MMU_CONTEXTID_SUN4U ctxid,ctx stxa \ctxid, [\ctx] ASI_DMMU; .endm #ifdef SUN4V .macro SET_MMU_CONTEXTID_SUN4V ctxid,ctx stxa \ctxid, [\ctx] ASI_MMU_CONTEXTID; .endm #endif .macro SET_MMU_CONTEXTID ctxid,ctx,scratch #ifdef SUN4V sethi %hi(cputyp), \scratch ld [\scratch + %lo(cputyp)], \scratch cmp \scratch, CPU_SUN4V bne,pt %icc, 2f nop /* sun4v */ SET_MMU_CONTEXTID_SUN4V \ctxid,\ctx ba 3f nop 2: #endif /* sun4u */ SET_MMU_CONTEXTID_SUN4U \ctxid,\ctx 3: .endm .macro GET_MMU_CONTEXTID_SUN4U ctxid,ctx ldxa [\ctx] ASI_DMMU, \ctxid .endm #ifdef SUN4V .macro GET_MMU_CONTEXTID_SUN4V ctxid,ctx ldxa [\ctx] ASI_MMU_CONTEXTID, \ctxid .endm #endif .macro GET_MMU_CONTEXTID ctxid,ctx,scratch #ifdef SUN4V sethi %hi(cputyp), \scratch ld [\scratch + %lo(cputyp)], \scratch cmp \scratch, CPU_SUN4V bne,pt %icc, 2f nop /* sun4v */ GET_MMU_CONTEXTID_SUN4V \ctxid,\ctx ba 3f nop 2: #endif /* sun4u */ GET_MMU_CONTEXTID_SUN4U \ctxid,\ctx 3: .endm #ifdef SUN4V .macro NORMAL_GLOBALS_SUN4V wrpr %g0, 0, %gl ! Set globals to level 0 .endm #endif .macro NORMAL_GLOBALS_SUN4U wrpr %g0, PSTATE_KERN, %pstate ! Alternate Globals (AG) bit set to zero .endm #ifdef SUN4V .macro ALTERNATE_GLOBALS_SUN4V wrpr %g0, 1, %gl ! Set globals to level 1 .endm #endif .macro ALTERNATE_GLOBALS_SUN4U wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Alternate Globals (AG) bit set to one .endm .macro ENABLE_INTERRUPTS scratch rdpr %pstate, \scratch or \scratch, PSTATE_IE, \scratch ! Interrupt Enable (IE) bit set to one wrpr %g0, \scratch, %pstate .endm .macro DISABLE_INTERRUPTS scratch rdpr %pstate, \scratch and \scratch, ~PSTATE_IE, \scratch ! Interrupt Enable (IE) bit set to zero wrpr %g0, \scratch, %pstate .endm #ifdef SUN4V /* Misc. sun4v macros */ .macro GET_MMFSA reg sethi %hi(CPUINFO_VA + CI_MMUFSA), \reg LDPTR [\reg + %lo(CPUINFO_VA + CI_MMUFSA)], \reg .endm .macro GET_CTXBUSY reg sethi %hi(CPUINFO_VA + CI_CTXBUSY), \reg LDPTR [\reg + %lo(CPUINFO_VA + CI_CTXBUSY)], \reg .endm .macro GET_TSB_DMMU reg sethi %hi(CPUINFO_VA + CI_TSB_DMMU), \reg LDPTR [\reg + %lo(CPUINFO_VA + CI_TSB_DMMU)], \reg .endm .macro sun4v_tl1_uspill_normal ba,a,pt %xcc, spill_normal_to_user_stack nop .align 128 .endm .macro sun4v_tl1_uspill_other ba,a,pt %xcc, pcbspill_other nop .align 128 .endm #endif #if 1 /* * Try to issue an elf note to ask the Solaris * bootloader to align the kernel properly. */ .section .note .word 0x0d .word 4 ! Dunno why .word 1 0: .asciz "SUNW Solaris" 1: .align 4 .word 0x0400000 #endif .register %g2,#scratch .register %g3,#scratch .data .globl _C_LABEL(data_start) _C_LABEL(data_start): ! Start of data segment #ifdef KGDB /* * Another item that must be aligned, easiest to put it here. */ KGDB_STACK_SIZE = 2048 .globl _C_LABEL(kgdb_stack) _C_LABEL(kgdb_stack): .space KGDB_STACK_SIZE ! hope this is enough #endif #ifdef NOTDEF_DEBUG /* * This stack is used when we detect kernel stack corruption. */ .space USPACE .align 16 panicstack: #endif /* * romp is the prom entry pointer * romtba is the prom trap table base address */ .globl romp romp: POINTER 0 .globl romtba romtba: POINTER 0 .globl cputyp cputyp: .word CPU_SUN4U ! Default to sun4u _ALIGN .text /* * The v9 trap frame is stored in the special trap registers. The * register window is only modified on window overflow, underflow, * and clean window traps, where it points to the register window * needing service. Traps have space for 8 instructions, except for * the window overflow, underflow, and clean window traps which are * 32 instructions long, large enough to in-line. * * The spitfire CPU (Ultra I) has 4 different sets of global registers. * (blah blah...) * * I used to generate these numbers by address arithmetic, but gas's * expression evaluator has about as much sense as your average slug * (oddly enough, the code looks about as slimy too). Thus, all the * trap numbers are given as arguments to the trap macros. This means * there is one line per trap. Sigh. * * Hardware interrupt vectors can be `linked'---the linkage is to regular * C code---or rewired to fast in-window handlers. The latter are good * for unbuffered hardware like the Zilog serial chip and the AMD audio * chip, where many interrupts can be handled trivially with pseudo-DMA * or similar. Only one `fast' interrupt can be used per level, however, * and direct and `fast' interrupts are incompatible. Routines in intr.c * handle setting these, with optional paranoia. */ /* * TA8 -- trap align for 8 instruction traps * TA32 -- trap align for 32 instruction traps */ #define TA8 .align 32 #define TA32 .align 128 /* * v9 trap macros: * * We have a problem with v9 traps; we have no registers to put the * trap type into. But we do have a %tt register which already has * that information. Trap types in these macros are all dummys. */ /* regular vectored traps */ #define VTRAP(type, label) \ ba,a,pt %icc,label; nop; NOTREACHED; TA8 /* hardware interrupts (can be linked or made `fast') */ #define HARDINT4U(lev) \ VTRAP(lev, _C_LABEL(sparc_interrupt)) #ifdef SUN4V #define HARDINT4V(lev) HARDINT4U(lev) #endif /* software interrupts (may not be made direct, sorry---but you should not be using them trivially anyway) */ #define SOFTINT4U(lev, bit) \ HARDINT4U(lev) /* traps that just call trap() */ #define TRAP(type) VTRAP(type, slowtrap) /* architecturally undefined traps (cause panic) */ #ifndef DEBUG #define UTRAP(type) sir; VTRAP(type, slowtrap) #else #define UTRAP(type) VTRAP(type, slowtrap) #endif /* software undefined traps (may be replaced) */ #define STRAP(type) VTRAP(type, slowtrap) /* breakpoint acts differently under kgdb */ #ifdef KGDB #define BPT VTRAP(T_BREAKPOINT, bpt) #define BPT_KGDB_EXEC VTRAP(T_KGDB_EXEC, bpt) #else #define BPT TRAP(T_BREAKPOINT) #define BPT_KGDB_EXEC TRAP(T_KGDB_EXEC) #endif #define SYSCALL VTRAP(0x100, syscall_setup) #ifdef notyet #define ZS_INTERRUPT ba,a,pt %icc, zshard; nop; TA8 #else #define ZS_INTERRUPT4U HARDINT4U(12) #endif /* * Macro to clear %tt so we don't get confused with old traps. */ #ifdef DEBUG #define CLRTT wrpr %g0,0x1ff,%tt #else #define CLRTT #endif /* * Some macros to load and store a register window */ .macro SPILL storer,base,size,asi .irpc n,01234567 \storer %l\n, [\base + (\n * \size)] \asi .endr .irpc n,01234567 \storer %i\n, [\base + ((8+\n) * \size)] \asi .endr .endm .macro FILL loader, base, size, asi .irpc n,01234567 \loader [\base + (\n * \size)] \asi, %l\n .endr .irpc n,01234567 \loader [\base + ((8+\n) * \size)] \asi, %i\n .endr .endm /* * Here are some oft repeated traps as macros. */ /* spill a 64-bit register window */ #define SPILL64(label,as) \ label: \ wr %g0, as, %asi; \ stxa %l0, [%sp+BIAS+0x00]%asi; \ stxa %l1, [%sp+BIAS+0x08]%asi; \ stxa %l2, [%sp+BIAS+0x10]%asi; \ stxa %l3, [%sp+BIAS+0x18]%asi; \ stxa %l4, [%sp+BIAS+0x20]%asi; \ stxa %l5, [%sp+BIAS+0x28]%asi; \ stxa %l6, [%sp+BIAS+0x30]%asi; \ \ stxa %l7, [%sp+BIAS+0x38]%asi; \ stxa %i0, [%sp+BIAS+0x40]%asi; \ stxa %i1, [%sp+BIAS+0x48]%asi; \ stxa %i2, [%sp+BIAS+0x50]%asi; \ stxa %i3, [%sp+BIAS+0x58]%asi; \ stxa %i4, [%sp+BIAS+0x60]%asi; \ stxa %i5, [%sp+BIAS+0x68]%asi; \ stxa %i6, [%sp+BIAS+0x70]%asi; \ \ stxa %i7, [%sp+BIAS+0x78]%asi; \ saved; \ CLRTT; \ retry; \ NOTREACHED; \ TA32 /* spill a 32-bit register window */ #define SPILL32(label,as) \ label: \ wr %g0, as, %asi; \ srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ stwa %l0, [%sp+0x00]%asi; \ stwa %l1, [%sp+0x04]%asi; \ stwa %l2, [%sp+0x08]%asi; \ stwa %l3, [%sp+0x0c]%asi; \ stwa %l4, [%sp+0x10]%asi; \ stwa %l5, [%sp+0x14]%asi; \ \ stwa %l6, [%sp+0x18]%asi; \ stwa %l7, [%sp+0x1c]%asi; \ stwa %i0, [%sp+0x20]%asi; \ stwa %i1, [%sp+0x24]%asi; \ stwa %i2, [%sp+0x28]%asi; \ stwa %i3, [%sp+0x2c]%asi; \ stwa %i4, [%sp+0x30]%asi; \ stwa %i5, [%sp+0x34]%asi; \ \ stwa %i6, [%sp+0x38]%asi; \ stwa %i7, [%sp+0x3c]%asi; \ saved; \ CLRTT; \ retry; \ NOTREACHED; \ TA32 /* Spill either 32-bit or 64-bit register window. */ #define SPILLBOTH(label64,label32,as) \ andcc %sp, 1, %g0; \ bnz,pt %xcc, label64+4; /* Is it a v9 or v8 stack? */ \ wr %g0, as, %asi; \ ba,pt %xcc, label32+8; \ srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ NOTREACHED; \ TA32 /* fill a 64-bit register window */ #define FILL64(label,as) \ label: \ wr %g0, as, %asi; \ ldxa [%sp+BIAS+0x00]%asi, %l0; \ ldxa [%sp+BIAS+0x08]%asi, %l1; \ ldxa [%sp+BIAS+0x10]%asi, %l2; \ ldxa [%sp+BIAS+0x18]%asi, %l3; \ ldxa [%sp+BIAS+0x20]%asi, %l4; \ ldxa [%sp+BIAS+0x28]%asi, %l5; \ ldxa [%sp+BIAS+0x30]%asi, %l6; \ \ ldxa [%sp+BIAS+0x38]%asi, %l7; \ ldxa [%sp+BIAS+0x40]%asi, %i0; \ ldxa [%sp+BIAS+0x48]%asi, %i1; \ ldxa [%sp+BIAS+0x50]%asi, %i2; \ ldxa [%sp+BIAS+0x58]%asi, %i3; \ ldxa [%sp+BIAS+0x60]%asi, %i4; \ ldxa [%sp+BIAS+0x68]%asi, %i5; \ ldxa [%sp+BIAS+0x70]%asi, %i6; \ \ ldxa [%sp+BIAS+0x78]%asi, %i7; \ restored; \ CLRTT; \ retry; \ NOTREACHED; \ TA32 /* fill a 32-bit register window */ #define FILL32(label,as) \ label: \ wr %g0, as, %asi; \ srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ lda [%sp+0x00]%asi, %l0; \ lda [%sp+0x04]%asi, %l1; \ lda [%sp+0x08]%asi, %l2; \ lda [%sp+0x0c]%asi, %l3; \ lda [%sp+0x10]%asi, %l4; \ lda [%sp+0x14]%asi, %l5; \ \ lda [%sp+0x18]%asi, %l6; \ lda [%sp+0x1c]%asi, %l7; \ lda [%sp+0x20]%asi, %i0; \ lda [%sp+0x24]%asi, %i1; \ lda [%sp+0x28]%asi, %i2; \ lda [%sp+0x2c]%asi, %i3; \ lda [%sp+0x30]%asi, %i4; \ lda [%sp+0x34]%asi, %i5; \ \ lda [%sp+0x38]%asi, %i6; \ lda [%sp+0x3c]%asi, %i7; \ restored; \ CLRTT; \ retry; \ NOTREACHED; \ TA32 /* fill either 32-bit or 64-bit register window. */ #define FILLBOTH(label64,label32,as) \ andcc %sp, 1, %i0; \ bnz (label64)+4; /* See if it's a v9 stack or v8 */ \ wr %g0, as, %asi; \ ba (label32)+8; \ srl %sp, 0, %sp; /* fixup 32-bit pointers */ \ NOTREACHED; \ TA32 /* handle clean window trap when trap level = 0 */ .macro CLEANWIN0 rdpr %cleanwin, %o7 inc %o7 ! This handler is in-lined and cannot fault #ifdef DEBUG set 0xbadcafe, %l0 ! DEBUG -- compiler should not rely on zero-ed registers. #else clr %l0 #endif wrpr %g0, %o7, %cleanwin ! Nucleus (trap&IRQ) code does not need clean windows mov %l0,%l1; mov %l0,%l2 ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done mov %l0,%l3; mov %l0,%l4 #if 0 #ifdef DIAGNOSTIC !! !! Check the sp redzone !! !! Since we can't spill the current window, we'll just keep !! track of the frame pointer. Problems occur when the routine !! allocates and uses stack storage. !! ! rdpr %wstate, %l5 ! User stack? ! cmp %l5, WSTATE_KERN ! bne,pt %icc, 7f sethi %hi(CPCB), %l5 LDPTR [%l5 + %lo(CPCB)], %l5 ! If pcb < fp < pcb+sizeof(pcb) inc PCB_SIZE, %l5 ! then we have a stack overflow btst %fp, 1 ! 64-bit stack? sub %fp, %l5, %l7 bnz,a,pt %icc, 1f inc BIAS, %l7 ! Remove BIAS 1: cmp %l7, PCB_SIZE blu %xcc, cleanwin_overflow #endif #endif mov %l0, %l5 mov %l0, %l6; mov %l0, %l7; mov %l0, %o0; mov %l0, %o1 mov %l0, %o2; mov %l0, %o3; mov %l0, %o4; mov %l0, %o5; mov %l0, %o6; mov %l0, %o7 CLRTT retry; nop; NOTREACHED; TA32 .endm /* handle clean window trap when trap level = 1 */ .macro CLEANWIN1 clr %l0 #ifdef DEBUG set 0xbadbeef, %l0 ! DEBUG #endif mov %l0, %l1; mov %l0, %l2 rdpr %cleanwin, %o7 ! This handler is in-lined and cannot fault inc %o7; mov %l0, %l3 ! Nucleus (trap&IRQ) code does not need clean windows wrpr %g0, %o7, %cleanwin ! Clear out %l0-%l8 and %o0-%o8 and inc %cleanwin and done #ifdef NOT_DEBUG !! !! Check the sp redzone !! rdpr %wstate, t1 cmp t1, WSTATE_KERN bne,pt icc, 7f sethi %hi(_C_LABEL(redzone)), t1 ldx [t1 + %lo(_C_LABEL(redzone))], t2 cmp %sp, t2 ! if sp >= t2, not in red zone blu panic_red ! and can continue normally 7: #endif mov %l0, %l4; mov %l0, %l5; mov %l0, %l6; mov %l0, %l7 mov %l0, %o0; mov %l0, %o1; mov %l0, %o2; mov %l0, %o3 mov %l0, %o4; mov %l0, %o5; mov %l0, %o6; mov %l0, %o7 CLRTT retry; nop; TA32 .endm .globl start, _C_LABEL(kernel_text) _C_LABEL(kernel_text) = kernel_start ! for kvm_mkdb(8) kernel_start: /* Traps from TL=0 -- traps from user mode */ #ifdef __STDC__ #define TABLE(name) user_ ## name #else #define TABLE(name) user_/**/name #endif .globl _C_LABEL(trapbase) _C_LABEL(trapbase): b dostart; nop; TA8 ! 000 = reserved -- Use it to boot /* We should not get the next 5 traps */ UTRAP(0x001) ! 001 = POR Reset -- ROM should get this UTRAP(0x002) ! 002 = WDR -- ROM should get this UTRAP(0x003) ! 003 = XIR -- ROM should get this UTRAP(0x004) ! 004 = SIR -- ROM should get this UTRAP(0x005) ! 005 = RED state exception UTRAP(0x006); UTRAP(0x007) VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) TRAP(T_ILLINST) ! 010 = illegal instruction TRAP(T_PRIVINST) ! 011 = privileged instruction UTRAP(0x012) ! 012 = unimplemented LDD UTRAP(0x013) ! 013 = unimplemented STD UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) UTRAP(0x01e); UTRAP(0x01f) TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception TRAP(T_FP_OTHER) ! 022 = other fp exception TRAP(T_TAGOF) ! 023 = tag overflow CLEANWIN0 ! 024-027 = clean window trap TRAP(T_DIV0) ! 028 = divide by zero UTRAP(0x029) ! 029 = internal processor error UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault UTRAP(0x031) ! 031 = data MMU miss -- no MMU VTRAP(T_DATA_ERROR, winfault) ! 032 = data access error VTRAP(T_DATA_PROT, winfault) ! 033 = data protection fault TRAP(T_ALIGN) ! 034 = address alignment error -- we could fix it inline... TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... TRAP(T_PRIVACT) ! 037 = privileged action UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt HARDINT4U(2) ! 042 = level 2 interrupt HARDINT4U(3) ! 043 = level 3 interrupt SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt HARDINT4U(5) ! 045 = level 5 interrupt SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt HARDINT4U(7) ! 047 = level 7 interrupt HARDINT4U(8) ! 048 = level 8 interrupt HARDINT4U(9) ! 049 = level 9 interrupt HARDINT4U(10) ! 04a = level 10 interrupt HARDINT4U(11) ! 04b = level 11 interrupt ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt HARDINT4U(13) ! 04d = level 13 interrupt HARDINT4U(14) ! 04e = level 14 interrupt HARDINT4U(15) ! 04f = nonmaskable interrupt UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint TRAP(T_ECCERR) ! 063 = corrected ECC error ufast_IMMU_miss: ! 064 = fast instr access MMU miss ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer #ifdef NO_TSB ba,a %icc, instr_miss #endif ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 brgez,pn %g5, instr_miss ! Entry invalid? Punt cmp %g1, %g4 ! Compare TLB tags bne,pn %xcc, instr_miss ! Got right tag? nop CLRTT stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping retry ! Try new mapping 1: sir TA32 ufast_DMMU_miss: ! 068 = fast data access MMU miss ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer #ifdef NO_TSB ba,a %icc, data_miss #endif ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 brgez,pn %g5, data_miss ! Entry invalid? Punt cmp %g1, %g4 ! Compare TLB tags bnz,pn %xcc, data_miss ! Got right tag? nop CLRTT #ifdef TRAPSTATS sethi %hi(_C_LABEL(udhit)), %g1 lduw [%g1+%lo(_C_LABEL(udhit))], %g2 inc %g2 stw %g2, [%g1+%lo(_C_LABEL(udhit))] #endif stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping retry ! Try new mapping 1: sir TA32 ufast_DMMU_protection: ! 06c = fast data access MMU protection #ifdef TRAPSTATS sethi %hi(_C_LABEL(udprot)), %g1 lduw [%g1+%lo(_C_LABEL(udprot))], %g2 inc %g2 stw %g2, [%g1+%lo(_C_LABEL(udprot))] #endif #ifdef HWREF ba,a,pt %xcc, dmmu_write_fault #else ba,a,pt %xcc, winfault #endif nop TA32 TRAP(0x070) ! 0x070 fast_ECC_error ! Implementation dependent traps UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) TABLE(uspill): SPILL64(uspill8,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode SPILL32(uspill4,ASI_AIUS) ! 0x084 spill_1_normal SPILLBOTH(uspill8,uspill4,ASI_AIUS) ! 0x088 spill_2_normal UTRAP(0x08c); TA32 ! 0x08c spill_3_normal TABLE(kspill): SPILL64(kspill8,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows SPILL32(kspill4,ASI_N) ! 0x094 spill_5_normal SPILLBOTH(kspill8,kspill4,ASI_N) ! 0x098 spill_6_normal UTRAP(0x09c); TA32 ! 0x09c spill_7_normal TABLE(uspillk): SPILL64(uspillk8,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode SPILL32(uspillk4,ASI_AIUS) ! 0x0a4 spill_1_other SPILLBOTH(uspillk8,uspillk4,ASI_AIUS) ! 0x0a8 spill_2_other UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other TABLE(ufill): FILL64(ufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode FILL32(ufill4,ASI_AIUS) ! 0x0c4 fill_1_normal FILLBOTH(ufill8,ufill4,ASI_AIUS) ! 0x0c8 fill_2_normal UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal TABLE(kfill): FILL64(kfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode FILL32(kfill4,ASI_N) ! 0x0d4 fill_5_normal FILLBOTH(kfill8,kfill4,ASI_N) ! 0x0d8 fill_6_normal UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal TABLE(ufillk): FILL64(ufillk8,ASI_AIUS) ! 0x0e0 fill_0_other FILL32(ufillk4,ASI_AIUS) ! 0x0e4 fill_1_other FILLBOTH(ufillk8,ufillk4,ASI_AIUS) ! 0x0e8 fill_2_other UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other TABLE(syscall): SYSCALL ! 0x100 = sun syscall BPT ! 0x101 = pseudo breakpoint instruction STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) SYSCALL ! 0x108 = svr4 syscall SYSCALL ! 0x109 = bsd syscall BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) SYSCALL ! 0x140 SVID syscall (Solaris 2.7) SYSCALL ! 0x141 SPARC International syscall SYSCALL ! 0x142 OS Vendor syscall SYSCALL ! 0x143 HW OEM syscall STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) ! Traps beyond 0x17f are reserved UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) /* Traps from TL>0 -- traps from supervisor mode */ #undef TABLE #ifdef __STDC__ #define TABLE(name) nucleus_ ## name #else #define TABLE(name) nucleus_/**/name #endif trapbase_priv: UTRAP(0x000) ! 000 = reserved -- Use it to boot /* We should not get the next 5 traps */ UTRAP(0x001) ! 001 = POR Reset -- ROM should get this UTRAP(0x002) ! 002 = WDR Watchdog -- ROM should get this UTRAP(0x003) ! 003 = XIR -- ROM should get this UTRAP(0x004) ! 004 = SIR -- ROM should get this UTRAP(0x005) ! 005 = RED state exception UTRAP(0x006); UTRAP(0x007) ktextfault: VTRAP(T_INST_EXCEPT, textfault) ! 008 = instr. access except VTRAP(T_TEXTFAULT, textfault) ! 009 = instr access MMU miss -- no MMU VTRAP(T_INST_ERROR, textfault) ! 00a = instr. access err UTRAP(0x00b); UTRAP(0x00c); UTRAP(0x00d); UTRAP(0x00e); UTRAP(0x00f) TRAP(T_ILLINST) ! 010 = illegal instruction TRAP(T_PRIVINST) ! 011 = privileged instruction UTRAP(0x012) ! 012 = unimplemented LDD UTRAP(0x013) ! 013 = unimplemented STD UTRAP(0x014); UTRAP(0x015); UTRAP(0x016); UTRAP(0x017); UTRAP(0x018) UTRAP(0x019); UTRAP(0x01a); UTRAP(0x01b); UTRAP(0x01c); UTRAP(0x01d) UTRAP(0x01e); UTRAP(0x01f) TRAP(T_FPDISABLED) ! 020 = fp instr, but EF bit off in psr TRAP(T_FP_IEEE_754) ! 021 = ieee 754 exception TRAP(T_FP_OTHER) ! 022 = other fp exception TRAP(T_TAGOF) ! 023 = tag overflow CLEANWIN1 ! 024-027 = clean window trap TRAP(T_DIV0) ! 028 = divide by zero UTRAP(0x029) ! 029 = internal processor error UTRAP(0x02a); UTRAP(0x02b); UTRAP(0x02c); UTRAP(0x02d); UTRAP(0x02e); UTRAP(0x02f) kdatafault: VTRAP(T_DATAFAULT, winfault) ! 030 = data fetch fault UTRAP(0x031) ! 031 = data MMU miss -- no MMU VTRAP(T_DATA_ERROR, winfault) ! 032 = data fetch fault VTRAP(T_DATA_PROT, winfault) ! 033 = data fetch fault VTRAP(T_ALIGN, checkalign) ! 034 = address alignment error -- we could fix it inline... TRAP(T_LDDF_ALIGN) ! 035 = LDDF address alignment error -- we could fix it inline... TRAP(T_STDF_ALIGN) ! 036 = STDF address alignment error -- we could fix it inline... TRAP(T_PRIVACT) ! 037 = privileged action UTRAP(0x038); UTRAP(0x039); UTRAP(0x03a); UTRAP(0x03b); UTRAP(0x03c); UTRAP(0x03d); UTRAP(0x03e); UTRAP(0x03f); VTRAP(T_ASYNC_ERROR, winfault) ! 040 = data fetch fault SOFTINT4U(1, IE_L1) ! 041 = level 1 interrupt HARDINT4U(2) ! 042 = level 2 interrupt HARDINT4U(3) ! 043 = level 3 interrupt SOFTINT4U(4, IE_L4) ! 044 = level 4 interrupt HARDINT4U(5) ! 045 = level 5 interrupt SOFTINT4U(6, IE_L6) ! 046 = level 6 interrupt HARDINT4U(7) ! 047 = level 7 interrupt HARDINT4U(8) ! 048 = level 8 interrupt HARDINT4U(9) ! 049 = level 9 interrupt HARDINT4U(10) ! 04a = level 10 interrupt HARDINT4U(11) ! 04b = level 11 interrupt ZS_INTERRUPT4U ! 04c = level 12 (zs) interrupt HARDINT4U(13) ! 04d = level 13 interrupt HARDINT4U(14) ! 04e = level 14 interrupt HARDINT4U(15) ! 04f = nonmaskable interrupt UTRAP(0x050); UTRAP(0x051); UTRAP(0x052); UTRAP(0x053); UTRAP(0x054); UTRAP(0x055) UTRAP(0x056); UTRAP(0x057); UTRAP(0x058); UTRAP(0x059); UTRAP(0x05a); UTRAP(0x05b) UTRAP(0x05c); UTRAP(0x05d); UTRAP(0x05e); UTRAP(0x05f) VTRAP(0x060, interrupt_vector); ! 060 = interrupt vector TRAP(T_PA_WATCHPT) ! 061 = physical address data watchpoint TRAP(T_VA_WATCHPT) ! 062 = virtual address data watchpoint TRAP(T_ECCERR) ! 063 = corrected ECC error kfast_IMMU_miss: ! 064 = fast instr access MMU miss ldxa [%g0] ASI_IMMU_8KPTR, %g2 ! Load IMMU 8K TSB pointer #ifdef NO_TSB ba,a %icc, instr_miss #endif ldxa [%g0] ASI_IMMU, %g1 ! Load IMMU tag target register ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag:data into %g4:%g5 brgez,pn %g5, instr_miss ! Entry invalid? Punt cmp %g1, %g4 ! Compare TLB tags bne,pn %xcc, instr_miss ! Got right tag? nop CLRTT stxa %g5, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping retry ! Try new mapping 1: sir TA32 kfast_DMMU_miss: ! 068 = fast data access MMU miss ldxa [%g0] ASI_DMMU_8KPTR, %g2! Load DMMU 8K TSB pointer #ifdef NO_TSB ba,a %icc, data_miss #endif ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register ldda [%g2] ASI_NUCLEUS_QUAD_LDD, %g4 ! Load TSB tag and data into %g4 and %g5 brgez,pn %g5, data_miss ! Entry invalid? Punt cmp %g1, %g4 ! Compare TLB tags bnz,pn %xcc, data_miss ! Got right tag? nop CLRTT #ifdef TRAPSTATS sethi %hi(_C_LABEL(kdhit)), %g1 lduw [%g1+%lo(_C_LABEL(kdhit))], %g2 inc %g2 stw %g2, [%g1+%lo(_C_LABEL(kdhit))] #endif stxa %g5, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping retry ! Try new mapping 1: sir TA32 kfast_DMMU_protection: ! 06c = fast data access MMU protection #ifdef TRAPSTATS sethi %hi(_C_LABEL(kdprot)), %g1 lduw [%g1+%lo(_C_LABEL(kdprot))], %g2 inc %g2 stw %g2, [%g1+%lo(_C_LABEL(kdprot))] #endif #ifdef HWREF ba,a,pt %xcc, dmmu_write_fault #else ba,a,pt %xcc, winfault #endif nop TA32 TRAP(0x070) ! 0x070 fast_ECC_error ! Implementation dependent traps UTRAP(0x071); UTRAP(0x072); UTRAP(0x073); UTRAP(0x074); UTRAP(0x075); UTRAP(0x076) UTRAP(0x077); UTRAP(0x078); UTRAP(0x079); UTRAP(0x07a); UTRAP(0x07b); UTRAP(0x07c) UTRAP(0x07d); UTRAP(0x07e); UTRAP(0x07f) TABLE(uspill): SPILL64(1,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows SPILL32(2,ASI_AIUS) ! 0x084 spill_1_normal SPILLBOTH(1b,2b,ASI_AIUS) ! 0x088 spill_2_normal UTRAP(0x08c); TA32 ! 0x08c spill_3_normal TABLE(kspill): SPILL64(1,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows SPILL32(2,ASI_N) ! 0x094 spill_5_normal SPILLBOTH(1b,2b,ASI_N) ! 0x098 spill_6_normal UTRAP(0x09c); TA32 ! 0x09c spill_7_normal TABLE(uspillk): SPILL64(1,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in nucleus mode SPILL32(2,ASI_AIUS) ! 0x0a4 spill_1_other SPILLBOTH(1b,2b,ASI_AIUS) ! 0x0a8 spill_2_other UTRAP(0x0ac); TA32 ! 0x0ac spill_3_other UTRAP(0x0b0); TA32 ! 0x0b0 spill_4_other UTRAP(0x0b4); TA32 ! 0x0b4 spill_5_other UTRAP(0x0b8); TA32 ! 0x0b8 spill_6_other UTRAP(0x0bc); TA32 ! 0x0bc spill_7_other TABLE(ufill): FILL64(nufill8,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running nucleus mode from user FILL32(nufill4,ASI_AIUS) ! 0x0c4 fill_1_normal FILLBOTH(nufill8,nufill4,ASI_AIUS) ! 0x0c8 fill_2_normal UTRAP(0x0cc); TA32 ! 0x0cc fill_3_normal TABLE(sfill): FILL64(sfill8,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running nucleus mode from supervisor FILL32(sfill4,ASI_N) ! 0x0d4 fill_5_normal FILLBOTH(sfill8,sfill4,ASI_N) ! 0x0d8 fill_6_normal UTRAP(0x0dc); TA32 ! 0x0dc fill_7_normal TABLE(kfill): FILL64(nkfill8,ASI_AIUS) ! 0x0e0 fill_0_other -- used to fill user windows when running nucleus mode -- will we ever use this? FILL32(nkfill4,ASI_AIUS) ! 0x0e4 fill_1_other FILLBOTH(nkfill8,nkfill4,ASI_AIUS)! 0x0e8 fill_2_other UTRAP(0x0ec); TA32 ! 0x0ec fill_3_other UTRAP(0x0f0); TA32 ! 0x0f0 fill_4_other UTRAP(0x0f4); TA32 ! 0x0f4 fill_5_other UTRAP(0x0f8); TA32 ! 0x0f8 fill_6_other UTRAP(0x0fc); TA32 ! 0x0fc fill_7_other TABLE(syscall): SYSCALL ! 0x100 = sun syscall BPT ! 0x101 = pseudo breakpoint instruction STRAP(0x102); STRAP(0x103); STRAP(0x104); STRAP(0x105); STRAP(0x106); STRAP(0x107) SYSCALL ! 0x108 = svr4 syscall SYSCALL ! 0x109 = bsd syscall BPT_KGDB_EXEC ! 0x10a = enter kernel gdb on kernel startup STRAP(0x10b); STRAP(0x10c); STRAP(0x10d); STRAP(0x10e); STRAP(0x10f); STRAP(0x110); STRAP(0x111); STRAP(0x112); STRAP(0x113); STRAP(0x114); STRAP(0x115); STRAP(0x116); STRAP(0x117) STRAP(0x118); STRAP(0x119); STRAP(0x11a); STRAP(0x11b); STRAP(0x11c); STRAP(0x11d); STRAP(0x11e); STRAP(0x11f) STRAP(0x120); STRAP(0x121); STRAP(0x122); STRAP(0x123); STRAP(0x124); STRAP(0x125); STRAP(0x126); STRAP(0x127) STRAP(0x128); STRAP(0x129); STRAP(0x12a); STRAP(0x12b); STRAP(0x12c); STRAP(0x12d); STRAP(0x12e); STRAP(0x12f) STRAP(0x130); STRAP(0x131); STRAP(0x132); STRAP(0x133); STRAP(0x134); STRAP(0x135); STRAP(0x136); STRAP(0x137) STRAP(0x138); STRAP(0x139); STRAP(0x13a); STRAP(0x13b); STRAP(0x13c); STRAP(0x13d); STRAP(0x13e); STRAP(0x13f) STRAP(0x140); STRAP(0x141); STRAP(0x142); STRAP(0x143); STRAP(0x144); STRAP(0x145); STRAP(0x146); STRAP(0x147) STRAP(0x148); STRAP(0x149); STRAP(0x14a); STRAP(0x14b); STRAP(0x14c); STRAP(0x14d); STRAP(0x14e); STRAP(0x14f) STRAP(0x150); STRAP(0x151); STRAP(0x152); STRAP(0x153); STRAP(0x154); STRAP(0x155); STRAP(0x156); STRAP(0x157) STRAP(0x158); STRAP(0x159); STRAP(0x15a); STRAP(0x15b); STRAP(0x15c); STRAP(0x15d); STRAP(0x15e); STRAP(0x15f) STRAP(0x160); STRAP(0x161); STRAP(0x162); STRAP(0x163); STRAP(0x164); STRAP(0x165); STRAP(0x166); STRAP(0x167) STRAP(0x168); STRAP(0x169); STRAP(0x16a); STRAP(0x16b); STRAP(0x16c); STRAP(0x16d); STRAP(0x16e); STRAP(0x16f) STRAP(0x170); STRAP(0x171); STRAP(0x172); STRAP(0x173); STRAP(0x174); STRAP(0x175); STRAP(0x176); STRAP(0x177) STRAP(0x178); STRAP(0x179); STRAP(0x17a); STRAP(0x17b); STRAP(0x17c); STRAP(0x17d); STRAP(0x17e); STRAP(0x17f) ! Traps beyond 0x17f are reserved UTRAP(0x180); UTRAP(0x181); UTRAP(0x182); UTRAP(0x183); UTRAP(0x184); UTRAP(0x185); UTRAP(0x186); UTRAP(0x187) UTRAP(0x188); UTRAP(0x189); UTRAP(0x18a); UTRAP(0x18b); UTRAP(0x18c); UTRAP(0x18d); UTRAP(0x18e); UTRAP(0x18f) UTRAP(0x190); UTRAP(0x191); UTRAP(0x192); UTRAP(0x193); UTRAP(0x194); UTRAP(0x195); UTRAP(0x196); UTRAP(0x197) UTRAP(0x198); UTRAP(0x199); UTRAP(0x19a); UTRAP(0x19b); UTRAP(0x19c); UTRAP(0x19d); UTRAP(0x19e); UTRAP(0x19f) UTRAP(0x1a0); UTRAP(0x1a1); UTRAP(0x1a2); UTRAP(0x1a3); UTRAP(0x1a4); UTRAP(0x1a5); UTRAP(0x1a6); UTRAP(0x1a7) UTRAP(0x1a8); UTRAP(0x1a9); UTRAP(0x1aa); UTRAP(0x1ab); UTRAP(0x1ac); UTRAP(0x1ad); UTRAP(0x1ae); UTRAP(0x1af) UTRAP(0x1b0); UTRAP(0x1b1); UTRAP(0x1b2); UTRAP(0x1b3); UTRAP(0x1b4); UTRAP(0x1b5); UTRAP(0x1b6); UTRAP(0x1b7) UTRAP(0x1b8); UTRAP(0x1b9); UTRAP(0x1ba); UTRAP(0x1bb); UTRAP(0x1bc); UTRAP(0x1bd); UTRAP(0x1be); UTRAP(0x1bf) UTRAP(0x1c0); UTRAP(0x1c1); UTRAP(0x1c2); UTRAP(0x1c3); UTRAP(0x1c4); UTRAP(0x1c5); UTRAP(0x1c6); UTRAP(0x1c7) UTRAP(0x1c8); UTRAP(0x1c9); UTRAP(0x1ca); UTRAP(0x1cb); UTRAP(0x1cc); UTRAP(0x1cd); UTRAP(0x1ce); UTRAP(0x1cf) UTRAP(0x1d0); UTRAP(0x1d1); UTRAP(0x1d2); UTRAP(0x1d3); UTRAP(0x1d4); UTRAP(0x1d5); UTRAP(0x1d6); UTRAP(0x1d7) UTRAP(0x1d8); UTRAP(0x1d9); UTRAP(0x1da); UTRAP(0x1db); UTRAP(0x1dc); UTRAP(0x1dd); UTRAP(0x1de); UTRAP(0x1df) UTRAP(0x1e0); UTRAP(0x1e1); UTRAP(0x1e2); UTRAP(0x1e3); UTRAP(0x1e4); UTRAP(0x1e5); UTRAP(0x1e6); UTRAP(0x1e7) UTRAP(0x1e8); UTRAP(0x1e9); UTRAP(0x1ea); UTRAP(0x1eb); UTRAP(0x1ec); UTRAP(0x1ed); UTRAP(0x1ee); UTRAP(0x1ef) UTRAP(0x1f0); UTRAP(0x1f1); UTRAP(0x1f2); UTRAP(0x1f3); UTRAP(0x1f4); UTRAP(0x1f5); UTRAP(0x1f6); UTRAP(0x1f7) UTRAP(0x1f8); UTRAP(0x1f9); UTRAP(0x1fa); UTRAP(0x1fb); UTRAP(0x1fc); UTRAP(0x1fd); UTRAP(0x1fe); UTRAP(0x1ff) #ifdef SUN4V /* Macros for sun4v traps */ .macro sun4v_trap_entry count .rept \count ba slowtrap nop .align 32 .endr .endm .macro sun4v_trap_entry_fail count .rept \count sir .align 32 .endr .endm .macro sun4v_trap_entry_spill_fill_fail count .rept \count sir .align 128 .endr .endm /* The actual trap base for sun4v */ .align 0x8000 .globl _C_LABEL(trapbase_sun4v) _C_LABEL(trapbase_sun4v): ! ! trap level 0 ! sun4v_trap_entry 8 ! 0x000-0x007 VTRAP(T_INST_EXCEPT, sun4v_tl0_itsb_miss) ! 0x008 - inst except VTRAP(T_TEXTFAULT, sun4v_tl0_itsb_miss) ! 0x009 - inst MMU miss sun4v_trap_entry 26 ! 0x00a-0x023 CLEANWIN0 ! 0x24-0x27 = clean window sun4v_trap_entry 9 ! 0x028-0x030 VTRAP(T_DATA_MMU_MISS, sun4v_dtsb_miss) ! 0x031 = data MMU miss sun4v_trap_entry 2 ! 0x032-0x033 TRAP(T_ALIGN) ! 0x034 = address alignment error sun4v_trap_entry 12 ! 0x035-0x040 HARDINT4V(1) ! 0x041 = level 1 interrupt HARDINT4V(2) ! 0x042 = level 2 interrupt HARDINT4V(3) ! 0x043 = level 3 interrupt HARDINT4V(4) ! 0x044 = level 4 interrupt HARDINT4V(5) ! 0x045 = level 5 interrupt HARDINT4V(6) ! 0x046 = level 6 interrupt HARDINT4V(7) ! 0x047 = level 7 interrupt HARDINT4V(8) ! 0x048 = level 8 interrupt HARDINT4V(9) ! 0x049 = level 9 interrupt HARDINT4V(10) ! 0x04a = level 10 interrupt HARDINT4V(11) ! 0x04b = level 11 interrupt HARDINT4V(12) ! 0x04c = level 12 interrupt HARDINT4V(13) ! 0x04d = level 13 interrupt HARDINT4V(14) ! 0x04e = level 14 interrupt HARDINT4V(15) ! 0x04f = level 15 interrupt sun4v_trap_entry 28 ! 0x050-0x06b VTRAP(T_FDMMU_PROT, sun4v_tl0_dtsb_prot) ! 0x06c sun4v_trap_entry 15 ! 0x06d-0x07b VTRAP(T_CPU_MONDO, sun4v_cpu_mondo) ! 0x07c = cpu mondo VTRAP(T_DEV_MONDO, sun4v_dev_mondo) ! 0x07d = dev mondo sun4v_trap_entry 2 ! 0x07e-0x07f SPILL64(uspill8_sun4vt0,ASI_AIUS) ! 0x080 spill_0_normal -- used to save user windows in user mode SPILL32(uspill4_sun4vt0,ASI_AIUS) ! 0x084 spill_1_normal SPILLBOTH(uspill8_sun4vt0,uspill4_sun4vt0,ASI_AIUS) ! 0x088 spill_2_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x08c spill_3_normal SPILL64(kspill8_sun4vt0,ASI_N) ! 0x090 spill_4_normal -- used to save supervisor windows SPILL32(kspill4_sun4vt0,ASI_N) ! 0x094 spill_5_normal SPILLBOTH(kspill8_sun4vt0,kspill4_sun4vt0,ASI_N) ! 0x098 spill_6_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x09c spill_7_normal SPILL64(uspillk8_sun4vt0,ASI_AIUS) ! 0x0a0 spill_0_other -- used to save user windows in supervisor mode SPILL32(uspillk4_sun4vt0,ASI_AIUS) ! 0x0a4 spill_1_other SPILLBOTH(uspillk8_sun4vt0,uspillk4_sun4vt0,ASI_AIUS) ! 0x0a8 spill_2_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0ac spill_3_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0b0 spill_4_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0b4 spill_5_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0b8 spill_6_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0bc spill_7_other FILL64(ufill8_sun4vt0,ASI_AIUS) ! 0x0c0 fill_0_normal -- used to fill windows when running user mode FILL32(ufill4_sun4vt0,ASI_AIUS) ! 0x0c4 fill_1_normal FILLBOTH(ufill8_sun4vt0,ufill4_sun4vt0,ASI_AIUS) ! 0x0c8 fill_2_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x0cc fill_3_normal FILL64(kfill8_sun4vt0,ASI_N) ! 0x0d0 fill_4_normal -- used to fill windows when running supervisor mode FILL32(kfill4_sun4vt0,ASI_N) ! 0x0d4 fill_5_normal FILLBOTH(kfill8_sun4vt0,kfill4_sun4vt0,ASI_N) ! 0x0d8 fill_6_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x0dc fill_7_normal FILL64(ufillk8_sun4vt0,ASI_AIUS) ! 0x0e0 fill_0_other FILL32(ufillk4_sun4vt0,ASI_AIUS) ! 0x0e4 fill_1_other FILLBOTH(ufillk8_sun4vt0,ufillk4_sun4vt0,ASI_AIUS) ! 0x0e8 fill_2_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0ec fill_3_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0f0 fill_4_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0f4 fill_5_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0f8 fill_6_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0fc fill_7_other SYSCALL ! 0x100 = syscall BPT ! 0x101 = pseudo breakpoint instruction sun4v_trap_entry 254 ! 0x102-0x1ff ! ! trap level 1 ! sun4v_trap_entry 36 ! 0x000-0x023 CLEANWIN1 ! 0x24-0x27 = clean window sun4v_trap_entry 8 ! 0x028-0x02F VTRAP(T_DATAFAULT, sun4v_tl1_ptbl_miss) ! 0x030 = ??? VTRAP(T_DATA_MMU_MISS, sun4v_tl1_dtsb_miss) ! 0x031 = data MMU miss VTRAP(T_DATA_ERROR, sun4v_tl1_ptbl_miss) ! 0x032 = ??? VTRAP(T_DATA_PROT, sun4v_tl1_ptbl_miss) ! 0x033 = ??? sun4v_trap_entry 56 ! 0x034-0x06b VTRAP(T_FDMMU_PROT, sun4v_tl1_dtsb_prot) ! 0x06c sun4v_trap_entry 19 ! 0x06d-0x07f sun4v_tl1_uspill_normal ! 0x080 spill_0_normal -- save user windows sun4v_tl1_uspill_normal ! 0x084 spill_1_normal sun4v_tl1_uspill_normal ! 0x088 spill_2_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x08c spill_3_normal SPILL64(kspill8_sun4vt1,ASI_N) ! 0x090 spill_4_normal -- save supervisor windows SPILL32(kspill4_sun4vt1,ASI_N) ! 0x094 spill_5_normal SPILLBOTH(kspill8_sun4vt1,kspill4_sun4vt1,ASI_N) ! 0x098 spill_6_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x09c spill_7_normal sun4v_tl1_uspill_other ! 0x0a0 spill_0_other -- save user windows in nucleus mode sun4v_tl1_uspill_other ! 0x0a4 spill_1_other sun4v_tl1_uspill_other ! 0x0a8 spill_2_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0ac spill_3_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0b0 spill_4_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0b4 spill_5_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0b8 spill_6_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0bc spill_7_other FILL64(ufill8_sun4vt1,ASI_AIUS) ! 0x0c0 fill_0_normal -- fill windows when running nucleus mode from user FILL32(ufill4_sun4vt1,ASI_AIUS) ! 0x0c4 fill_1_normal FILLBOTH(ufill8_sun4vt1,ufill4_sun4vt1,ASI_AIUS) ! 0x0c8 fill_2_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x0cc fill_3_normal FILL64(kfill8_sun4vt1,ASI_N) ! 0x0d0 fill_4_normal -- fill windows when running nucleus mode from supervisor FILL32(kfill4_sun4vt1,ASI_N) ! 0x0d4 fill_5_normal FILLBOTH(kfill8_sun4vt1,kfill4_sun4vt1,ASI_N) ! 0x0d8 fill_6_normal sun4v_trap_entry_spill_fill_fail 1 ! 0x0dc fill_7_normal FILL64(ufillk8_sun4vt1,ASI_AIUS) ! 0x0e0 fill_0_other -- fill user windows when running nucleus mode -- will we ever use this? FILL32(ufillk4_sun4vt1,ASI_AIUS) ! 0x0e4 fill_1_other FILLBOTH(ufillk8_sun4vt1,ufillk4_sun4vt1,ASI_AIUS) ! 0x0e8 fill_2_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0ec fill_3_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0f0 fill_4_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0f4 fill_5_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0f8 fill_6_other sun4v_trap_entry_spill_fill_fail 1 ! 0x0fc fill_7_other sun4v_trap_entry_fail 256 ! 0x100-0x1ff #endif #if 0 /* * If the cleanwin trap handler detects an overflow we come here. * We need to fix up the window registers, switch to the interrupt * stack, and then trap to the debugger. */ cleanwin_overflow: !! We've already incremented %cleanwin !! So restore %cwp rdpr %cwp, %l0 dec %l0 wrpr %l0, %g0, %cwp set EINTSTACK-STKB-CC64FSZ, %l0 save %l0, 0, %sp ta 1 ! Enter debugger sethi %hi(1f), %o0 call _C_LABEL(panic) or %o0, %lo(1f), %o0 restore retry .data 1: .asciz "Kernel stack overflow!" _ALIGN .text #endif #ifdef NOTDEF_DEBUG /* * A hardware red zone is impossible. We simulate one in software by * keeping a `red zone' pointer; if %sp becomes less than this, we panic. * This is expensive and is only enabled when debugging. */ #define REDSIZE (PCB_SIZE) /* Mark used portion of pcb structure out of bounds */ #define REDSTACK 2048 /* size of `panic: stack overflow' region */ .data _ALIGN redzone: .xword _C_LABEL(XXX) + REDSIZE redstack: .space REDSTACK eredstack: Lpanic_red: .asciz "kernel stack overflow" _ALIGN .text /* set stack pointer redzone to base+minstack; alters base */ #define SET_SP_REDZONE(base, tmp) \ add base, REDSIZE, base; \ sethi %hi(_C_LABEL(redzone)), tmp; \ stx base, [tmp + %lo(_C_LABEL(redzone))] /* variant with a constant */ #define SET_SP_REDZONE_CONST(const, tmp1, tmp2) \ set (const) + REDSIZE, tmp1; \ sethi %hi(_C_LABEL(redzone)), tmp2; \ stx tmp1, [tmp2 + %lo(_C_LABEL(redzone))] /* check stack pointer against redzone (uses two temps) */ #define CHECK_SP_REDZONE(t1, t2) \ sethi KERNBASE, t1; \ cmp %sp, t1; \ blu,pt %xcc, 7f; \ sethi %hi(_C_LABEL(redzone)), t1; \ ldx [t1 + %lo(_C_LABEL(redzone))], t2; \ cmp %sp, t2; /* if sp >= t2, not in red zone */ \ blu panic_red; nop; /* and can continue normally */ \ 7: panic_red: /* move to panic stack */ stx %g0, [t1 + %lo(_C_LABEL(redzone))]; set eredstack - BIAS, %sp; /* prevent panic() from lowering ipl */ sethi %hi(_C_LABEL(panicstr)), t2; set Lpanic_red, t2; st t2, [t1 + %lo(_C_LABEL(panicstr))]; wrpr g0, 15, %pil /* t1 = splhigh() */ save %sp, -CCF64SZ, %sp; /* preserve current window */ sethi %hi(Lpanic_red), %o0; call _C_LABEL(panic); or %o0, %lo(Lpanic_red), %o0; #else #define SET_SP_REDZONE(base, tmp) #define SET_SP_REDZONE_CONST(const, t1, t2) #define CHECK_SP_REDZONE(t1, t2) #endif #define TRACESIZ 0x01000 .globl _C_LABEL(trap_trace) .globl _C_LABEL(trap_trace_ptr) .globl _C_LABEL(trap_trace_end) .globl _C_LABEL(trap_trace_dis) .data _C_LABEL(trap_trace_dis): .word 1, 1 ! Starts disabled. DDB turns it on. _C_LABEL(trap_trace_ptr): .word 0, 0, 0, 0 _C_LABEL(trap_trace): .space TRACESIZ _C_LABEL(trap_trace_end): .space 0x20 ! safety margin /* * v9 machines do not have a trap window. * * When we take a trap the trap state is pushed on to the stack of trap * registers, interrupts are disabled, then we switch to an alternate set * of global registers. * * The trap handling code needs to allocate a trap frame on the kernel, or * for interrupts, the interrupt stack, save the out registers to the trap * frame, then switch to the normal globals and save them to the trap frame * too. * * XXX it would be good to save the interrupt stack frame to the kernel * stack so we wouldn't have to copy it later if we needed to handle a AST. * * Since kernel stacks are all on one page and the interrupt stack is entirely * within the locked TLB, we can use physical addressing to save out our * trap frame so we don't trap during the TRAP_SETUP() operation. There * is unfortunately no supportable method for issuing a non-trapping save. * * However, if we use physical addresses to save our trapframe, we will need * to clear out the data cache before continuing much further. * * In short, what we need to do is: * * all preliminary processing is done using the alternate globals * * When we allocate our trap windows we must give up our globals because * their state may have changed during the save operation * * we need to save our normal globals as soon as we have a stack * * Finally, we may now call C code. * * This macro will destroy %g5-%g7. %g0-%g4 remain unchanged. * * In order to properly handle nested traps without lossage, alternate * global %g6 is used as a kernel stack pointer. It is set to the last * allocated stack pointer (trapframe) and the old value is stored in * tf_kstack. It is restored when returning from a trap. It is cleared * on entering user mode. */ /* * Other misc. design criteria: * * When taking an address fault, fault info is in the sfsr, sfar, * TLB_TAG_ACCESS registers. If we take another address fault * while trying to handle the first fault then that information, * the only information that tells us what address we trapped on, * can potentially be lost. This trap can be caused when allocating * a register window with which to handle the trap because the save * may try to store or restore a register window that corresponds * to part of the stack that is not mapped. Preventing this trap, * while possible, is much too complicated to do in a trap handler, * and then we will need to do just as much work to restore the processor * window state. * * Possible solutions to the problem: * * Since we have separate AG, MG, and IG, we could have all traps * above level-1 preserve AG and use other registers. This causes * a problem for the return from trap code which is coded to use * alternate globals only. * * We could store the trapframe and trap address info to the stack * using physical addresses. Then we need to read it back using * physical addressing, or flush the D$. * * We could identify certain registers to hold address fault info. * this means that these registers need to be preserved across all * fault handling. But since we only have 7 useable globals, that * really puts a cramp in our style. * * Finally, there is the issue of returning from kernel mode to user * mode. If we need to issue a restore of a user window in kernel * mode, we need the window control registers in a user mode setup. * If the trap handlers notice the register windows are in user mode, * they will allocate a trapframe at the bottom of the kernel stack, * overwriting the frame we were trying to return to. This means that * we must complete the restoration of all registers *before* switching * to a user-mode window configuration. * * Essentially we need to be able to write re-entrant code w/no stack. */ .data trap_setup_msg: .asciz "TRAP_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" _ALIGN intr_setup_msg: .asciz "INTR_SETUP: tt=%x osp=%x nsp=%x tl=%x tpc=%x\n" _ALIGN .text #ifdef DEBUG /* Only save a snapshot of locals and ins in DEBUG kernels */ #define SAVE_LOCALS_INS \ /* Save local registers to trap frame */ \ stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)]; \ stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)]; \ stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)]; \ stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)]; \ stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)]; \ stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)]; \ stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)]; \ stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)]; \ \ /* Save in registers to trap frame */ \ stx %i0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)]; \ stx %i1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)]; \ stx %i2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)]; \ stx %i3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)]; \ stx %i4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)]; \ stx %i5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)]; \ stx %i6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)]; \ stx %i7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)]; \ \ stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT]; #else #define SAVE_LOCALS_INS #endif #ifdef _LP64 #define FIXUP_TRAP_STACK \ btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ bnz,pt %icc, 1f; \ add %g6, %g5, %g6; /* Allocate a stack frame */ \ inc -BIAS, %g6; \ 1: #else #define FIXUP_TRAP_STACK \ srl %g6, 0, %g6; /* truncate at 32-bits */ \ btst 1, %g6; /* Fixup 64-bit stack if necessary */ \ add %g6, %g5, %g6; /* Allocate a stack frame */ \ add %g6, BIAS, %g5; \ movne %icc, %g5, %g6; #endif #ifdef _LP64 #define TRAP_SETUP(stackspace) \ sethi %hi(CPCB), %g6; \ sethi %hi((stackspace)), %g5; \ LDPTR [%g6 + %lo(CPCB)], %g6; \ sethi %hi(USPACE), %g7; /* Always multiple of page size */ \ or %g5, %lo((stackspace)), %g5; \ add %g6, %g7, %g6; \ rdpr %wstate, %g7; /* Find if we're from user mode */ \ sra %g5, 0, %g5; /* Sign extend the damn thing */ \ \ sub %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ movrz %g7, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ FIXUP_TRAP_STACK \ SAVE_LOCALS_INS \ save %g6, 0, %sp; /* If we fault we should come right back here */ \ stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ \ stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ brz,pt %g7, 1f; /* If we were in kernel mode start saving globals */ \ stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ mov CTX_PRIMARY, %g7; \ /* came from user mode -- switch to kernel mode stack */ \ rdpr %canrestore, %g5; /* Fixup register window state registers */ \ wrpr %g0, 0, %canrestore; \ wrpr %g0, %g5, %otherwin; \ wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ \ SET_MMU_CONTEXTID %g0, %g7,%g5; /* Switch MMU to kernel primary context */ \ sethi %hi(KERNBASE), %g5; \ flush %g5; /* Some convenient address that won't trap */ \ 1: /* * Interrupt setup is almost exactly like trap setup, but we need to * go to the interrupt stack if (a) we came from user mode or (b) we * came from kernel mode on the kernel stack. * * We don't guarantee any registers are preserved during this operation. * So we can be more efficient. */ #define INTR_SETUP(stackspace) \ rdpr %wstate, %g7; /* Find if we're from user mode */ \ \ sethi %hi(EINTSTACK-BIAS), %g6; \ sethi %hi(EINTSTACK-INTSTACK), %g4; \ \ or %g6, %lo(EINTSTACK-BIAS), %g6; /* Base of interrupt stack */ \ dec %g4; /* Make it into a mask */ \ \ sub %g6, %sp, %g1; /* Offset from interrupt stack */ \ sethi %hi((stackspace)), %g5; \ \ or %g5, %lo((stackspace)), %g5; \ \ andn %g1, %g4, %g4; /* Are we out of the interrupt stack range? */ \ xor %g7, WSTATE_KERN, %g3; /* Are we on the user stack ? */ \ \ sra %g5, 0, %g5; /* Sign extend the damn thing */ \ orcc %g3, %g4, %g0; /* Definitely not off the interrupt stack */ \ \ sethi %hi(CPUINFO_VA + CI_EINTSTACK), %g4; \ bz,a,pt %xcc, 1f; \ mov %sp, %g6; \ \ ldx [%g4 + %lo(CPUINFO_VA + CI_EINTSTACK)], %g4; \ movrnz %g4, %g4, %g6; /* Use saved intr stack if exists */ \ \ 1: add %g6, %g5, %g5; /* Allocate a stack frame */ \ btst 1, %g6; \ bnz,pt %icc, 1f; \ \ mov %g5, %g6; \ \ add %g5, -BIAS, %g6; \ \ 1: SAVE_LOCALS_INS \ save %g6, 0, %sp; /* If we fault we should come right back here */ \ stx %i0, [%sp + CC64FSZ + BIAS + TF_O + (0*8)]; /* Save out registers to trap frame */ \ stx %i1, [%sp + CC64FSZ + BIAS + TF_O + (1*8)]; \ stx %i2, [%sp + CC64FSZ + BIAS + TF_O + (2*8)]; \ stx %i3, [%sp + CC64FSZ + BIAS + TF_O + (3*8)]; \ stx %i4, [%sp + CC64FSZ + BIAS + TF_O + (4*8)]; \ \ stx %i5, [%sp + CC64FSZ + BIAS + TF_O + (5*8)]; \ stx %i6, [%sp + CC64FSZ + BIAS + TF_O + (6*8)]; \ stx %i6, [%sp + CC64FSZ + BIAS + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ brz,pt %g3, 1f; /* If we were in kernel mode start saving globals */ \ stx %i7, [%sp + CC64FSZ + BIAS + TF_O + (7*8)]; \ /* came from user mode -- switch to kernel mode stack */ \ rdpr %otherwin, %g5; /* Has this already been done? */ \ \ brnz,pn %g5, 1f; /* Don't set this twice */ \ \ rdpr %canrestore, %g5; /* Fixup register window state registers */ \ \ wrpr %g0, 0, %canrestore; \ \ wrpr %g0, %g5, %otherwin; \ \ mov CTX_PRIMARY, %g7; \ \ wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ \ SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ \ sethi %hi(KERNBASE), %g5; \ flush %g5; /* Some convenient address that won't trap */ \ 1: #else /* _LP64 */ #define TRAP_SETUP(stackspace) \ sethi %hi(CPCB), %g6; \ sethi %hi((stackspace)), %g5; \ LDPTR [%g6 + %lo(CPCB)], %g6; \ sethi %hi(USPACE), %g7; \ or %g5, %lo((stackspace)), %g5; \ add %g6, %g7, %g6; \ rdpr %wstate, %g7; /* Find if we're from user mode */ \ \ sra %g5, 0, %g5; /* Sign extend the damn thing */ \ subcc %g7, WSTATE_KERN, %g7; /* Compare & leave in register */ \ movz %icc, %sp, %g6; /* Select old (kernel) stack or base of kernel stack */ \ FIXUP_TRAP_STACK \ SAVE_LOCALS_INS \ save %g6, 0, %sp; /* If we fault we should come right back here */ \ stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ \ stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ brz,pn %g7, 1f; /* If we were in kernel mode start saving globals */ \ stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ mov CTX_PRIMARY, %g7; \ /* came from user mode -- switch to kernel mode stack */ \ rdpr %canrestore, %g5; /* Fixup register window state registers */ \ wrpr %g0, 0, %canrestore; \ wrpr %g0, %g5, %otherwin; \ wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ \ SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ sethi %hi(KERNBASE), %g5; \ flush %g5; /* Some convenient address that won't trap */ \ 1: /* * Interrupt setup is almost exactly like trap setup, but we need to * go to the interrupt stack if (a) we came from user mode or (b) we * came from kernel mode on the kernel stack. * * We don't guarantee any registers are preserved during this operation. */ #define INTR_SETUP(stackspace) \ sethi %hi(EINTSTACK), %g1; \ sethi %hi((stackspace)), %g5; \ btst 1, %sp; \ add %sp, BIAS, %g6; \ movz %icc, %sp, %g6; \ or %g1, %lo(EINTSTACK), %g1; \ srl %g6, 0, %g6; /* truncate at 32-bits */ \ set (EINTSTACK-INTSTACK), %g7; \ or %g5, %lo((stackspace)), %g5; \ sub %g1, %g6, %g2; /* Determine if we need to switch to intr stack or not */ \ dec %g7; /* Make it into a mask */ \ sethi %hi(CPUINFO_VA + CI_EINTSTACK), %g3; \ andncc %g2, %g7, %g0; /* XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ LDPTR [%g3 + %lo(CPUINFO_VA + CI_EINTSTACK)], %g3; \ rdpr %wstate, %g7; /* Find if we're from user mode */ \ movrnz %g3, %g3, %g1; /* Use saved intr stack if exists */ \ sra %g5, 0, %g5; /* Sign extend the damn thing */ \ movnz %xcc, %g1, %g6; /* Stay on interrupt stack? */ \ cmp %g7, WSTATE_KERN; /* User or kernel sp? */ \ movnz %icc, %g1, %g6; /* Stay on interrupt stack? */ \ add %g6, %g5, %g6; /* Allocate a stack frame */ \ \ SAVE_LOCALS_INS \ save %g6, 0, %sp; /* If we fault we should come right back here */ \ stx %i0, [%sp + CC64FSZ + STKB + TF_O + (0*8)]; /* Save out registers to trap frame */ \ stx %i1, [%sp + CC64FSZ + STKB + TF_O + (1*8)]; \ stx %i2, [%sp + CC64FSZ + STKB + TF_O + (2*8)]; \ stx %i3, [%sp + CC64FSZ + STKB + TF_O + (3*8)]; \ stx %i4, [%sp + CC64FSZ + STKB + TF_O + (4*8)]; \ stx %i5, [%sp + CC64FSZ + STKB + TF_O + (5*8)]; \ stx %i6, [%sp + CC64FSZ + STKB + TF_O + (6*8)]; \ stx %i6, [%sp + CC64FSZ + STKB + TF_G + (0*8)]; /* Save fp in clockframe->cf_fp */ \ rdpr %wstate, %g7; /* Find if we're from user mode */ \ stx %i7, [%sp + CC64FSZ + STKB + TF_O + (7*8)]; \ cmp %g7, WSTATE_KERN; /* Compare & leave in register */ \ be,pn %icc, 1f; /* If we were in kernel mode start saving globals */ \ /* came from user mode -- switch to kernel mode stack */ \ rdpr %otherwin, %g5; /* Has this already been done? */ \ tst %g5; tnz %xcc, 1; nop; /* DEBUG -- this should _NEVER_ happen */ \ brnz,pn %g5, 1f; /* Don't set this twice */ \ rdpr %canrestore, %g5; /* Fixup register window state registers */ \ wrpr %g0, 0, %canrestore; \ mov CTX_PRIMARY, %g7; \ wrpr %g0, %g5, %otherwin; \ wrpr %g0, WSTATE_KERN, %wstate; /* Enable kernel mode window traps -- now we can trap again */ \ SET_MMU_CONTEXTID %g0, %g7, %g5; /* Switch MMU to kernel primary context */ \ sethi %hi(KERNBASE), %g5; \ flush %g5; /* Some convenient address that won't trap */ \ 1: #endif /* _LP64 */ #ifdef DEBUG /* Look up kpte to test algorithm */ .globl asmptechk asmptechk: mov %o0, %g4 ! pmap->pm_segs mov %o1, %g3 ! Addr to lookup -- mind the context srax %g3, HOLESHIFT, %g5 ! Check for valid address brz,pt %g5, 0f ! Should be zero or -1 inc %g5 ! Make -1 -> 0 brnz,pn %g5, 1f ! Error! 0: srlx %g3, STSHIFT, %g5 and %g5, STMASK, %g5 sll %g5, 3, %g5 add %g4, %g5, %g4 DLFLUSH(%g4,%g5) ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED DLFLUSH2(%g5) brz,pn %g4, 1f ! NULL entry? check somewhere else srlx %g3, PDSHIFT, %g5 and %g5, PDMASK, %g5 sll %g5, 3, %g5 add %g4, %g5, %g4 DLFLUSH(%g4,%g5) ldxa [%g4] ASI_PHYS_CACHED, %g4 ! Remember -- UNSIGNED DLFLUSH2(%g5) brz,pn %g4, 1f ! NULL entry? check somewhere else srlx %g3, PTSHIFT, %g5 ! Convert to ptab offset and %g5, PTMASK, %g5 sll %g5, 3, %g5 add %g4, %g5, %g4 DLFLUSH(%g4,%g5) ldxa [%g4] ASI_PHYS_CACHED, %g6 DLFLUSH2(%g5) brgez,pn %g6, 1f ! Entry invalid? Punt srlx %g6, 32, %o0 retl srl %g6, 0, %o1 1: mov %g0, %o1 retl mov %g0, %o0 .data 2: .asciz "asmptechk: %x %x %x %x:%x\n" _ALIGN .text #endif /* * This is the MMU protection handler. It's too big to fit * in the trap table so I moved it here. It's relatively simple. * It looks up the page mapping in the page table associated with * the trapping context. It checks to see if the S/W writable bit * is set. If so, it sets the H/W write bit, marks the tte modified, * and enters the mapping into the MMU. Otherwise it does a regular * data fault. */ ICACHE_ALIGN dmmu_write_fault: mov TLB_TAG_ACCESS, %g3 sethi %hi(0x1fff), %g6 ! 8K context mask ldxa [%g3] ASI_DMMU, %g3 ! Get fault addr from Tag Target sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 or %g6, %lo(0x1fff), %g6 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 srax %g3, HOLESHIFT, %g5 ! Check for valid address and %g3, %g6, %g6 ! Isolate context inc %g5 ! (0 or -1) -> (1 or 0) sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy ldx [%g4+%g6], %g4 ! Load up our page table. srlx %g3, STSHIFT, %g6 cmp %g5, 1 bgu,pn %xcc, winfix ! Error! srlx %g3, PDSHIFT, %g5 and %g6, STMASK, %g6 sll %g6, 3, %g6 and %g5, PDMASK, %g5 sll %g5, 3, %g5 add %g6, %g4, %g4 DLFLUSH(%g4,%g6) ldxa [%g4] ASI_PHYS_CACHED, %g4 DLFLUSH2(%g6) srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 add %g5, %g4, %g5 brz,pn %g4, winfix ! NULL entry? check somewhere else nop ldxa [%g5] ASI_PHYS_CACHED, %g4 sll %g6, 3, %g6 brz,pn %g4, winfix ! NULL entry? check somewhere else add %g6, %g4, %g6 1: ldxa [%g6] ASI_PHYS_CACHED, %g4 brgez,pn %g4, winfix ! Entry invalid? Punt or %g4, SUN4U_TTE_MODIFY|SUN4U_TTE_ACCESS|SUN4U_TTE_W, %g7 ! Update the modified bit btst SUN4U_TTE_REAL_W|SUN4U_TTE_W, %g4 ! Is it a ref fault? bz,pn %xcc, winfix ! No -- really fault #ifdef DEBUG /* Make sure we don't try to replace a kernel translation */ /* This should not be necessary */ sllx %g3, 64-13, %g2 ! Isolate context bits sethi %hi(KERNBASE), %g5 ! Don't need %lo brnz,pt %g2, 0f ! Ignore context != 0 set 0x0800000, %g2 ! 8MB sub %g3, %g5, %g5 cmp %g5, %g2 tlu %xcc, 1; nop blu,pn %xcc, winfix ! Next insn in delay slot is unimportant 0: #endif /* Need to check for and handle large pages. */ srlx %g4, 61, %g5 ! Isolate the size bits ldxa [%g0] ASI_DMMU_8KPTR, %g2 ! Load DMMU 8K TSB pointer andcc %g5, 0x3, %g5 ! 8K? bnz,pn %icc, winfix ! We punt to the pmap code since we can't handle policy ldxa [%g0] ASI_DMMU, %g1 ! Load DMMU tag target register casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out membar #StoreLoad cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4U_TTE_MODIFY|SUN4U_TTE_ACCESS|SUN4U_TTE_W, %g4 ! Update the modified bit stx %g1, [%g2] ! Update TSB entry tag mov SFSR, %g7 stx %g4, [%g2+8] ! Update TSB entry data nop #ifdef TRAPSTATS sethi %hi(_C_LABEL(protfix)), %g1 lduw [%g1+%lo(_C_LABEL(protfix))], %g2 inc %g2 stw %g2, [%g1+%lo(_C_LABEL(protfix))] #endif mov DEMAP_PAGE_SECONDARY, %g1 ! Secondary flush mov DEMAP_PAGE_NUCLEUS, %g5 ! Nucleus flush stxa %g0, [%g7] ASI_DMMU ! clear out the fault sllx %g3, (64-13), %g7 ! Need to demap old entry first andn %g3, 0xfff, %g6 movrz %g7, %g5, %g1 ! Pick one or %g6, %g1, %g6 membar #Sync stxa %g6, [%g6] ASI_DMMU_DEMAP ! Do the demap membar #Sync stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping membar #Sync retry /* * Each memory data access fault from a fast access miss handler comes here. * We will quickly check if this is an original prom mapping before going * to the generic fault handler * * We will assume that %pil is not lost so we won't bother to save it * unless we're in an interrupt handler. * * On entry: * We are on one of the alternate set of globals * %g1 = MMU tag target * %g2 = 8Kptr * %g3 = TLB TAG ACCESS * * On return: * */ ICACHE_ALIGN data_miss: #ifdef TRAPSTATS set _C_LABEL(kdmiss), %g3 set _C_LABEL(udmiss), %g4 rdpr %tl, %g6 dec %g6 movrz %g6, %g4, %g3 lduw [%g3], %g4 inc %g4 stw %g4, [%g3] #endif mov TLB_TAG_ACCESS, %g3 ! Get real fault page sethi %hi(0x1fff), %g6 ! 8K context mask ldxa [%g3] ASI_DMMU, %g3 ! from tag access register sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 or %g6, %lo(0x1fff), %g6 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 srax %g3, HOLESHIFT, %g5 ! Check for valid address and %g3, %g6, %g6 ! Isolate context inc %g5 ! (0 or -1) -> (1 or 0) sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy ldx [%g4+%g6], %g4 ! Load up our page table. #ifdef DEBUG /* Make sure we don't try to replace a kernel translation */ /* This should not be necessary */ brnz,pt %g6, 1f ! If user context continue miss sethi %hi(KERNBASE), %g7 ! Don't need %lo set 0x0800000, %g6 ! 8MB sub %g3, %g7, %g7 cmp %g7, %g6 tlu %xcc, 1; nop 1: #endif srlx %g3, STSHIFT, %g6 cmp %g5, 1 bgu,pn %xcc, winfix ! Error! srlx %g3, PDSHIFT, %g5 and %g6, STMASK, %g6 sll %g6, 3, %g6 and %g5, PDMASK, %g5 sll %g5, 3, %g5 add %g6, %g4, %g4 ldxa [%g4] ASI_PHYS_CACHED, %g4 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 add %g5, %g4, %g5 brz,pn %g4, data_nfo ! NULL entry? check somewhere else nop ldxa [%g5] ASI_PHYS_CACHED, %g4 sll %g6, 3, %g6 brz,pn %g4, data_nfo ! NULL entry? check somewhere else add %g6, %g4, %g6 1: ldxa [%g6] ASI_PHYS_CACHED, %g4 brgez,pn %g4, data_nfo ! Entry invalid? Punt or %g4, SUN4U_TTE_ACCESS, %g7 ! Update the access bit btst SUN4U_TTE_ACCESS, %g4 ! Need to update access git? bne,pt %xcc, 1f nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4U_TTE_ACCESS, %g4 ! Update the access bit 1: stx %g1, [%g2] ! Update TSB entry tag stx %g4, [%g2+8] ! Update TSB entry data stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping membar #Sync CLRTT retry NOTREACHED /* * We had a data miss but did not find a mapping. Insert * a NFO mapping to satisfy speculative loads and return. * If this had been a real load, it will re-execute and * result in a data fault or protection fault rather than * a TLB miss. We insert an 8K TTE with the valid and NFO * bits set. All others should zero. The TTE looks like this: * * 0x9000000000000000 * */ data_nfo: sethi %hi(0x90000000), %g4 ! V(0x8)|NFO(0x1) sllx %g4, 32, %g4 stxa %g4, [%g0] ASI_DMMU_DATA_IN ! Enter new mapping membar #Sync CLRTT retry /* * Handler for making the trap window shiny clean. * * If the store that trapped was to a kernel address, panic. * * If the store that trapped was to a user address, stick it in the PCB. * Since we don't want to force user code to use the standard register * convention if we don't have to, we will not assume that %fp points to * anything valid. * * On entry: * We are on one of the alternate set of globals * %g1 = %tl - 1, tstate[tl-1], scratch - local * %g2 = %tl - local * %g3 = MMU tag access - in * %g4 = %cwp - local * %g5 = scratch - local * %g6 = cpcb - local * %g7 = scratch - local * * On return: * * NB: remove most of this from main codepath & cleanup I$ */ winfault: mov TLB_TAG_ACCESS, %g3 ! Get real fault page from tag access register ldxa [%g3] ASI_DMMU, %g3 ! And put it into the non-MMU alternate regs winfix: rdpr %tl, %g2 subcc %g2, 1, %g1 ble,pt %icc, datafault ! Don't go below trap level 1 sethi %hi(CPCB), %g6 ! get current pcb wrpr %g1, 0, %tl ! Pop a trap level rdpr %tt, %g7 ! Read type of prev. trap rdpr %tstate, %g4 ! Try to restore prev %cwp if we were executing a restore andn %g7, 0x3f, %g5 ! window fill traps are all 0b 0000 11xx xxxx #if 1 cmp %g7, 0x30 ! If we took a datafault just before this trap bne,pt %icc, winfixfill ! our stack's probably bad so we need to switch somewhere else nop !! !! Double data fault -- bad stack? !! wrpr %g2, %tl ! Restore trap level. sir ! Just issue a reset and don't try to recover. mov %fp, %l6 ! Save the frame pointer set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack add %fp, -CC64FSZ, %sp ! Create a stackframe wrpr %g0, 15, %pil ! Disable interrupts, too wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB wrpr %g0, 7, %cansave ! probably is too, so blow away ba slowtrap ! all our register windows. wrpr %g0, 0x101, %tt #endif winfixfill: cmp %g5, 0x0c0 ! so we mask lower bits & compare to 0b 0000 1100 0000 bne,pt %icc, winfixspill ! Dump our trap frame -- we will retry the fill when the page is loaded cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx !! !! This was a fill !! #ifdef TRAPSTATS set _C_LABEL(wfill), %g1 lduw [%g1], %g5 inc %g5 stw %g5, [%g1] #endif btst TSTATE_PRIV, %g4 ! User mode? and %g4, CWP, %g5 ! %g4 = %cwp of trap wrpr %g7, 0, %tt bz,a,pt %icc, datafault ! We were in user mode -- normal fault wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consistent /* * We're in a pickle here. We were trying to return to user mode * and the restore of the user window failed, so now we have one valid * kernel window and a user window state. If we do a TRAP_SETUP() now, * our kernel window will be considered a user window and cause a * fault when we try to save it later due to an invalid user address. * If we return to where we faulted, our window state will not be valid * and we will fault trying to enter user with our primary context of zero. * * What we'll do is arrange to have us return to return_from_trap so we will * start the whole business over again. But first, switch to a kernel window * setup. Let's see, canrestore and otherwin are zero. Set WSTATE_KERN and * make sure we're in kernel context and we're done. */ #ifdef TRAPSTATS set _C_LABEL(kwfill), %g4 lduw [%g4], %g7 inc %g7 stw %g7, [%g4] #endif #if 0 /* Need to switch over to new stuff to fix WDR bug */ wrpr %g5, %cwp ! Restore cwp from before fill trap -- regs should now be consistent wrpr %g2, %g0, %tl ! Restore trap level -- we need to reuse it set return_from_trap, %g4 ! XXX - need to set %g1 to tstate set CTX_PRIMARY, %g7 wrpr %g4, 0, %tpc stxa %g0, [%g7] ASI_DMMU inc 4, %g4 membar #Sync flush %g4 ! Isn't this convenient? wrpr %g0, WSTATE_KERN, %wstate wrpr %g0, 0, %canrestore ! These should be zero but wrpr %g0, 0, %otherwin ! clear them just in case rdpr %ver, %g5 and %g5, CWP, %g5 wrpr %g0, 0, %cleanwin dec 1, %g5 ! NWINDOWS-1-1 wrpr %g5, 0, %cansave ! Invalidate all windows ! flushw ! DEBUG ba,pt %icc, datafault wrpr %g4, 0, %tnpc #else wrpr %g2, %g0, %tl ! Restore trap level cmp %g2, 3 tne %icc, 1 rdpr %tt, %g5 wrpr %g0, 1, %tl ! Revert to TL==1 XXX what if this wasn't in rft_user? Oh well. wrpr %g5, %g0, %tt ! Set trap type correctly /* * Here we need to implement the beginning of datafault. * TRAP_SETUP expects to come from either kernel mode or * user mode with at least one valid register window. It * will allocate a trap frame, save the out registers, and * fix the window registers to think we have one user * register window. * * However, under these circumstances we don't have any * valid register windows, so we need to clean up the window * registers to prevent garbage from being saved to either * the user stack or the PCB before calling the datafault * handler. * * We could simply jump to datafault if we could somehow * make the handler issue a `saved' instruction immediately * after creating the trapframe. * * The following is duplicated from datafault: */ #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals #else wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals #endif wr %g0, ASI_DMMU, %asi ! We need to re-load trap info ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first ldxa [SFSR] %asi, %g3 ! get sync fault status register stxa %g0, [SFSR] %asi ! Clear out fault now TRAP_SETUP(-CC64FSZ-TF_SIZE) saved ! Blow away that one register window we didn't ever use. ba,a,pt %icc, Ldatafault_internal ! Now we should return directly to user mode nop #endif winfixspill: bne,a,pt %xcc, datafault ! Was not a spill -- handle it normally wrpr %g2, 0, %tl ! Restore trap level for now XXXX !! !! This was a spill !! #if 1 btst TSTATE_PRIV, %g4 ! From user mode? wrpr %g2, 0, %tl ! We need to load the fault type so we can rdpr %tt, %g5 ! overwrite the lower trap and get it to the fault handler wrpr %g1, 0, %tl wrpr %g5, 0, %tt ! Copy over trap type for the fault handler and %g4, CWP, %g5 ! find %cwp from trap be,a,pt %xcc, datafault ! Let's do a regular datafault. When we try a save in datafault we'll wrpr %g5, 0, %cwp ! return here and write out all dirty windows. #endif wrpr %g2, 0, %tl ! Restore trap level for now XXXX LDPTR [%g6 + %lo(CPCB)], %g6 ! This is in the locked TLB and should not fault #ifdef TRAPSTATS set _C_LABEL(wspill), %g7 lduw [%g7], %g5 inc %g5 stw %g5, [%g7] #endif /* * Traverse kernel map to find paddr of cpcb and only us ASI_PHYS_CACHED to * prevent any faults while saving the windows. BTW if it isn't mapped, we * will trap and hopefully panic. */ ! ba 0f ! DEBUG -- don't use phys addresses wr %g0, ASI_NUCLEUS, %asi ! In case of problems finding PA sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g1 LDPTR [%g1 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g1 ! Load start of ctxbusy #ifdef DEBUG srax %g6, HOLESHIFT, %g7 ! Check for valid address brz,pt %g7, 1f ! Should be zero or -1 addcc %g7, 1, %g7 ! Make -1 -> 0 tnz %xcc, 1 ! Invalid address??? How did this happen? 1: #endif srlx %g6, STSHIFT, %g7 ldx [%g1], %g1 ! Load pointer to kernel_pmap and %g7, STMASK, %g7 sll %g7, 3, %g7 add %g7, %g1, %g1 DLFLUSH(%g1,%g7) ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory DLFLUSH2(%g7) srlx %g6, PDSHIFT, %g7 ! Do page directory and %g7, PDMASK, %g7 sll %g7, 3, %g7 brz,pn %g1, 0f add %g7, %g1, %g1 DLFLUSH(%g1,%g7) ldxa [%g1] ASI_PHYS_CACHED, %g1 DLFLUSH2(%g7) srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset and %g7, PTMASK, %g7 brz %g1, 0f sll %g7, 3, %g7 add %g1, %g7, %g7 DLFLUSH(%g7,%g1) ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not DLFLUSH2(%g1) brgez %g7, 0f srlx %g7, PGSHIFT, %g7 ! Isolate PA part sll %g6, 32-PGSHIFT, %g6 ! And offset sllx %g7, PGSHIFT+23, %g7 ! There are 23 bits to the left of the PA in the TTE srl %g6, 32-PGSHIFT, %g6 srax %g7, 23, %g7 or %g7, %g6, %g6 ! Then combine them to form PA wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults 0: /* * Now save all user windows to cpcb. */ #ifdef NOTDEF_DEBUG add %g6, PCB_NSAVED, %g7 DLFLUSH(%g7,%g5) lduba [%g6 + PCB_NSAVED] %asi, %g7 ! make sure that pcb_nsaved DLFLUSH2(%g5) brz,pt %g7, 1f ! is zero, else nop wrpr %g0, 4, %tl sir ! Force a watchdog 1: #endif rdpr %otherwin, %g7 brnz,pt %g7, 1f rdpr %canrestore, %g5 rdpr %cansave, %g1 add %g5, 1, %g7 ! add the %cwp window to the list to save ! movrnz %g1, %g5, %g7 ! If we're issuing a save ! mov %g5, %g7 ! DEBUG wrpr %g0, 0, %canrestore wrpr %g7, 0, %otherwin ! Still in user mode -- need to switch to kernel mode 1: mov %g7, %g1 add %g6, PCB_NSAVED, %g7 DLFLUSH(%g7,%g5) lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Start incrementing pcb_nsaved DLFLUSH2(%g5) #ifdef DEBUG wrpr %g0, 5, %tl #endif mov %g6, %g5 brz,pt %g7, winfixsave ! If it's in use, panic saved ! frob window registers /* PANIC */ ! sir ! Force a watchdog #ifdef DEBUG wrpr %g2, 0, %tl #endif mov %g7, %o2 rdpr %ver, %o1 sethi %hi(2f), %o0 and %o1, CWP, %o1 wrpr %g0, %o1, %cleanwin dec 1, %o1 wrpr %g0, %o1, %cansave ! kludge away any more window problems wrpr %g0, 0, %canrestore wrpr %g0, 0, %otherwin or %lo(2f), %o0, %o0 wrpr %g0, WSTATE_KERN, %wstate set PANICSTACK-CC64FSZ-STKB, %sp ta 1; nop ! This helps out traptrace. call _C_LABEL(panic) ! This needs to be fixed properly but we should panic here mov %g1, %o1 NOTREACHED .data 2: .asciz "winfault: double invalid window at %p, nsaved=%d" _ALIGN .text 3: saved save winfixsave: stxa %l0, [%g5 + PCB_RW + ( 0*8)] %asi ! Save the window in the pcb, we can schedule other stuff in here stxa %l1, [%g5 + PCB_RW + ( 1*8)] %asi stxa %l2, [%g5 + PCB_RW + ( 2*8)] %asi stxa %l3, [%g5 + PCB_RW + ( 3*8)] %asi stxa %l4, [%g5 + PCB_RW + ( 4*8)] %asi stxa %l5, [%g5 + PCB_RW + ( 5*8)] %asi stxa %l6, [%g5 + PCB_RW + ( 6*8)] %asi stxa %l7, [%g5 + PCB_RW + ( 7*8)] %asi stxa %i0, [%g5 + PCB_RW + ( 8*8)] %asi stxa %i1, [%g5 + PCB_RW + ( 9*8)] %asi stxa %i2, [%g5 + PCB_RW + (10*8)] %asi stxa %i3, [%g5 + PCB_RW + (11*8)] %asi stxa %i4, [%g5 + PCB_RW + (12*8)] %asi stxa %i5, [%g5 + PCB_RW + (13*8)] %asi stxa %i6, [%g5 + PCB_RW + (14*8)] %asi stxa %i7, [%g5 + PCB_RW + (15*8)] %asi ! rdpr %otherwin, %g1 ! Check to see if we's done dec %g1 wrpr %g0, 7, %cleanwin ! BUGBUG -- we should not hardcode this, but I have no spare globals inc 16*8, %g5 ! Move to next window inc %g7 ! inc pcb_nsaved brnz,pt %g1, 3b stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out /* fix up pcb fields */ stba %g7, [%g6 + PCB_NSAVED] %asi ! cpcb->pcb_nsaved = n #if 0 mov %g7, %g5 ! fixup window registers 5: dec %g5 brgz,a,pt %g5, 5b restore #ifdef NOT_DEBUG rdpr %wstate, %g5 ! DEBUG wrpr %g0, WSTATE_KERN, %wstate ! DEBUG wrpr %g0, 4, %tl rdpr %cansave, %g7 rdpr %canrestore, %g6 flushw ! DEBUG wrpr %g2, 0, %tl wrpr %g5, 0, %wstate ! DEBUG #endif #else /* * We just issued a bunch of saves, so %cansave is now 0, * probably (if we were doing a flushw then we may have * come in with only partially full register windows and * it may not be 0). * * %g7 contains the count of the windows we just finished * saving. * * What we need to do now is move some of the windows from * %canrestore to %cansave. What we should do is take * min(%canrestore, %g7) and move that over to %cansave. * * %g7 is the number of windows we flushed, so we should * use that as a base. Clear out %otherwin, set %cansave * to min(%g7, NWINDOWS - 2), set %cleanwin to %canrestore * + %cansave and the rest follows: * * %otherwin = 0 * %cansave = NWINDOWS - 2 - %canrestore */ wrpr %g0, 0, %otherwin rdpr %canrestore, %g1 sub %g1, %g7, %g1 ! Calculate %canrestore - %g7 movrlz %g1, %g0, %g1 ! Clamp at zero wrpr %g1, 0, %canrestore ! This is the new canrestore rdpr %ver, %g5 and %g5, CWP, %g5 ! NWINDOWS-1 dec %g5 ! NWINDOWS-2 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel sub %g5, %g1, %g5 ! NWINDOWS-2-%canrestore wrpr %g5, 0, %cansave #ifdef NOT_DEBUG rdpr %wstate, %g5 ! DEBUG wrpr %g0, WSTATE_KERN, %wstate ! DEBUG wrpr %g0, 4, %tl flushw ! DEBUG wrpr %g2, 0, %tl wrpr %g5, 0, %wstate ! DEBUG #endif #endif #ifdef NOTDEF_DEBUG set panicstack-CC64FSZ, %g1 save %g1, 0, %sp GLOBTOLOC rdpr %wstate, %l0 wrpr %g0, WSTATE_KERN, %wstate set 8f, %o0 mov %g7, %o1 call printf mov %g5, %o2 wrpr %l0, 0, %wstate LOCTOGLOB restore .data 8: .asciz "winfix: spill fixup\n" _ALIGN .text #endif ! rdpr %tl, %g2 ! DEBUG DEBUG -- did we trap somewhere? sub %g2, 1, %g1 rdpr %tt, %g2 wrpr %g1, 0, %tl ! We will not attempt to re-execute the spill, so dump our trap frame permanently wrpr %g2, 0, %tt ! Move trap type from fault frame here, overwriting spill /* Did we save a user or kernel window ? */ ! srax %g3, 48, %g5 ! User or kernel store? (TAG TARGET) sllx %g3, (64-13), %g5 ! User or kernel store? (TAG ACCESS) sethi %hi(dcache_size), %g7 ld [%g7 + %lo(dcache_size)], %g7 sethi %hi(dcache_line_size), %g6 ld [%g6 + %lo(dcache_line_size)], %g6 brnz,pt %g5, 1f ! User fault -- save windows to pcb sub %g7, %g6, %g7 and %g4, CWP, %g4 ! %g4 = %cwp of trap wrpr %g4, 0, %cwp ! Kernel fault -- restore %cwp and force and trap to debugger !! !! Here we managed to fault trying to access a kernel window !! This is a bug. Switch to the interrupt stack if we aren't !! there already and then trap into the debugger or panic. !! sethi %hi(EINTSTACK-BIAS), %g6 btst 1, %sp bnz,pt %icc, 0f mov %sp, %g1 add %sp, -BIAS, %g1 0: or %g6, %lo(EINTSTACK-BIAS), %g6 set (EINTSTACK-INTSTACK), %g7 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses sub %g6, %g1, %g2 ! Determine if we need to switch to intr stack or not dec %g7 ! Make it into a mask andncc %g2, %g7, %g0 ! XXXXXXXXXX This assumes kernel addresses are unique from user addresses */ \ movz %xcc, %g1, %g6 ! Stay on interrupt stack? add %g6, -CCFSZ, %g6 ! Allocate a stack frame mov %sp, %l6 ! XXXXX Save old stack pointer mov %g6, %sp ta 1; nop ! Enter debugger NOTREACHED 1: #if 1 /* Now we need to blast away the D$ to make sure we're in sync */ stxa %g0, [%g7] ASI_DCACHE_TAG brnz,pt %g7, 1b sub %g7, %g6, %g7 #endif #ifdef NOTDEF_DEBUG set panicstack-CC64FSZ, %g5 save %g5, 0, %sp GLOBTOLOC rdpr %wstate, %l0 wrpr %g0, WSTATE_KERN, %wstate set 8f, %o0 call printf mov %fp, %o1 wrpr %l0, 0, %wstate LOCTOGLOB restore .data 8: .asciz "winfix: kernel spill retry\n" _ALIGN .text #endif #ifdef TRAPSTATS set _C_LABEL(wspillskip), %g4 lduw [%g4], %g5 inc %g5 stw %g5, [%g4] #endif /* * If we had WSTATE_KERN then we had at least one valid kernel window. * We should re-execute the trapping save. */ rdpr %wstate, %g3 mov %g3, %g3 cmp %g3, WSTATE_KERN bne,pt %icc, 1f nop retry ! Now we can complete the save 1: /* * Since we had a WSTATE_USER, we had no valid kernel windows. This should * only happen inside TRAP_SETUP or INTR_SETUP. Emulate * the instruction, clean up the register windows, then done. */ rdpr %cwp, %g1 inc %g1 rdpr %tstate, %g2 wrpr %g1, %cwp andn %g2, CWP, %g2 wrpr %g1, %g2, %tstate #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG #else wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate #endif mov %g6, %sp done /* * Each memory data access fault, from user or kernel mode, * comes here. * * We will assume that %pil is not lost so we won't bother to save it * unless we're in an interrupt handler. * * On entry: * We are on one of the alternate set of globals * %g1 = MMU tag target * %g2 = %tl * * On return: * */ datafault: #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals #else wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals #endif wr %g0, ASI_DMMU, %asi ! We need to re-load trap info ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register ldxa [SFAR] %asi, %g2 ! sync virt addr; must be read first ldxa [SFSR] %asi, %g3 ! get sync fault status register stxa %g0, [SFSR] %asi ! Clear out fault now TRAP_SETUP(-CC64FSZ-TF_SIZE) Ldatafault_internal: INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) ! ldx [%sp + CC64FSZ + STKB + TF_FAULT], %g1 ! DEBUG make sure this has not changed mov %g1, %o0 ! Move these to the out regs so we can save the globals mov %g2, %o4 mov %g3, %o5 ldxa [%g0] ASI_AFAR, %o2 ! get async fault address ldxa [%g0] ASI_AFSR, %o3 ! get async fault status mov -1, %g7 stxa %g7, [%g0] ASI_AFSR ! And clear this out, too wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 rdpr %tt, %o1 ! find out what trap brought us here stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 rdpr %tstate, %g1 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) rdpr %tpc, %g2 stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 rdpr %tnpc, %g3 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 mov %g2, %o7 ! Make the fault address look like the return address stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 rd %y, %g5 ! save y stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 sth %o1, [%sp + CC64FSZ + STKB + TF_TT] stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc stx %g2, [%sp + CC64FSZ + STKB + TF_PC] ! set tf.tf_npc stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] rdpr %pil, %g4 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] #if 1 rdpr %tl, %g7 dec %g7 movrlz %g7, %g0, %g7 wrpr %g0, %g7, %tl ! Revert to kernel mode #else wrpr %g0, 0, %tl ! Revert to kernel mode #endif /* Finish stackframe, call C trap handler */ flushw ! Get this clean so we won't take any more user faults #ifdef NOTDEF_DEBUG set CPCB, %o7 LDPTR [%o7], %o7 ldub [%o7 + PCB_NSAVED], %o7 brz,pt %o7, 2f nop save %sp, -CC64FSZ, %sp set 1f, %o0 call printf mov %i7, %o1 ta 1; nop restore .data 1: .asciz "datafault: nsaved = %d\n" _ALIGN .text 2: #endif !! In the EMBEDANY memory model %g4 points to the start of the data segment. !! In our case we need to clear it before calling any C-code clr %g4 /* * Right now the registers have the following values: * * %o0 -- MMU_TAG_ACCESS * %o1 -- TT * %o2 -- afar * %o3 -- afsr * %o4 -- sfar * %o5 -- sfsr */ cmp %o1, T_DATA_ERROR st %g5, [%sp + CC64FSZ + STKB + TF_Y] wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI be,pn %icc, data_error wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts mov %o0, %o3 ! (argument: trap address) mov %g2, %o2 ! (argument: trap pc) call _C_LABEL(data_access_fault) ! data_access_fault(&tf, type, ! pc, addr, sfva, sfsr) add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) data_recover: #ifdef TRAPSTATS set _C_LABEL(uintrcnt), %g1 stw %g0, [%g1] set _C_LABEL(iveccnt), %g1 stw %g0, [%g1] #endif wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts b return_from_trap ! go return ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap NOTREACHED data_error: call _C_LABEL(data_access_error) ! data_access_error(&tf, type, ! afva, afsr, sfva, sfsr) add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) ba data_recover nop NOTREACHED /* * Each memory instruction access fault from a fast access handler comes here. * We will quickly check if this is an original prom mapping before going * to the generic fault handler * * We will assume that %pil is not lost so we won't bother to save it * unless we're in an interrupt handler. * * On entry: * We are on one of the alternate set of globals * %g1 = MMU tag target * %g2 = TSB entry ptr * %g3 = TLB Tag Access * * On return: * */ ICACHE_ALIGN instr_miss: #ifdef TRAPSTATS set _C_LABEL(ktmiss), %g3 set _C_LABEL(utmiss), %g4 rdpr %tl, %g6 dec %g6 movrz %g6, %g4, %g3 lduw [%g3], %g4 inc %g4 stw %g4, [%g3] #endif mov TLB_TAG_ACCESS, %g3 ! Get real fault page sethi %hi(0x1fff), %g7 ! 8K context mask ldxa [%g3] ASI_IMMU, %g3 ! from tag access register sethi %hi(CPUINFO_VA+CI_CTXBUSY), %g4 or %g7, %lo(0x1fff), %g7 LDPTR [%g4 + %lo(CPUINFO_VA+CI_CTXBUSY)], %g4 srax %g3, HOLESHIFT, %g5 ! Check for valid address and %g3, %g7, %g6 ! Isolate context sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy inc %g5 ! (0 or -1) -> (1 or 0) ldx [%g4+%g6], %g4 ! Load up our page table. #ifdef DEBUG /* Make sure we don't try to replace a kernel translation */ /* This should not be necessary */ brnz,pt %g6, 1f ! If user context continue miss sethi %hi(KERNBASE), %g7 ! Don't need %lo set 0x0800000, %g6 ! 8MB sub %g3, %g7, %g7 cmp %g7, %g6 tlu %xcc, 1; nop 1: #endif srlx %g3, STSHIFT, %g6 cmp %g5, 1 bgu,pn %xcc, textfault ! Error! srlx %g3, PDSHIFT, %g5 and %g6, STMASK, %g6 sll %g6, 3, %g6 and %g5, PDMASK, %g5 nop sll %g5, 3, %g5 add %g6, %g4, %g4 ldxa [%g4] ASI_PHYS_CACHED, %g4 srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 add %g5, %g4, %g5 brz,pn %g4, textfault ! NULL entry? check somewhere else nop ldxa [%g5] ASI_PHYS_CACHED, %g4 sll %g6, 3, %g6 brz,pn %g4, textfault ! NULL entry? check somewhere else add %g6, %g4, %g6 1: ldxa [%g6] ASI_PHYS_CACHED, %g4 brgez,pn %g4, textfault nop /* Check if it's an executable mapping. */ andcc %g4, SUN4U_TTE_EXEC, %g0 bz,pn %xcc, textfault nop or %g4, SUN4U_TTE_ACCESS, %g7 ! Update accessed bit btst SUN4U_TTE_ACCESS, %g4 ! Need to update access git? bne,pt %xcc, 1f nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and store it cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4U_TTE_ACCESS, %g4 ! Update accessed bit 1: stx %g1, [%g2] ! Update TSB entry tag stx %g4, [%g2+8] ! Update TSB entry data stxa %g4, [%g0] ASI_IMMU_DATA_IN ! Enter new mapping membar #Sync CLRTT retry NOTREACHED !! !! Check our prom mappings -- temporary !! /* * Each memory text access fault, from user or kernel mode, * comes here. * * We will assume that %pil is not lost so we won't bother to save it * unless we're in an interrupt handler. * * On entry: * We are on one of the alternate set of globals * %g1 = MMU tag target * %g2 = %tl * %g3 = %tl - 1 * * On return: * */ textfault: #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! We need to save volatile stuff to interrupt globals #else wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! We need to save volatile stuff to alternate globals #endif wr %g0, ASI_IMMU, %asi ldxa [%g0 + TLB_TAG_ACCESS] %asi, %g1 ! Get fault address from tag access register ldxa [SFSR] %asi, %g3 ! get sync fault status register membar #LoadStore stxa %g0, [SFSR] %asi ! Clear out old info TRAP_SETUP(-CC64FSZ-TF_SIZE) INCR64(CPUINFO_VA+CI_NFAULT) ! cnt.v_faults++ (clobbers %o0,%o1) mov %g3, %o3 wrpr %g0, PSTATE_KERN, %pstate ! Switch to normal globals ldxa [%g0] ASI_AFSR, %o4 ! get async fault status ldxa [%g0] ASI_AFAR, %o5 ! get async fault address mov -1, %o0 stxa %o0, [%g0] ASI_AFSR ! Clear this out stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] ! save g1 stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] ! save g2 stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] ! (sneak g3 in here) rdpr %tt, %o1 ! Find out what caused this trap stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] ! sneak in g4 rdpr %tstate, %g1 stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] ! sneak in g5 rdpr %tpc, %o2 ! sync virt addr; must be read first stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] ! sneak in g6 rdpr %tnpc, %g3 stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] ! sneak in g7 rd %y, %g5 ! save y /* Finish stackframe, call C trap handler */ stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set tf.tf_psr, tf.tf_pc sth %o1, [%sp + CC64FSZ + STKB + TF_TT] ! debug stx %o2, [%sp + CC64FSZ + STKB + TF_PC] stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] ! set tf.tf_npc rdpr %pil, %g4 stb %g4, [%sp + CC64FSZ + STKB + TF_PIL] stb %g4, [%sp + CC64FSZ + STKB + TF_OLDPIL] rdpr %tl, %g7 dec %g7 movrlz %g7, %g0, %g7 wrpr %g0, %g7, %tl ! Revert to kernel mode wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI flushw ! Get rid of any user windows so we don't deadlock !! In the EMBEDANY memory model %g4 points to the start of the data segment. !! In our case we need to clear it before calling any C-code clr %g4 /* Use trap type to see what handler to call */ cmp %o1, T_INST_ERROR be,pn %xcc, text_error st %g5, [%sp + CC64FSZ + STKB + TF_Y] ! set tf.tf_y wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts call _C_LABEL(text_access_fault) ! mem_access_fault(&tf, type, pc, sfsr) add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) text_recover: wrpr %g0, PSTATE_KERN, %pstate ! disable interrupts b return_from_trap ! go return ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap NOTREACHED text_error: wrpr %g0, PSTATE_INTR, %pstate ! reenable interrupts call _C_LABEL(text_access_error) ! mem_access_fault(&tfm type, sfva [pc], sfsr, ! afva, afsr); add %sp, CC64FSZ + STKB, %o0 ! (argument: &tf) ba text_recover nop NOTREACHED #ifdef SUN4V /* * Traps for sun4v. */ sun4v_dtsb_miss: GET_MMFSA %g1 ! MMU Fault status area add %g1, 0x48, %g3 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context GET_CTXBUSY %g4 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy LDPTR [%g4 + %g6], %g4 ! Load up our page table. srax %g3, HOLESHIFT, %g5 ! Check for valid address brz,pt %g5, 0f ! Should be zero or -1 inc %g5 ! Make -1 -> 0 brnz,pn %g5, sun4v_datatrap ! Error! In hole! 0: srlx %g3, STSHIFT, %g6 and %g6, STMASK, %g6 ! Index into pm_segs sll %g6, 3, %g6 add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer srlx %g3, PDSHIFT, %g6 and %g6, PDMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else add %g4, %g6, %g6 1: LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE brgez,pn %g4, sun4v_datatrap ! Entry invalid? Punt or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? bne,pt %xcc, 2f nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the access bit 2: GET_TSB_DMMU %g2 /* Construct TSB tag word. */ add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context mov %g3, %g1 ! Data fault address srlx %g1, 22, %g1 ! 63..22 of virt addr sllx %g6, 48, %g6 ! context_id in 63..48 or %g1, %g6, %g1 ! construct TTE tag srlx %g3, PTSHIFT, %g3 sethi %hi(_C_LABEL(tsbsize)), %g5 mov 512, %g6 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS sub %g5, 1, %g5 ! TSBENTS -> offset and %g3, %g5, %g3 ! mask out TTE index sllx %g3, 4, %g3 ! TTE size is 16 bytes add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu membar #StoreStore STPTR %g4, [%g2 + 8] ! store TTE data STPTR %g1, [%g2] ! store TTE tag retry NOTREACHED sun4v_tl1_dtsb_miss: GET_MMFSA %g1 ! MMU Fault status area add %g1, 0x48, %g3 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context GET_CTXBUSY %g4 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy LDPTR [%g4 + %g6], %g4 ! Load up our page table. srax %g3, HOLESHIFT, %g5 ! Check for valid address brz,pt %g5, 0f ! Should be zero or -1 inc %g5 ! Make -1 -> 0 brnz,pn %g5, sun4v_tl1_ptbl_miss ! Error! In hole! 0: srlx %g3, STSHIFT, %g6 and %g6, STMASK, %g6 ! Index into pm_segs sll %g6, 3, %g6 add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer srlx %g3, PDSHIFT, %g6 and %g6, PDMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else add %g4, %g6, %g6 1: LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE brgez,pn %g4, sun4v_tl1_ptbl_miss ! Entry invalid? Punt or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? bne,pt %xcc, 2f nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the access bit 2: GET_TSB_DMMU %g2 /* Construct TSB tag word. */ add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context mov %g3, %g1 ! Data fault address srlx %g1, 22, %g1 ! 63..22 of virt addr sllx %g6, 48, %g6 ! context_id in 63..48 or %g1, %g6, %g1 ! construct TTE tag srlx %g3, PTSHIFT, %g3 sethi %hi(_C_LABEL(tsbsize)), %g5 mov 512, %g6 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS sub %g5, 1, %g5 ! TSBENTS -> offset and %g3, %g5, %g3 ! mask out TTE index sllx %g3, 4, %g3 ! TTE size is 16 bytes add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu membar #StoreStore STPTR %g4, [%g2 + 8] ! store TTE data STPTR %g1, [%g2] ! store TTE tag retry NOTREACHED sun4v_datatrap: GET_MMFSA %g3 ! MMU Fault status area add %g3, 0x48, %g1 LDPTRA [%g1] ASI_PHYS_CACHED, %g1 ! Data fault address add %g3, 0x50, %g2 LDPTRA [%g2] ASI_PHYS_CACHED, %g2 ! Data fault context TRAP_SETUP(-CC64FSZ-TF_SIZE) or %g1, %g2, %o3 mov %g1, %o4 rdpr %tt, %g4 rdpr %tstate, %g1 rdpr %tpc, %g2 rdpr %tnpc, %g3 stx %g1, [%sp + CC64FSZ + BIAS + TF_TSTATE] mov %g4, %o1 ! (type) stx %g2, [%sp + CC64FSZ + BIAS + TF_PC] rd %y, %g5 stx %g3, [%sp + CC64FSZ + BIAS + TF_NPC] st %g5, [%sp + CC64FSZ + BIAS + TF_Y] mov %g2, %o2 ! (pc) sth %o1, [%sp + CC64FSZ + BIAS + TF_TT]! debug cmp %o1, T_FDMMU_PROT bne,pn %icc, 1f mov SFSR_FV, %o5 or %o5, SFSR_W, %o5 1: NORMAL_GLOBALS_SUN4V stx %g1, [%sp + CC64FSZ + BIAS + TF_G + (1*8)] stx %g2, [%sp + CC64FSZ + BIAS + TF_G + (2*8)] add %sp, CC64FSZ + BIAS, %o0 ! (&tf) stx %g3, [%sp + CC64FSZ + BIAS + TF_G + (3*8)] stx %g4, [%sp + CC64FSZ + BIAS + TF_G + (4*8)] stx %g5, [%sp + CC64FSZ + BIAS + TF_G + (5*8)] rdpr %pil, %g5 stx %g6, [%sp + CC64FSZ + BIAS + TF_G + (6*8)] stx %g7, [%sp + CC64FSZ + BIAS + TF_G + (7*8)] stb %g5, [%sp + CC64FSZ + BIAS + TF_PIL] stb %g5, [%sp + CC64FSZ + BIAS + TF_OLDPIL] /* * Phew, ready to enable traps and call C code. */ wrpr %g0, 0, %tl wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI wrpr %g0, PSTATE_INTR, %pstate ! traps on again call _C_LABEL(data_access_fault) ! data_acces_fault(tf, type, ...) nop ba,a,pt %icc, return_from_trap nop NOTREACHED sun4v_tl0_dtsb_prot: GET_MMFSA %g1 ! MMU Fault status area add %g1, 0x48, %g3 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context GET_CTXBUSY %g4 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy LDPTR [%g4 + %g6], %g4 ! Load up our page table. srax %g3, HOLESHIFT, %g5 ! Check for valid address brz,pt %g5, 0f ! Should be zero or -1 inc %g5 ! Make -1 -> 0 brnz,pn %g5, sun4v_datatrap ! Error! In hole! 0: srlx %g3, STSHIFT, %g6 and %g6, STMASK, %g6 ! Index into pm_segs sll %g6, 3, %g6 add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer srlx %g3, PDSHIFT, %g6 and %g6, PDMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_datatrap ! NULL entry? check somewhere else add %g4, %g6, %g6 1: LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE brgez,pn %g4, sun4v_datatrap ! Entry invalid? Punt or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g7 ! Update the modified bit # btst SUN4V_TLB_REAL_W|SUN4V_TLB_W, %g4 ! Is it a ref fault? mov 1, %g2 sllx %g2, 61, %g2 ! %g2 is now SUN4V_TLB_REAL_W or %g2, SUN4V_TLB_W, %g2 btst %g2, %g4 bz,pn %xcc, sun4v_datatrap ! No -- really fault nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g4 ! Update the modified bit 2: GET_TSB_DMMU %g2 mov %g1, %g7 ! save MMFSA /* Construct TSB tag word. */ add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context mov %g3, %g1 ! Data fault address srlx %g1, 22, %g1 ! 63..22 of virt addr sllx %g6, 48, %g6 ! context_id in 63..48 or %g1, %g6, %g1 ! construct TTE tag srlx %g3, PTSHIFT, %g3 sethi %hi(_C_LABEL(tsbsize)), %g5 mov 512, %g6 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS sub %g5, 1, %g5 ! TSBENTS -> offset and %g3, %g5, %g3 ! mask out TTE index sllx %g3, 4, %g3 ! TTE size is 16 bytes add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu membar #StoreStore STPTR %g4, [%g2 + 8] ! store TTE data STPTR %g1, [%g2] ! store TTE tag mov %o0, %g1 mov %o1, %g2 mov %o2, %g3 add %g7, 0x48, %o0 ldxa [%o0] ASI_PHYS_CACHED, %o0 ! Data fault address add %g7, 0x50, %o1 ldxa [%o1] ASI_PHYS_CACHED, %o1 ! Data fault context mov MAP_DTLB, %o2 ta ST_MMU_UNMAP_ADDR mov %g1, %o0 mov %g2, %o1 mov %g3, %o2 retry NOTREACHED sun4v_tl0_itsb_miss: GET_MMFSA %g1 ! MMU Fault status area add %g1, 0x8, %g3 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Instruction fault address add %g1, 0x10, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context GET_CTXBUSY %g4 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy LDPTR [%g4 + %g6], %g4 ! Load up our page table. srax %g3, HOLESHIFT, %g5 ! Check for valid address brz,pt %g5, 0f ! Should be zero or -1 inc %g5 ! Make -1 -> 0 brnz,pn %g5, sun4v_texttrap ! Error! In hole! 0: srlx %g3, STSHIFT, %g6 and %g6, STMASK, %g6 ! Index into pm_segs sll %g6, 3, %g6 add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer srlx %g3, PDSHIFT, %g6 and %g6, PDMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_texttrap ! NULL entry? check somewhere else add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_texttrap ! NULL entry? check somewhere else add %g4, %g6, %g6 1: LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE brgez,pn %g4, sun4v_texttrap ! Entry invalid? Punt or %g4, SUN4V_TLB_ACCESS, %g7 ! Update the access bit btst SUN4V_TLB_EXEC, %g4 ! Need to update exec bit? bz,pn %xcc, sun4v_texttrap nop btst SUN4V_TLB_ACCESS, %g4 ! Need to update access bit? bne,pt %xcc, 2f nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4V_TLB_ACCESS, %g4 ! Update the modified bit 2: GET_TSB_DMMU %g2 mov %g1, %g7 /* Construct TSB tag word. */ add %g1, 0x10, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Instruction fault context mov %g3, %g1 ! Instruction fault address srlx %g1, 22, %g1 ! 63..22 of virt addr sllx %g6, 48, %g6 ! context_id in 63..48 or %g1, %g6, %g1 ! construct TTE tag srlx %g3, PTSHIFT, %g3 sethi %hi(_C_LABEL(tsbsize)), %g5 mov 512, %g6 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS sub %g5, 1, %g5 ! TSBENTS -> offset and %g3, %g5, %g3 ! mask out TTE index sllx %g3, 4, %g3 ! TTE size is 16 bytes add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu (FIXME ci_tsb_immu?) membar #StoreStore STPTR %g4, [%g2 + 8] ! store TTE data stx %g1, [%g2] ! store TTE tag retry NOTREACHED sun4v_texttrap: GET_MMFSA %g3 ! MMU Fault status area add %g3, 0x08, %g1 LDPTRA [%g1] ASI_PHYS_CACHED, %g1 ! Instruction fault address add %g3, 0x10, %g2 LDPTRA [%g2] ASI_PHYS_CACHED, %g2 ! Instruction fault context TRAP_SETUP(-CC64FSZ-TF_SIZE) or %g1, %g2, %o2 clr %o3 rdpr %tt, %g4 rdpr %tstate, %g1 rdpr %tpc, %g2 rdpr %tnpc, %g3 stx %g1, [%sp + CC64FSZ + BIAS + TF_TSTATE] mov %g4, %o1 ! (type) stx %g2, [%sp + CC64FSZ + BIAS + TF_PC] rd %y, %g5 stx %g3, [%sp + CC64FSZ + BIAS + TF_NPC] st %g5, [%sp + CC64FSZ + BIAS + TF_Y] sth %o1, [%sp + CC64FSZ + BIAS + TF_TT]! debug ! Get back to normal globals wrpr %g0, PSTATE_KERN, %pstate NORMAL_GLOBALS_SUN4V stx %g1, [%sp + CC64FSZ + BIAS + TF_G + (1*8)] stx %g2, [%sp + CC64FSZ + BIAS + TF_G + (2*8)] add %sp, CC64FSZ + BIAS, %o0 ! (&tf) stx %g3, [%sp + CC64FSZ + BIAS + TF_G + (3*8)] stx %g4, [%sp + CC64FSZ + BIAS + TF_G + (4*8)] stx %g5, [%sp + CC64FSZ + BIAS + TF_G + (5*8)] rdpr %pil, %g5 stx %g6, [%sp + CC64FSZ + BIAS + TF_G + (6*8)] stx %g7, [%sp + CC64FSZ + BIAS + TF_G + (7*8)] stb %g5, [%sp + CC64FSZ + BIAS + TF_PIL] stb %g5, [%sp + CC64FSZ + BIAS + TF_OLDPIL] /* * Phew, ready to enable traps and call C code. */ wrpr %g0, 0, %tl wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI wrpr %g0, PSTATE_INTR, %pstate ! traps on again call _C_LABEL(text_access_fault) ! text_access_fault(tf, type, ...) nop ba,a,pt %icc, return_from_trap nop NOTREACHED sun4v_tl1_dtsb_prot: GET_MMFSA %g1 ! MMU Fault status area add %g1, 0x48, %g3 LDPTRA [%g3] ASI_PHYS_CACHED, %g3 ! Data fault address add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context GET_CTXBUSY %g4 sllx %g6, 3, %g6 ! Make it into an offset into ctxbusy LDPTR [%g4 + %g6], %g4 ! Load up our page table. srax %g3, HOLESHIFT, %g5 ! Check for valid address brz,pt %g5, 0f ! Should be zero or -1 inc %g5 ! Make -1 -> 0 brnz,pn %g5, sun4v_tl1_ptbl_miss ! Error! In hole! 0: srlx %g3, STSHIFT, %g6 and %g6, STMASK, %g6 ! Index into pm_segs sll %g6, 3, %g6 add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page directory pointer srlx %g3, PDSHIFT, %g6 and %g6, PDMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else add %g4, %g6, %g4 LDPTRA [%g4] ASI_PHYS_CACHED, %g4 ! Load page table pointer srlx %g3, PTSHIFT, %g6 ! Convert to ptab offset and %g6, PTMASK, %g6 sll %g6, 3, %g6 brz,pn %g4, sun4v_tl1_ptbl_miss ! NULL entry? check somewhere else add %g4, %g6, %g6 1: LDPTRA [%g6] ASI_PHYS_CACHED, %g4 ! Fetch TTE brgez,pn %g4, sun4v_tl1_ptbl_miss ! Entry invalid? Punt or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g7 ! Update the modified bit # btst SUN4V_TLB_REAL_W|SUN4V_TLB_W, %g4 ! Is it a ref fault? mov 1, %g2 sllx %g2, 61, %g2 ! %g2 is now SUN4V_TLB_REAL_W or %g2, SUN4V_TLB_W, %g2 btst %g2, %g4 bz,pn %xcc, sun4v_tl1_ptbl_miss ! No -- really fault nop casxa [%g6] ASI_PHYS_CACHED, %g4, %g7 ! and write it out cmp %g4, %g7 bne,pn %xcc, 1b or %g4, SUN4V_TLB_MODIFY|SUN4V_TLB_ACCESS|SUN4V_TLB_W, %g4 ! Update the modified bit 2: GET_TSB_DMMU %g2 mov %g1, %g7 ! save MMFSA /* Construct TSB tag word. */ add %g1, 0x50, %g6 LDPTRA [%g6] ASI_PHYS_CACHED, %g6 ! Data fault context mov %g3, %g1 ! Data fault address srlx %g1, 22, %g1 ! 63..22 of virt addr sllx %g6, 48, %g6 ! context_id in 63..48 or %g1, %g6, %g1 ! construct TTE tag srlx %g3, PTSHIFT, %g3 sethi %hi(_C_LABEL(tsbsize)), %g5 mov 512, %g6 ld [%g5 + %lo(_C_LABEL(tsbsize))], %g5 sllx %g6, %g5, %g5 ! %g5 = 512 << tsbsize = TSBENTS sub %g5, 1, %g5 ! TSBENTS -> offset and %g3, %g5, %g3 ! mask out TTE index sllx %g3, 4, %g3 ! TTE size is 16 bytes add %g2, %g3, %g2 ! location of TTE in ci_tsb_dmmu membar #StoreStore STPTR %g4, [%g2 + 8] ! store TTE data STPTR %g1, [%g2] ! store TTE tag mov %o0, %g1 mov %o1, %g2 mov %o2, %g3 add %g7, 0x48, %o0 ldxa [%o0] ASI_PHYS_CACHED, %o0 ! Data fault address add %g7, 0x50, %o1 ldxa [%o1] ASI_PHYS_CACHED, %o1 ! Data fault context mov MAP_DTLB, %o2 ta ST_MMU_UNMAP_ADDR mov %g1, %o0 mov %g2, %o1 mov %g3, %o2 retry NOTREACHED sun4v_tl1_ptbl_miss: rdpr %tpc, %g1 set rft_user_fault_start, %g2 cmp %g1, %g2 blu,pt %xcc, 1f set rft_user_fault_end, %g2 cmp %g1, %g2 bgeu,pt %xcc, 1f nop /* We had a miss inside rtf_user_fault_start/rtf_user_fault_end block (FILL) /* Fixup %cwp. */ rdpr %cwp, %g1 inc %g1 wrpr %g1, %cwp rdpr %tt, %g1 wrpr 1, %tl wrpr %g1, %tt rdpr %cwp, %g1 set TSTATE_KERN, %g2 wrpr %g1, %g2, %tstate set return_from_trap, %g1 wrpr %g1, %tpc add %g1, 4, %g1 wrpr %g1, %tnpc wrpr %g0, 1, %gl ba,pt %xcc, sun4v_datatrap wrpr WSTATE_KERN, %wstate 1: rdpr %tstate, %g3 rdpr %tt, %g4 rdpr %tl, %g1 dec %g1 wrpr %g1, %tl rdpr %tt, %g2 inc %g1 wrpr %g1, %tl wrpr %g0, %g3, %tstate wrpr %g0, %g4, %tt andn %g2, 0x00f, %g3 cmp %g3, 0x080 be,pn %icc, flush_normals nop cmp %g3, 0x0a0 be,pn %icc, flush_others nop cmp %g3, 0x0c0 be,pn %icc, ufill_trap nop Debugger() NOTREACHED flush_others: set pcbspill_others, %g1 wrpr %g1, %tnpc done NOTREACHED flush_normals: ufill_trap: /* * Rearrange our trap state such that it appears as if we got * this trap directly from user mode. Then process it at TL = 1. * We'll take the spill/fill trap again once we return to user mode. */ rdpr %tt, %g1 rdpr %tstate, %g3 wrpr %g0, 1, %tl wrpr %g0, %g1, %tt rdpr %tstate, %g2 wrpr %g0, 2, %tl and %g2, TSTATE_CWP, %g2 andn %g3, TSTATE_CWP, %g3 wrpr %g2, %g3, %tstate set sun4v_datatrap, %g4 wrpr %g0, %g4, %tnpc done /* * Spill user windows into the PCB. */ pcbspill_normals: ba,pt %xcc, pcbspill wrpr 0x80, %tt pcbspill_others: wrpr 0xa0, %tt pcbspill: set CPUINFO_VA, %g6 ldx [%g6 + CI_CPCB], %g6 GET_CTXBUSY %g1 ldx [%g1], %g1 ! kernel pmap is ctx 0 srlx %g6, STSHIFT, %g7 and %g7, STMASK, %g7 sll %g7, 3, %g7 ! byte offset into ctxbusy add %g7, %g1, %g1 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory srlx %g6, PDSHIFT, %g7 ! Do page directory and %g7, PDMASK, %g7 sll %g7, 3, %g7 brz,pn %g1, pcbspill_fail add %g7, %g1, %g1 ldxa [%g1] ASI_PHYS_CACHED, %g1 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset and %g7, PTMASK, %g7 brz %g1, pcbspill_fail sll %g7, 3, %g7 add %g1, %g7, %g7 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not brgez %g7, pcbspill_fail srlx %g7, PGSHIFT, %g7 ! Isolate PA part sll %g6, 32-PGSHIFT, %g6 ! And offset sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE srl %g6, 32-PGSHIFT, %g6 srax %g7, 8, %g7 or %g7, %g6, %g6 ! Then combine them to form PA wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Fetch current nsaved from the pcb sllx %g7, 7, %g5 ! 8+8 registers each 8 bytes = 128 bytes (2^7) add %g6, %g5, %g5 ! Offset into pcb_rw SPILL stxa, %g5 + PCB_RW, 8, %asi ! Store the locals and ins saved sllx %g7, 3, %g5 add %g6, %g5, %g5 inc %g7 stba %g7, [%g6 + PCB_NSAVED] %asi retry NOTREACHED pcbspill_fail: Debugger() NOTREACHED pcbspill_other: set CPUINFO_VA, %g6 ldx [%g6 + CI_CPCB], %g6 GET_CTXBUSY %g1 ldx [%g1], %g1 ! kernel pmap is ctx 0 srlx %g6, STSHIFT, %g7 and %g7, STMASK, %g7 sll %g7, 3, %g7 ! byte offset into ctxbusy add %g7, %g1, %g1 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory srlx %g6, PDSHIFT, %g7 ! Do page directory and %g7, PDMASK, %g7 sll %g7, 3, %g7 brz,pn %g1, pcbspill_other_fail add %g7, %g1, %g1 ldxa [%g1] ASI_PHYS_CACHED, %g1 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset and %g7, PTMASK, %g7 brz %g1, pcbspill_other_fail sll %g7, 3, %g7 add %g1, %g7, %g7 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not brgez %g7, pcbspill_other_fail srlx %g7, PGSHIFT, %g7 ! Isolate PA part sll %g6, 32-PGSHIFT, %g6 ! And offset sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE srl %g6, 32-PGSHIFT, %g6 srax %g7, 8, %g7 or %g7, %g6, %g6 ! Then combine them to form PA wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults lduba [%g6 + PCB_NSAVED] %asi, %g7 ! Fetch current nsaved from the pcb sllx %g7, 7, %g5 ! 8+8 registers each 8 bytes = 128 bytes (2^7) add %g6, %g5, %g5 ! Offset into pcb_rw 1: SPILL stxa, %g5 + PCB_RW, 8, %asi ! Store the locals and ins add %g5, 16*8, %g5 ! Next location for saved register windows stxa %o6, [%g5 + PCB_RW + (14*8)] %asi ! Save %sp so we can write these all out saved ! Increments %cansave and decrements %otherwin rdpr %cwp, %g1 ! shift register window forward inc %g1 wrpr %g1, %cwp inc %g7 ! increment number of saved register windows rdpr %otherwin, %g1 ! Check to see if done spill'ing otherwin brnz,pt %g1, 1b nop stba %g7, [%g6 + PCB_NSAVED] %asi retry NOTREACHED pcbspill_other_fail: Debugger() NOTREACHED spill_normal_to_user_stack: mov %sp, %g6 ! calculate virtual address of destination stack add %g6, BIAS, %g6 mov CTX_SECONDARY, %g2 ! Is this context ok or should it be CTX_PRIMARY? XXX GET_MMU_CONTEXTID %g3, %g2, %g1 sllx %g3, 3, %g3 ! Make it into an offset into ctxbusy (see below) GET_CTXBUSY %g1 ldx [%g1 + %g3], %g1 ! Fetch pmap for current context id ! Start of code to extract PA srlx %g6, STSHIFT, %g7 and %g7, STMASK, %g7 sll %g7, 3, %g7 ! byte offset into ctxbusy add %g7, %g1, %g1 ldxa [%g1] ASI_PHYS_CACHED, %g1 ! Load pointer to directory srlx %g6, PDSHIFT, %g7 ! Do page directory and %g7, PDMASK, %g7 sll %g7, 3, %g7 brz,pn %g1, spill_normal_to_user_stack_fail add %g7, %g1, %g1 ldxa [%g1] ASI_PHYS_CACHED, %g1 srlx %g6, PTSHIFT, %g7 ! Convert to ptab offset and %g7, PTMASK, %g7 brz %g1, spill_normal_to_user_stack_fail sll %g7, 3, %g7 add %g1, %g7, %g7 ldxa [%g7] ASI_PHYS_CACHED, %g7 ! This one is not brgez %g7, spill_normal_to_user_stack_fail srlx %g7, PGSHIFT, %g7 ! Isolate PA part sll %g6, 32-PGSHIFT, %g6 ! And offset sllx %g7, PGSHIFT+8, %g7 ! There are 8 bits to the left of the PA in the TTE srl %g6, 32-PGSHIFT, %g6 srax %g7, 8, %g7 or %g7, %g6, %g6 ! Then combine them to form PA ! End of code to extract PA wr %g0, ASI_PHYS_CACHED, %asi ! Use ASI_PHYS_CACHED to prevent possible page faults SPILL stxa, %g6, 8, %asi ! Store the locals and ins saved retry NOTREACHED spill_normal_to_user_stack_fail: sir nop /* * End of traps for sun4v. */ #endif /* * We're here because we took an alignment fault in NUCLEUS context. * This could be a kernel bug or it could be due to saving a user * window to an invalid stack pointer. * * If the latter is the case, we could try to emulate unaligned accesses, * but we really don't know where to store the registers since we can't * determine if there's a stack bias. Or we could store all the regs * into the PCB and punt, until the user program uses up all the CPU's * register windows and we run out of places to store them. So for * simplicity we'll just blow them away and enter the trap code which * will generate a bus error. Debugging the problem will be a bit * complicated since lots of register windows will be lost, but what * can we do? */ checkalign: rdpr %tl, %g2 subcc %g2, 1, %g1 bneg,pn %icc, slowtrap ! Huh? sethi %hi(CPCB), %g6 ! get current pcb wrpr %g1, 0, %tl rdpr %tt, %g7 rdpr %tstate, %g4 andn %g7, 0x3f, %g5 cmp %g5, 0x080 ! window spill traps are all 0b 0000 10xx xxxx bne,a,pn %icc, slowtrap wrpr %g1, 0, %tl ! Revert TL XXX wrpr in a delay slot... #ifdef DEBUG cmp %g7, 0x34 ! If we took a datafault just before this trap bne,pt %icc, checkalignspill ! our stack's probably bad so we need to switch somewhere else nop !! !! Double data fault -- bad stack? !! wrpr %g2, %tl ! Restore trap level. sir ! Just issue a reset and don't try to recover. mov %fp, %l6 ! Save the frame pointer set EINTSTACK+USPACE+CC64FSZ-STKB, %fp ! Set the frame pointer to the middle of the idle stack add %fp, -CC64FSZ, %sp ! Create a stackframe wrpr %g0, 15, %pil ! Disable interrupts, too wrpr %g0, %g0, %canrestore ! Our stack is hozed and our PCB wrpr %g0, 7, %cansave ! probably is too, so blow away ba slowtrap ! all our register windows. wrpr %g0, 0x101, %tt #endif checkalignspill: /* * %g1 -- current tl * %g2 -- original tl * %g4 -- tstate * %g7 -- tt */ and %g4, CWP, %g5 wrpr %g5, %cwp ! Go back to the original register win /* * Remember: * * %otherwin = 0 * %cansave = NWINDOWS - 2 - %canrestore */ rdpr %otherwin, %g6 rdpr %canrestore, %g3 rdpr %ver, %g5 sub %g3, %g6, %g3 ! Calculate %canrestore - %g7 and %g5, CWP, %g5 ! NWINDOWS-1 movrlz %g3, %g0, %g3 ! Clamp at zero wrpr %g0, 0, %otherwin wrpr %g3, 0, %canrestore ! This is the new canrestore dec %g5 ! NWINDOWS-2 wrpr %g5, 0, %cleanwin ! Set cleanwin to max, since we're in-kernel sub %g5, %g3, %g5 ! NWINDOWS-2-%canrestore wrpr %g5, 0, %cansave wrpr %g0, T_ALIGN, %tt ! This was an alignment fault /* * Now we need to determine if this was a userland store or not. * Userland stores occur in anything other than the kernel spill * handlers (trap type 09x). */ and %g7, 0xff0, %g5 cmp %g5, 0x90 bz,pn %icc, slowtrap nop bclr TSTATE_PRIV, %g4 wrpr %g4, 0, %tstate ba,a,pt %icc, slowtrap nop /* * slowtrap() builds a trap frame and calls trap(). * This is called `slowtrap' because it *is*.... * We have to build a full frame for ptrace(), for instance. * * Registers: * */ slowtrap: #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG #endif #ifdef DIAGNOSTIC /* Make sure kernel stack is aligned */ btst 0x03, %sp ! 32-bit stack OK? and %sp, 0x07, %g4 ! 64-bit stack OK? bz,pt %icc, 1f cmp %g4, 0x1 ! Must end in 0b001 be,pt %icc, 1f rdpr %wstate, %g7 cmp %g7, WSTATE_KERN bnz,pt %icc, 1f ! User stack -- we'll blow it away nop set PANICSTACK-CC64FSZ-STKB, %sp 1: #endif rdpr %tt, %g4 rdpr %tstate, %g1 rdpr %tpc, %g2 rdpr %tnpc, %g3 TRAP_SETUP(-CC64FSZ-TF_SIZE) Lslowtrap_reenter: stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] mov %g4, %o1 ! (type) stx %g2, [%sp + CC64FSZ + STKB + TF_PC] rd %y, %g5 stx %g3, [%sp + CC64FSZ + STKB + TF_NPC] mov %g1, %o3 ! (pstate) st %g5, [%sp + CC64FSZ + STKB + TF_Y] mov %g2, %o2 ! (pc) sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug ! Get back to normal globals #ifdef SUN4V sethi %hi(cputyp), %g5 ld [%g5 + %lo(cputyp)], %g5 cmp %g5, CPU_SUN4V bne,pt %icc, 1f nop NORMAL_GLOBALS_SUN4V ba 2f nop 1: #endif NORMAL_GLOBALS_SUN4U 2: stx %g1, [%sp + CC64FSZ + STKB + TF_G + (1*8)] stx %g2, [%sp + CC64FSZ + STKB + TF_G + (2*8)] add %sp, CC64FSZ + STKB, %o0 ! (&tf) stx %g3, [%sp + CC64FSZ + STKB + TF_G + (3*8)] stx %g4, [%sp + CC64FSZ + STKB + TF_G + (4*8)] stx %g5, [%sp + CC64FSZ + STKB + TF_G + (5*8)] rdpr %pil, %g5 stx %g6, [%sp + CC64FSZ + STKB + TF_G + (6*8)] stx %g7, [%sp + CC64FSZ + STKB + TF_G + (7*8)] stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] /* * Phew, ready to enable traps and call C code. */ rdpr %tl, %g1 dec %g1 movrlz %g1, %g0, %g1 wrpr %g0, %g1, %tl ! Revert to kernel mode !! In the EMBEDANY memory model %g4 points to the start of the data segment. !! In our case we need to clear it before calling any C-code clr %g4 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI wrpr %g0, PSTATE_INTR, %pstate ! traps on again call _C_LABEL(trap) ! trap(tf, type, pc, pstate) nop b return_from_trap ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap NOTREACHED #if 1 /* * This code is no longer needed. */ /* * Do a `software' trap by re-entering the trap code, possibly first * switching from interrupt stack to kernel stack. This is used for * scheduling and signal ASTs (which generally occur from softclock or * tty or net interrupts). * * We enter with the trap type in %g1. All we have to do is jump to * Lslowtrap_reenter above, but maybe after switching stacks.... * * We should be running alternate globals. The normal globals and * out registers were just loaded from the old trap frame. * * Input Params: * %g1 = tstate * %g2 = tpc * %g3 = tnpc * %g4 = tt == T_AST */ softtrap: sethi %hi(EINTSTACK-STKB), %g5 sethi %hi(EINTSTACK-INTSTACK), %g7 or %g5, %lo(EINTSTACK-STKB), %g5 dec %g7 sub %g5, %sp, %g5 sethi %hi(CPCB), %g6 andncc %g5, %g7, %g0 bnz,pt %xcc, Lslowtrap_reenter LDPTR [%g6 + %lo(CPCB)], %g7 set USPACE-CC64FSZ-TF_SIZE-STKB, %g5 add %g7, %g5, %g6 SET_SP_REDZONE(%g7, %g5) #ifdef DEBUG stx %g1, [%g6 + CC64FSZ + STKB + TF_FAULT] ! Generate a new trapframe #endif stx %i0, [%g6 + CC64FSZ + STKB + TF_O + (0*8)] ! but don't bother with stx %i1, [%g6 + CC64FSZ + STKB + TF_O + (1*8)] ! locals and ins stx %i2, [%g6 + CC64FSZ + STKB + TF_O + (2*8)] stx %i3, [%g6 + CC64FSZ + STKB + TF_O + (3*8)] stx %i4, [%g6 + CC64FSZ + STKB + TF_O + (4*8)] stx %i5, [%g6 + CC64FSZ + STKB + TF_O + (5*8)] stx %i6, [%g6 + CC64FSZ + STKB + TF_O + (6*8)] stx %i7, [%g6 + CC64FSZ + STKB + TF_O + (7*8)] #ifdef DEBUG ldx [%sp + CC64FSZ + STKB + TF_I + (0*8)], %l0 ! Copy over the rest of the regs ldx [%sp + CC64FSZ + STKB + TF_I + (1*8)], %l1 ! But just dirty the locals ldx [%sp + CC64FSZ + STKB + TF_I + (2*8)], %l2 ldx [%sp + CC64FSZ + STKB + TF_I + (3*8)], %l3 ldx [%sp + CC64FSZ + STKB + TF_I + (4*8)], %l4 ldx [%sp + CC64FSZ + STKB + TF_I + (5*8)], %l5 ldx [%sp + CC64FSZ + STKB + TF_I + (6*8)], %l6 ldx [%sp + CC64FSZ + STKB + TF_I + (7*8)], %l7 stx %l0, [%g6 + CC64FSZ + STKB + TF_I + (0*8)] stx %l1, [%g6 + CC64FSZ + STKB + TF_I + (1*8)] stx %l2, [%g6 + CC64FSZ + STKB + TF_I + (2*8)] stx %l3, [%g6 + CC64FSZ + STKB + TF_I + (3*8)] stx %l4, [%g6 + CC64FSZ + STKB + TF_I + (4*8)] stx %l5, [%g6 + CC64FSZ + STKB + TF_I + (5*8)] stx %l6, [%g6 + CC64FSZ + STKB + TF_I + (6*8)] stx %l7, [%g6 + CC64FSZ + STKB + TF_I + (7*8)] ldx [%sp + CC64FSZ + STKB + TF_L + (0*8)], %l0 ldx [%sp + CC64FSZ + STKB + TF_L + (1*8)], %l1 ldx [%sp + CC64FSZ + STKB + TF_L + (2*8)], %l2 ldx [%sp + CC64FSZ + STKB + TF_L + (3*8)], %l3 ldx [%sp + CC64FSZ + STKB + TF_L + (4*8)], %l4 ldx [%sp + CC64FSZ + STKB + TF_L + (5*8)], %l5 ldx [%sp + CC64FSZ + STKB + TF_L + (6*8)], %l6 ldx [%sp + CC64FSZ + STKB + TF_L + (7*8)], %l7 stx %l0, [%g6 + CC64FSZ + STKB + TF_L + (0*8)] stx %l1, [%g6 + CC64FSZ + STKB + TF_L + (1*8)] stx %l2, [%g6 + CC64FSZ + STKB + TF_L + (2*8)] stx %l3, [%g6 + CC64FSZ + STKB + TF_L + (3*8)] stx %l4, [%g6 + CC64FSZ + STKB + TF_L + (4*8)] stx %l5, [%g6 + CC64FSZ + STKB + TF_L + (5*8)] stx %l6, [%g6 + CC64FSZ + STKB + TF_L + (6*8)] stx %l7, [%g6 + CC64FSZ + STKB + TF_L + (7*8)] #endif ba,pt %xcc, Lslowtrap_reenter mov %g6, %sp #endif #if 0 /* * breakpoint: capture as much info as possible and then call DDB * or trap, as the case may be. * * First, we switch to interrupt globals, and blow away %g7. Then * switch down one stackframe -- just fiddle w/cwp, don't save or * we'll trap. Then slowly save all the globals into our static * register buffer. etc. etc. */ breakpoint: wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! Get IG to use rdpr %cwp, %g7 inc 1, %g7 ! Equivalent of save wrpr %g7, 0, %cwp ! Now we have some unused locals to fiddle with XXX ddb_regs is now ddb-regp and is a pointer not a symbol. set _C_LABEL(ddb_regs), %l0 stx %g1, [%l0+DBR_IG+(1*8)] ! Save IGs stx %g2, [%l0+DBR_IG+(2*8)] stx %g3, [%l0+DBR_IG+(3*8)] stx %g4, [%l0+DBR_IG+(4*8)] stx %g5, [%l0+DBR_IG+(5*8)] stx %g6, [%l0+DBR_IG+(6*8)] stx %g7, [%l0+DBR_IG+(7*8)] wrpr %g0, PSTATE_KERN|PSTATE_MG, %pstate ! Get MG to use stx %g1, [%l0+DBR_MG+(1*8)] ! Save MGs stx %g2, [%l0+DBR_MG+(2*8)] stx %g3, [%l0+DBR_MG+(3*8)] stx %g4, [%l0+DBR_MG+(4*8)] stx %g5, [%l0+DBR_MG+(5*8)] stx %g6, [%l0+DBR_MG+(6*8)] stx %g7, [%l0+DBR_MG+(7*8)] wrpr %g0, PSTATE_KERN|PSTATE_AG, %pstate ! Get AG to use stx %g1, [%l0+DBR_AG+(1*8)] ! Save AGs stx %g2, [%l0+DBR_AG+(2*8)] stx %g3, [%l0+DBR_AG+(3*8)] stx %g4, [%l0+DBR_AG+(4*8)] stx %g5, [%l0+DBR_AG+(5*8)] stx %g6, [%l0+DBR_AG+(6*8)] stx %g7, [%l0+DBR_AG+(7*8)] wrpr %g0, PSTATE_KERN, %pstate ! Get G to use stx %g1, [%l0+DBR_G+(1*8)] ! Save Gs stx %g2, [%l0+DBR_G+(2*8)] stx %g3, [%l0+DBR_G+(3*8)] stx %g4, [%l0+DBR_G+(4*8)] stx %g5, [%l0+DBR_G+(5*8)] stx %g6, [%l0+DBR_G+(6*8)] stx %g7, [%l0+DBR_G+(7*8)] rdpr %canrestore, %l1 stb %l1, [%l0+DBR_CANRESTORE] rdpr %cansave, %l2 stb %l2, [%l0+DBR_CANSAVE] rdpr %cleanwin, %l3 stb %l3, [%l0+DBR_CLEANWIN] rdpr %wstate, %l4 stb %l4, [%l0+DBR_WSTATE] rd %y, %l5 stw %l5, [%l0+DBR_Y] rdpr %tl, %l6 stb %l6, [%l0+DBR_TL] dec 1, %g7 #endif /* * I will not touch any of the DDB or KGDB stuff until I know what's going * on with the symbol table. This is all still v7/v8 code and needs to be fixed. */ #ifdef KGDB /* * bpt is entered on all breakpoint traps. * If this is a kernel breakpoint, we do not want to call trap(). * Among other reasons, this way we can set breakpoints in trap(). */ bpt: set TSTATE_PRIV, %l4 andcc %l4, %l0, %g0 ! breakpoint from kernel? bz slowtrap ! no, go do regular trap nop /* * Build a trap frame for kgdb_trap_glue to copy. * Enable traps but set ipl high so that we will not * see interrupts from within breakpoints. */ save %sp, -CCFSZ-TF_SIZE, %sp ! allocate a trap frame TRAP_SETUP(-CCFSZ-TF_SIZE) or %l0, PSR_PIL, %l4 ! splhigh() wr %l4, 0, %psr ! the manual claims that this wr %l4, PSR_ET, %psr ! song and dance is necessary std %l0, [%sp + CCFSZ + 0] ! tf.tf_psr, tf.tf_pc mov %l3, %o0 ! trap type arg for kgdb_trap_glue rd %y, %l3 std %l2, [%sp + CCFSZ + 8] ! tf.tf_npc, tf.tf_y rd %wim, %l3 st %l3, [%sp + CCFSZ + 16] ! tf.tf_wim (a kgdb-only r/o field) st %g1, [%sp + CCFSZ + 20] ! tf.tf_global[1] std %g2, [%sp + CCFSZ + 24] ! etc std %g4, [%sp + CCFSZ + 32] std %g6, [%sp + CCFSZ + 40] std %i0, [%sp + CCFSZ + 48] ! tf.tf_in[0..1] std %i2, [%sp + CCFSZ + 56] ! etc std %i4, [%sp + CCFSZ + 64] std %i6, [%sp + CCFSZ + 72] /* * Now call kgdb_trap_glue(); if it returns, call trap(). */ mov %o0, %l3 ! gotta save trap type call _C_LABEL(kgdb_trap_glue) ! kgdb_trap_glue(type, &trapframe) add %sp, CCFSZ, %o1 ! (&trapframe) /* * Use slowtrap to call trap---but first erase our tracks * (put the registers back the way they were). */ mov %l3, %o0 ! slowtrap will need trap type ld [%sp + CCFSZ + 12], %l3 wr %l3, 0, %y ld [%sp + CCFSZ + 20], %g1 ldd [%sp + CCFSZ + 24], %g2 ldd [%sp + CCFSZ + 32], %g4 b Lslowtrap_reenter ldd [%sp + CCFSZ + 40], %g6 /* * Enter kernel breakpoint. Write all the windows (not including the * current window) into the stack, so that backtrace works. Copy the * supplied trap frame to the kgdb stack and switch stacks. * * kgdb_trap_glue(type, tf0) * int type; * struct trapframe *tf0; */ ENTRY_NOPROFILE(kgdb_trap_glue) save %sp, -CCFSZ, %sp flushw ! flush all windows mov %sp, %l4 ! %l4 = current %sp /* copy trapframe to top of kgdb stack */ set _C_LABEL(kgdb_stack) + KGDB_STACK_SIZE - 80, %l0 ! %l0 = tfcopy -> end_of_kgdb_stack mov 80, %l1 1: ldd [%i1], %l2 inc 8, %i1 deccc 8, %l1 std %l2, [%l0] bg 1b inc 8, %l0 #ifdef NOTDEF_DEBUG /* save old red zone and then turn it off */ sethi %hi(_C_LABEL(redzone)), %l7 ld [%l7 + %lo(_C_LABEL(redzone))], %l6 st %g0, [%l7 + %lo(_C_LABEL(redzone))] #endif /* switch to kgdb stack */ add %l0, -CCFSZ-TF_SIZE, %sp /* if (kgdb_trap(type, tfcopy)) kgdb_rett(tfcopy); */ mov %i0, %o0 call _C_LABEL(kgdb_trap) add %l0, -80, %o1 tst %o0 bnz,a kgdb_rett add %l0, -80, %g1 /* * kgdb_trap() did not handle the trap at all so the stack is * still intact. A simple `restore' will put everything back, * after we reset the stack pointer. */ mov %l4, %sp #ifdef NOTDEF_DEBUG st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! restore red zone #endif ret restore /* * Return from kgdb trap. This is sort of special. * * We know that kgdb_trap_glue wrote the window above it, so that we will * be able to (and are sure to have to) load it up. We also know that we * came from kernel land and can assume that the %fp (%i6) we load here * is proper. We must also be sure not to lower ipl (it is at splhigh()) * until we have traps disabled, due to the SPARC taking traps at the * new ipl before noticing that PSR_ET has been turned off. We are on * the kgdb stack, so this could be disastrous. * * Note that the trapframe argument in %g1 points into the current stack * frame (current window). We abandon this window when we move %g1->tf_psr * into %psr, but we will not have loaded the new %sp yet, so again traps * must be disabled. */ kgdb_rett: rd %psr, %g4 ! turn off traps wr %g4, PSR_ET, %psr /* use the three-instruction delay to do something useful */ ld [%g1], %g2 ! pick up new %psr ld [%g1 + 12], %g3 ! set %y wr %g3, 0, %y #ifdef NOTDEF_DEBUG st %l6, [%l7 + %lo(_C_LABEL(redzone))] ! and restore red zone #endif wr %g0, 0, %wim ! enable window changes nop; nop; nop /* now safe to set the new psr (changes CWP, leaves traps disabled) */ wr %g2, 0, %psr ! set rett psr (including cond codes) /* 3 instruction delay before we can use the new window */ /*1*/ ldd [%g1 + 24], %g2 ! set new %g2, %g3 /*2*/ ldd [%g1 + 32], %g4 ! set new %g4, %g5 /*3*/ ldd [%g1 + 40], %g6 ! set new %g6, %g7 /* now we can use the new window */ mov %g1, %l4 ld [%l4 + 4], %l1 ! get new pc ld [%l4 + 8], %l2 ! get new npc ld [%l4 + 20], %g1 ! set new %g1 /* set up returnee's out registers, including its %sp */ ldd [%l4 + 48], %i0 ldd [%l4 + 56], %i2 ldd [%l4 + 64], %i4 ldd [%l4 + 72], %i6 /* load returnee's window, making the window above it be invalid */ restore restore %g0, 1, %l1 ! move to inval window and set %l1 = 1 rd %psr, %l0 srl %l1, %l0, %l1 wr %l1, 0, %wim ! %wim = 1 << (%psr & 31) sethi %hi(CPCB), %l1 LDPTR [%l1 + %lo(CPCB)], %l1 and %l0, 31, %l0 ! CWP = %psr & 31; ! st %l0, [%l1 + PCB_WIM] ! cpcb->pcb_wim = CWP; save %g0, %g0, %g0 ! back to window to reload ! LOADWIN(%sp) save %g0, %g0, %g0 ! back to trap window /* note, we have not altered condition codes; safe to just rett */ RETT #endif /* * syscall_setup() builds a trap frame and calls syscall(). * sun_syscall is same but delivers sun system call number * XXX should not have to save&reload ALL the registers just for * ptrace... */ syscall_setup: #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG #endif TRAP_SETUP(-CC64FSZ-TF_SIZE) #ifdef DEBUG rdpr %tt, %o1 ! debug sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug #endif ! Get back to normal globals #ifdef SUN4V sethi %hi(cputyp), %g5 ld [%g5 + %lo(cputyp)], %g5 cmp %g5, CPU_SUN4V bne,pt %icc, 1f nop NORMAL_GLOBALS_SUN4V ba 2f nop 1: #endif NORMAL_GLOBALS_SUN4U 2: stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] mov %g1, %o1 ! code rdpr %tpc, %o2 ! (pc) stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] rdpr %tstate, %g1 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] rdpr %tnpc, %o3 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] rd %y, %o4 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] wrpr %g0, 0, %tl ! return to tl=0 stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] add %sp, CC64FSZ + STKB, %o0 ! (&tf) stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] stx %o2, [%sp + CC64FSZ + STKB + TF_PC] stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] st %o4, [%sp + CC64FSZ + STKB + TF_Y] rdpr %pil, %g5 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] !! In the EMBEDANY memory model %g4 points to the start of the data segment. !! In our case we need to clear it before calling any C-code clr %g4 wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI sethi %hi(CURLWP), %l1 LDPTR [%l1 + %lo(CURLWP)], %l1 LDPTR [%l1 + L_PROC], %l1 ! now %l1 points to p LDPTR [%l1 + P_MD_SYSCALL], %l1 call %l1 wrpr %g0, PSTATE_INTR, %pstate ! turn on interrupts /* see `lwp_trampoline' for the reason for this label */ return_from_syscall: wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts wrpr %g0, 0, %tl ! Return to tl==0 b return_from_trap nop NOTREACHED /* * interrupt_vector: * * Spitfire chips never get level interrupts directly from H/W. * Instead, all interrupts come in as interrupt_vector traps. * The interrupt number or handler address is an 11 bit number * encoded in the first interrupt data word. Additional words * are application specific and used primarily for cross-calls. * * The interrupt vector handler then needs to identify the * interrupt source from the interrupt number and arrange to * invoke the interrupt handler. This can either be done directly * from here, or a softint at a particular level can be issued. * * To call an interrupt directly and not overflow the trap stack, * the trap registers should be saved on the stack, registers * cleaned, trap-level decremented, the handler called, and then * the process must be reversed. * * To simplify life all we do here is issue an appropriate softint. * * Note: It is impossible to identify or change a device's * interrupt number until it is probed. That's the * purpose for all the funny interrupt acknowledge * code. * */ /* * Vectored interrupts: * * When an interrupt comes in, interrupt_vector uses the interrupt * vector number to lookup the appropriate intrhand from the intrlev * array. It then looks up the interrupt level from the intrhand * structure. It uses the level to index the intrpending array, * which is 8 slots for each possible interrupt level (so we can * shift instead of multiply for address calculation). It hunts for * any available slot at that level. Available slots are NULL. * * Then interrupt_vector uses the interrupt level in the intrhand * to issue a softint of the appropriate level. The softint handler * figures out what level interrupt it's handling and pulls the first * intrhand pointer out of the intrpending array for that interrupt * level, puts a NULL in its place, clears the interrupt generator, * and invokes the interrupt handler. */ /* intrpending array is now in per-CPU structure. */ #ifdef DEBUG #define INTRDEBUG_VECTOR 0x1 #define INTRDEBUG_LEVEL 0x2 #define INTRDEBUG_FUNC 0x4 #define INTRDEBUG_SPUR 0x8 .data .globl _C_LABEL(intrdebug) _C_LABEL(intrdebug): .word 0x0 /* * Note: we use the local label `97' to branch forward to, to skip * actual debugging code following a `intrdebug' bit test. */ #endif .text interrupt_vector: #ifdef TRAPSTATS set _C_LABEL(kiveccnt), %g1 set _C_LABEL(iveccnt), %g2 rdpr %tl, %g3 dec %g3 movrz %g3, %g2, %g1 lduw [%g1], %g2 inc %g2 stw %g2, [%g1] #endif ldxa [%g0] ASI_IRSR, %g1 mov IRDR_0H, %g7 ldxa [%g7] ASI_IRDR, %g7 ! Get interrupt number membar #Sync btst IRSR_BUSY, %g1 bz,pn %icc, 3f ! spurious interrupt #ifdef MULTIPROCESSOR sethi %hi(KERNBASE), %g1 cmp %g7, %g1 bl,a,pt %xcc, Lsoftint_regular ! >= KERNBASE is a fast cross-call and %g7, (MAXINTNUM-1), %g7 ! XXX make sun4us work mov IRDR_1H, %g2 ldxa [%g2] ASI_IRDR, %g2 ! Get IPI handler argument 1 mov IRDR_2H, %g3 ldxa [%g3] ASI_IRDR, %g3 ! Get IPI handler argument 2 stxa %g0, [%g0] ASI_IRSR ! Ack IRQ membar #Sync ! Should not be needed due to retry jmpl %g7, %g0 nop #else and %g7, (MAXINTNUM-1), %g7 ! XXX make sun4us work #endif Lsoftint_regular: stxa %g0, [%g0] ASI_IRSR ! Ack IRQ membar #Sync ! Should not be needed due to retry sethi %hi(_C_LABEL(intrlev)), %g3 sllx %g7, PTRSHFT, %g5 ! Calculate entry number or %g3, %lo(_C_LABEL(intrlev)), %g3 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler brz,pn %g5, 3f ! NULL means it isn't registered yet. Skip it. nop ! increment per-ivec counter ldx [%g5 + IH_CNT], %g1 inc %g1 stx %g1, [%g5 + IH_CNT] setup_sparcintr: LDPTR [%g5+IH_PEND], %g6 ! Read pending flag brnz,pn %g6, ret_from_intr_vector ! Skip it if it's running ldub [%g5+IH_PIL], %g6 ! Read interrupt level sethi %hi(CPUINFO_VA+CI_INTRPENDING), %g1 sll %g6, PTRSHFT, %g3 ! Find start of table for this IPL or %g1, %lo(CPUINFO_VA+CI_INTRPENDING), %g1 add %g1, %g3, %g1 1: LDPTR [%g1], %g3 ! Load list head STPTR %g3, [%g5+IH_PEND] ! Link our intrhand node in mov %g5, %g7 CASPTRA [%g1] ASI_N, %g3, %g7 cmp %g7, %g3 ! Did it work? bne,pn CCCR, 1b ! No, try again .empty 2: #ifdef NOT_DEBUG set _C_LABEL(intrdebug), %g7 ld [%g7], %g7 btst INTRDEBUG_VECTOR, %g7 bz,pt %icc, 97f nop cmp %g6, 0xa ! ignore clock interrupts? bz,pt %icc, 97f nop STACKFRAME(-CC64FSZ) ! Get a clean register window LOAD_ASCIZ(%o0,\ "interrupt_vector: number %lx softint mask %lx pil %lu slot %p\n") mov %g2, %o1 rdpr %pil, %o3 mov %g1, %o4 GLOBTOLOC clr %g4 call prom_printf mov %g6, %o2 LOCTOGLOB restore 97: #endif mov 1, %g7 sll %g7, %g6, %g6 wr %g6, 0, SET_SOFTINT ! Invoke a softint .global ret_from_intr_vector ret_from_intr_vector: retry NOTREACHED 3: #ifdef NOT_DEBUG /* always do this */ set _C_LABEL(intrdebug), %g6 ld [%g6], %g6 btst INTRDEBUG_SPUR, %g6 bz,pt %icc, 97f nop #endif #if 1 set PANICSTACK-STKB, %g1 ! Use panic stack temporarily save %g1, -CC64FSZ, %sp ! Get a clean register window LOAD_ASCIZ(%o0, "interrupt_vector: spurious vector %lx at pil %d\n") mov %g7, %o1 GLOBTOLOC clr %g4 call prom_printf rdpr %pil, %o2 LOCTOGLOB restore 97: #endif ba,a ret_from_intr_vector nop ! XXX spitfire bug? sun4v_cpu_mondo: ! XXX Rework this when a UP kernel works - crash for now sir mov 0x3c0, %g1 ! CPU Mondo Queue Head ldxa [%g1] ASI_QUEUE, %g2 ! fetch index value for head set CPUINFO_VA, %g3 ldx [%g3 + CI_PADDR], %g3 add %g3, CI_CPUMQ, %g3 ldxa [%g3] ASI_PHYS_CACHED, %g3 ! fetch head element ldxa [%g3 + %g2] ASI_PHYS_CACHED, %g4 ! fetch func add %g2, 8, %g5 ldxa [%g3 + %g5] ASI_PHYS_CACHED, %g5 ! fetch arg1 add %g2, 16, %g6 ldxa [%g3 + %g6] ASI_PHYS_CACHED, %g6 ! fetch arg2 add %g2, 64, %g2 ! point to next element in queue and %g2, 0x7ff, %g2 ! modulo queue size 2048 (32*64) stxa %g2, [%g1] ASI_QUEUE ! update head index membar #Sync mov %g4, %g2 mov %g5, %g3 mov %g6, %g5 jmpl %g2, %g0 nop ! No store here! retry NOTREACHED sun4v_dev_mondo: mov 0x3d0, %g1 ! Dev Mondo Queue Head ldxa [%g1] ASI_QUEUE, %g2 ! fetch index value mov 0x3d8, %g1 ! Dev Mondo Queue Tail ldxa [%g1] ASI_QUEUE, %g4 ! fetch index value cmp %g2, %g4 ! head = queue? bne,pt %xcc, 2f ! unusually not the case nop retry ! unlikely, ignore interrupt 2: set CPUINFO_VA, %g3 ! fetch cpuinfo pa ldx [%g3 + CI_PADDR], %g3 ! fetch intstack pa set CPUINFO_VA-INTSTACK, %g4 ! offset to cpuinfo add %g4, %g3, %g3 ! %g3 is now cpuifo add %g3, CI_DEVMQ, %g3 ! calc offset to devmq ldxa [%g3] ASI_PHYS_CACHED, %g3 ! fetch address of devmq ldxa [%g3 + %g2] ASI_PHYS_CACHED, %g5 ! add %g2, 64, %g2 ! each element is 64 bytes and %g2, 0x7ff, %g2 ! assume 32 elements mov 0x3d0, %g1 ! Dev Mondo Queue Head stxa %g2, [%g1] ASI_QUEUE ! adjust head index value membar #Sync cmp %g5, MAXINTNUM ! Handle both sun4v legacy (sysino) and cookies. bgeu,pn %xcc, 1f ! See UltraSPARC Virtual Machine Specification nop ! version 3 chapter 6 (Interrupt model) sethi %hi(_C_LABEL(intrlev)), %g3 sllx %g5, PTRSHFT, %g5 ! Calculate entry number or %g3, %lo(_C_LABEL(intrlev)), %g3 LDPTR [%g3 + %g5], %g5 ! We have a pointer to the handler 1: brnz,pt %g5, setup_sparcintr ! branch if valid handle nop ba,a 3b ! log if invalid handle nop /* * Ultra1 and Ultra2 CPUs use soft interrupts for everything. What we do * on a soft interrupt, is we should check which bits in SOFTINT(%asr22) * are set, handle those interrupts, then clear them by setting the * appropriate bits in CLEAR_SOFTINT(%asr21). * * We have an array of 8 interrupt vector slots for each of 15 interrupt * levels. If a vectored interrupt can be dispatched, the dispatch * routine will place a pointer to an intrhand structure in one of * the slots. The interrupt handler will go through the list to look * for an interrupt to dispatch. If it finds one it will pull it off * the list, free the entry, and call the handler. The code is like * this: * * for (i=0; i<8; i++) * if (ih = intrpending[intlev][i]) { * intrpending[intlev][i] = NULL; * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) * return; * strayintr(&frame); * return; * } * * Otherwise we go back to the old style of polled interrupts. * * After preliminary setup work, the interrupt is passed to each * registered handler in turn. These are expected to return nonzero if * they took care of the interrupt. If a handler claims the interrupt, * we exit (hardware interrupts are latched in the requestor so we'll * just take another interrupt in the unlikely event of simultaneous * interrupts from two different devices at the same level). If we go * through all the registered handlers and no one claims it, we report a * stray interrupt. This is more or less done as: * * for (ih = intrhand[intlev]; ih; ih = ih->ih_next) * if ((*ih->ih_fun)(ih->ih_arg ? ih->ih_arg : &frame)) * return; * strayintr(&frame); * * Inputs: * %l0 = %tstate * %l1 = return pc * %l2 = return npc * %l3 = interrupt level * (software interrupt only) %l4 = bits to clear in interrupt register * * Internal: * %l4, %l5: local variables * %l6 = %y * %l7 = %g1 * %g2..%g7 go to stack * * An interrupt frame is built in the space for a full trapframe; * this contains the psr, pc, npc, and interrupt level. * * The level of this interrupt is determined by: * * IRQ# = %tt - 0x40 */ ENTRY_NOPROFILE(sparc_interrupt) #ifdef TRAPS_USE_IG ! This is for interrupt debugging wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG #endif /* * If this is a %tick or %stick softint, clear it then call * interrupt_vector. Only one of them should be enabled at any given * time. */ rd SOFTINT, %g1 set TICK_INT|STICK_INT, %g5 andcc %g5, %g1, %g5 bz,pt %icc, 0f sethi %hi(CPUINFO_VA+CI_TICK_IH), %g3 wr %g0, %g5, CLEAR_SOFTINT ba,pt %icc, setup_sparcintr LDPTR [%g3 + %lo(CPUINFO_VA+CI_TICK_IH)], %g5 0: #ifdef TRAPSTATS sethi %hi(_C_LABEL(kintrcnt)), %g1 sethi %hi(_C_LABEL(uintrcnt)), %g2 or %g1, %lo(_C_LABEL(kintrcnt)), %g1 or %g1, %lo(_C_LABEL(uintrcnt)), %g2 rdpr %tl, %g3 dec %g3 movrz %g3, %g2, %g1 lduw [%g1], %g2 inc %g2 stw %g2, [%g1] /* See if we're on the interrupt stack already. */ set EINTSTACK, %g2 set (EINTSTACK-INTSTACK), %g1 btst 1, %sp add %sp, BIAS, %g3 movz %icc, %sp, %g3 srl %g3, 0, %g3 sub %g2, %g3, %g3 cmp %g3, %g1 bgu 1f set _C_LABEL(intristk), %g1 lduw [%g1], %g2 inc %g2 stw %g2, [%g1] 1: #endif INTR_SETUP(-CC64FSZ-TF_SIZE) ! Switch to normal globals so we can save them #ifdef SUN4V sethi %hi(cputyp), %g5 ld [%g5 + %lo(cputyp)], %g5 cmp %g5, CPU_SUN4V bne,pt %icc, 1f nop NORMAL_GLOBALS_SUN4V ! Save the normal globals stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] /* * In the EMBEDANY memory model %g4 points to the start of the * data segment. In our case we need to clear it before calling * any C-code. */ clr %g4 ba 2f nop 1: #endif NORMAL_GLOBALS_SUN4U ! Save the normal globals stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] /* * In the EMBEDANY memory model %g4 points to the start of the * data segment. In our case we need to clear it before calling * any C-code. */ clr %g4 flushw ! Do not remove this insn -- causes interrupt loss 2: rd %y, %l6 INCR64(CPUINFO_VA+CI_NINTR) ! cnt.v_ints++ (clobbers %o0,%o1) rdpr %tt, %l5 ! Find out our current IPL rdpr %tstate, %l0 rdpr %tpc, %l1 rdpr %tnpc, %l2 rdpr %tl, %l3 ! Dump our trap frame now we have taken the IRQ stw %l6, [%sp + CC64FSZ + STKB + TF_Y] ! Silly, but we need to save this for rft dec %l3 wrpr %g0, %l3, %tl sth %l5, [%sp + CC64FSZ + STKB + TF_TT]! debug stx %l0, [%sp + CC64FSZ + STKB + TF_TSTATE] ! set up intrframe/clockframe stx %l1, [%sp + CC64FSZ + STKB + TF_PC] btst TSTATE_PRIV, %l0 ! User mode? stx %l2, [%sp + CC64FSZ + STKB + TF_NPC] sub %l5, 0x40, %l6 ! Convert to interrupt level sethi %hi(_C_LABEL(intr_evcnts)), %l4 stb %l6, [%sp + CC64FSZ + STKB + TF_PIL] ! set up intrframe/clockframe rdpr %pil, %o1 mulx %l6, EVC_SIZE, %l3 or %l4, %lo(_C_LABEL(intr_evcnts)), %l4 ! intrcnt[intlev]++; stb %o1, [%sp + CC64FSZ + STKB + TF_OLDPIL] ! old %pil ldx [%l4 + %l3], %o0 add %l4, %l3, %l4 clr %l5 ! Zero handled count #ifdef MULTIPROCESSOR mov 1, %l3 ! Ack softint 1: add %o0, 1, %l7 casxa [%l4] ASI_N, %o0, %l7 cmp %o0, %l7 bne,a,pn %xcc, 1b ! retry if changed mov %l7, %o0 #else inc %o0 mov 1, %l3 ! Ack softint stx %o0, [%l4] #endif sll %l3, %l6, %l3 ! Generate IRQ mask wrpr %l6, %pil #define SOFTINT_INT \ (1<ih_pending = prev mov %l2, %o0 ! prev = ih mov %l7, %l2 ! ih = ih->ih_pending LDPTR [%l2 + IH_PEND], %l7 cmp %l7, -1 ! Last slot? bne,pn CCCR, 1b membar #LoadStore ba,pt CCCR, 3f mov %o0, %l7 ! save ih->ih_pending 2: add %sp, CC64FSZ+STKB, %o2 ! tf = %sp + CC64FSZ + STKB LDPTR [%l2 + IH_PEND], %l7 ! save ih->ih_pending membar #LoadStore 3: STPTR %g0, [%l2 + IH_PEND] ! Clear pending flag membar #Sync LDPTR [%l2 + IH_FUN], %o4 ! ih->ih_fun LDPTR [%l2 + IH_ARG], %o0 ! ih->ih_arg #ifdef NOT_DEBUG set _C_LABEL(intrdebug), %o3 ld [%o2], %o3 btst INTRDEBUG_FUNC, %o3 bz,a,pt %icc, 97f nop cmp %l6, 0xa ! ignore clock interrupts? bz,pt %icc, 97f nop STACKFRAME(-CC64FSZ) ! Get a clean register window LOAD_ASCIZ(%o0, "sparc_interrupt: func %p arg %p\n") mov %i0, %o2 ! arg GLOBTOLOC call prom_printf mov %i4, %o1 ! func LOCTOGLOB restore 97: mov %l4, %o1 #endif wrpr %g0, PSTATE_INTR, %pstate ! Reenable interrupts jmpl %o4, %o7 ! handled = (*ih->ih_fun)(...) movrz %o0, %o2, %o0 ! arg = (arg == 0) ? arg : tf wrpr %g0, PSTATE_KERN, %pstate ! Disable interrupts LDPTR [%l2 + IH_CLR], %l1 membar #Sync brz,pn %l1, 0f add %l5, %o0, %l5 stx %g0, [%l1] ! Clear intr source membar #Sync ! Should not be needed 0: LDPTR [%l2 + IH_ACK], %l1 ! ih->ih_ack brz,pn %l1, 1f nop jmpl %l1, %o7 ! (*ih->ih_ack)(ih) mov %l2, %o0 1: cmp %l7, -1 bne,pn CCCR, 2b ! 'Nother? mov %l7, %l2 intrcmplt: /* * Re-read SOFTINT to see if any new pending interrupts * at this level. */ mov 1, %l3 ! Ack softint rd SOFTINT, %l7 ! %l5 contains #intr handled. sll %l3, %l6, %l3 ! Generate IRQ mask btst %l3, %l7 ! leave mask in %l3 for retry code bnz,pn %icc, sparc_intr_retry mov 1, %l5 ! initialize intr count for next run ! Decrement this cpu's interrupt depth in case of hardintrs btst SOFTINT_INT, %l3 bnz,pn %icc, 1f sethi %hi(CPUINFO_VA+CI_IDEPTH), %l4 ld [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)], %l5 dec %l5 st %l5, [%l4 + %lo(CPUINFO_VA+CI_IDEPTH)] 1: #ifdef NOT_DEBUG set _C_LABEL(intrdebug), %o2 ld [%o2], %o2 btst INTRDEBUG_FUNC, %o2 bz,a,pt %icc, 97f nop cmp %l6, 0xa ! ignore clock interrupts? bz,pt %icc, 97f nop STACKFRAME(-CC64FSZ) ! Get a clean register window LOAD_ASCIZ(%o0, "sparc_interrupt: done\n") GLOBTOLOC call prom_printf nop LOCTOGLOB restore 97: #endif ldub [%sp + CC64FSZ + STKB + TF_OLDPIL], %l3 ! restore old %pil wrpr %l3, 0, %pil b return_from_trap ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap #ifdef notyet /* * Level 12 (ZS serial) interrupt. Handle it quickly, schedule a * software interrupt, and get out. Do the software interrupt directly * if we would just take it on the way out. * * Input: * %l0 = %psr * %l1 = return pc * %l2 = return npc * Internal: * %l3 = zs device * %l4, %l5 = temporary * %l6 = rr3 (or temporary data) + 0x100 => need soft int * %l7 = zs soft status */ zshard: #endif /* notyet */ .globl return_from_trap, rft_kernel, rft_user .globl softtrap, slowtrap /* * Various return-from-trap routines (see return_from_trap). */ /* * Return from trap. * registers are: * * [%sp + CC64FSZ + STKB] => trap frame * %g1 => tstate from trap frame * * We must load all global, out, and trap registers from the trap frame. * * If returning to kernel, we should be at the proper trap level because * we don't touch %tl. * * When returning to user mode, the trap level does not matter, as it * will be set explicitly. * * If we are returning to user code, we must: * 1. Check for register windows in the pcb that belong on the stack. * If there are any, reload them */ return_from_trap: #ifdef DEBUG !! Make sure we don't have pc == npc == 0 or we suck. ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 orcc %g2, %g3, %g0 tz %icc, 1 #endif !! !! We'll make sure we flush our pcb here, rather than later. !! ! ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! already passed in, no need to reload btst TSTATE_PRIV, %g1 ! returning to userland? !! !! Let all pending interrupts drain before returning to userland !! bnz,pn %icc, 1f ! Returning to userland? nop ENABLE_INTERRUPTS %g5 wrpr %g0, %g0, %pil ! Lower IPL 1: !! Make sure we have no IRQs DISABLE_INTERRUPTS %g5 #ifdef SUN4V sethi %hi(cputyp), %g5 ld [%g5 + %lo(cputyp)], %g5 cmp %g5, CPU_SUN4V bne,pt %icc, 1f nop !! Make sure we have normal globals NORMAL_GLOBALS_SUN4V /* Restore normal globals */ ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 /* Switch to alternate globals */ ALTERNATE_GLOBALS_SUN4V ba 2f nop 1: #endif !! Make sure we have normal globals NORMAL_GLOBALS_SUN4U /* Restore normal globals */ ldx [%sp + CC64FSZ + STKB + TF_G + (1*8)], %g1 ldx [%sp + CC64FSZ + STKB + TF_G + (2*8)], %g2 ldx [%sp + CC64FSZ + STKB + TF_G + (3*8)], %g3 ldx [%sp + CC64FSZ + STKB + TF_G + (4*8)], %g4 ldx [%sp + CC64FSZ + STKB + TF_G + (5*8)], %g5 ldx [%sp + CC64FSZ + STKB + TF_G + (6*8)], %g6 ldx [%sp + CC64FSZ + STKB + TF_G + (7*8)], %g7 /* Switch to alternate globals */ #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG #else ALTERNATE_GLOBALS_SUN4U #endif 2: /* Load outs */ ldx [%sp + CC64FSZ + STKB + TF_O + (0*8)], %i0 ldx [%sp + CC64FSZ + STKB + TF_O + (1*8)], %i1 ldx [%sp + CC64FSZ + STKB + TF_O + (2*8)], %i2 ldx [%sp + CC64FSZ + STKB + TF_O + (3*8)], %i3 ldx [%sp + CC64FSZ + STKB + TF_O + (4*8)], %i4 ldx [%sp + CC64FSZ + STKB + TF_O + (5*8)], %i5 ldx [%sp + CC64FSZ + STKB + TF_O + (6*8)], %i6 ldx [%sp + CC64FSZ + STKB + TF_O + (7*8)], %i7 /* Now load trap registers into alternate globals */ ld [%sp + CC64FSZ + STKB + TF_Y], %g4 ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! load new values wr %g4, 0, %y ldx [%sp + CC64FSZ + STKB + TF_PC], %g2 ldx [%sp + CC64FSZ + STKB + TF_NPC], %g3 #ifdef NOTDEF_DEBUG ldub [%sp + CC64FSZ + STKB + TF_PIL], %g5 ! restore %pil wrpr %g5, %pil ! DEBUG #endif /* Returning to user mode or kernel mode? */ btst TSTATE_PRIV, %g1 ! returning to userland? bz,pt %icc, rft_user sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! first instr of rft_user /* * Return from trap, to kernel. * * We will assume, for the moment, that all kernel traps are properly stacked * in the trap registers, so all we have to do is insert the (possibly modified) * register values into the trap registers then do a retry. * */ rft_kernel: rdpr %tl, %g4 ! Grab a set of trap registers inc %g4 wrpr %g4, %g0, %tl wrpr %g3, 0, %tnpc wrpr %g2, 0, %tpc wrpr %g1, 0, %tstate rdpr %canrestore, %g2 brnz %g2, 1f nop wr %g0, ASI_NUCLEUS, %asi rdpr %cwp, %g1 dec %g1 wrpr %g1, %cwp #ifdef _LP64 FILL ldxa, %sp+BIAS, 8, %asi #else FILL lda, %sp, 4, %asi #endif restored inc %g1 wrpr %g1, %cwp 1: restore rdpr %tstate, %g1 ! Since we may have trapped our regs may be toast rdpr %cwp, %g2 andn %g1, CWP, %g1 wrpr %g1, %g2, %tstate ! Put %cwp in %tstate CLRTT #ifdef TRAPSTATS rdpr %tl, %g2 set _C_LABEL(rftkcnt), %g1 sllx %g2, 2, %g2 add %g1, %g2, %g1 lduw [%g1], %g2 inc %g2 stw %g2, [%g1] #endif #if 0 wrpr %g0, 0, %cleanwin ! DEBUG #endif #if defined(DDB) && defined(MULTIPROCESSOR) set sparc64_ipi_pause_trap_point, %g1 rdpr %tpc, %g2 cmp %g1, %g2 bne,pt %icc, 0f nop done 0: #endif retry NOTREACHED /* * Return from trap, to user. Checks for scheduling trap (`ast') first; * will re-enter trap() if set. Note that we may have to switch from * the interrupt stack to the kernel stack in this case. * %g1 = %tstate * %g2 = return %pc * %g3 = return %npc * If returning to a valid window, just set psr and return. */ .data rft_wcnt: .word 0 .text rft_user: ! sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 ! (done above) lduw [%g7 + %lo(CPUINFO_VA+CI_WANT_AST)], %g7! want AST trap? brnz,pn %g7, softtrap ! yes, re-enter trap with type T_AST mov T_AST, %g4 #ifdef NOTDEF_DEBUG sethi %hi(CPCB), %g4 LDPTR [%g4 + %lo(CPCB)], %g4 ldub [%g4 + PCB_NSAVED], %g4 ! nsaved brz,pt %g4, 2f ! Only print if nsaved <> 0 nop set 1f, %o0 mov %g4, %o1 mov %g2, %o2 ! pc wr %g0, ASI_DMMU, %asi ! restore the user context ldxa [CTX_SECONDARY] %asi, %o3 ! ctx GLOBTOLOC mov %g3, %o5 call printf mov %i6, %o4 ! sp ! wrpr %g0, PSTATE_INTR, %pstate ! Allow IRQ service ! wrpr %g0, PSTATE_KERN, %pstate ! DenyIRQ service LOCTOGLOB 1: .data .asciz "rft_user: nsaved=%x pc=%d ctx=%x sp=%x npc=%p\n" _ALIGN .text #endif /* * NB: only need to do this after a cache miss */ #ifdef TRAPSTATS set _C_LABEL(rftucnt), %g6 lduw [%g6], %g7 inc %g7 stw %g7, [%g6] #endif /* * Now check to see if any regs are saved in the pcb and restore them. * * Here we need to undo the damage caused by switching to a kernel * stack. * * We will use alternate globals %g4..%g7 because %g1..%g3 are used * by the data fault trap handlers and we don't want possible conflict. */ sethi %hi(CPCB), %g6 rdpr %otherwin, %g7 ! restore register window controls #ifdef DEBUG rdpr %canrestore, %g5 ! DEBUG tst %g5 ! DEBUG tnz %icc, 1; nop ! DEBUG ! mov %g0, %g5 ! There should be *NO* %canrestore add %g7, %g5, %g7 ! DEBUG #endif wrpr %g0, %g7, %canrestore LDPTR [%g6 + %lo(CPCB)], %g6 wrpr %g0, 0, %otherwin ldub [%g6 + PCB_NSAVED], %g7 ! Any saved reg windows? wrpr %g0, WSTATE_USER, %wstate ! Need to know where our sp points #ifdef DEBUG set rft_wcnt, %g4 ! Keep track of all the windows we restored stw %g7, [%g4] #endif brz,pt %g7, 5f ! No saved reg wins nop dec %g7 ! We can do this now or later. Move to last entry #ifdef DEBUG rdpr %canrestore, %g4 ! DEBUG Make sure we've restored everything brnz,a,pn %g4, 0f ! DEBUG sir ! DEBUG we should NOT have any usable windows here 0: ! DEBUG wrpr %g0, 5, %tl #endif rdpr %otherwin, %g4 sll %g7, 7, %g5 ! calculate ptr into rw64 array 8*16 == 128 or 7 bits brz,pt %g4, 6f ! We should not have any user windows left add %g5, %g6, %g5 set 1f, %o0 mov %g7, %o1 mov %g4, %o2 call printf wrpr %g0, PSTATE_KERN, %pstate set 2f, %o0 call panic nop NOTREACHED .data 1: .asciz "pcb_nsaved=%x and otherwin=%x\n" 2: .asciz "rft_user\n" _ALIGN .text 6: 3: restored ! Load in the window restore ! This should not trap! ldx [%g5 + PCB_RW + ( 0*8)], %l0 ! Load the window from the pcb ldx [%g5 + PCB_RW + ( 1*8)], %l1 ldx [%g5 + PCB_RW + ( 2*8)], %l2 ldx [%g5 + PCB_RW + ( 3*8)], %l3 ldx [%g5 + PCB_RW + ( 4*8)], %l4 ldx [%g5 + PCB_RW + ( 5*8)], %l5 ldx [%g5 + PCB_RW + ( 6*8)], %l6 ldx [%g5 + PCB_RW + ( 7*8)], %l7 ldx [%g5 + PCB_RW + ( 8*8)], %i0 ldx [%g5 + PCB_RW + ( 9*8)], %i1 ldx [%g5 + PCB_RW + (10*8)], %i2 ldx [%g5 + PCB_RW + (11*8)], %i3 ldx [%g5 + PCB_RW + (12*8)], %i4 ldx [%g5 + PCB_RW + (13*8)], %i5 ldx [%g5 + PCB_RW + (14*8)], %i6 ldx [%g5 + PCB_RW + (15*8)], %i7 #ifdef DEBUG stx %g0, [%g5 + PCB_RW + (14*8)] ! DEBUG mark that we've saved this one #endif cmp %g5, %g6 bgu,pt %xcc, 3b ! Next one? dec 8*16, %g5 stb %g0, [%g6 + PCB_NSAVED] ! Clear them out so we won't do this again GET_MAXCWP %g5 add %g5, %g7, %g4 dec 1, %g5 ! NWINDOWS-1-1 wrpr %g5, 0, %cansave wrpr %g0, 0, %canrestore ! Make sure we have no freeloaders XXX wrpr %g0, WSTATE_USER, %wstate ! Save things to user space mov %g7, %g5 ! We already did one restore 4: rdpr %canrestore, %g4 inc %g4 deccc %g5 wrpr %g4, 0, %cleanwin ! Make *sure* we don't trap to cleanwin bge,a,pt %xcc, 4b ! return to starting regwin save %g0, %g0, %g0 ! This may force a datafault #ifdef DEBUG wrpr %g0, 0, %tl #endif #ifdef TRAPSTATS set _C_LABEL(rftuld), %g5 lduw [%g5], %g4 inc %g4 stw %g4, [%g5] #endif !! !! We can't take any save faults in here 'cause they will never be serviced !! #ifdef DEBUG sethi %hi(CPCB), %g5 LDPTR [%g5 + %lo(CPCB)], %g5 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? tst %g5 tnz %icc, 1; nop ! Debugger if we still have saved windows bne,a rft_user ! Try starting over again sethi %hi(CPUINFO_VA+CI_WANT_AST), %g7 #endif /* * Set up our return trapframe so we can recover if we trap from here * on in. */ wrpr %g0, 1, %tl ! Set up the trap state wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc ba,pt %icc, 6f wrpr %g1, %g0, %tstate 5: /* * Set up our return trapframe so we can recover if we trap from here * on in. */ wrpr %g0, 1, %tl ! Set up the trap state wrpr %g2, 0, %tpc wrpr %g3, 0, %tnpc wrpr %g1, %g0, %tstate /* * The restore instruction further down may cause the trap level * to exceed the maximum trap level on sun4v, so a manual fill * may be necessary. */ #ifdef SUN4V sethi %hi(cputyp), %g5 ld [%g5 + %lo(cputyp)], %g5 cmp %g5, CPU_SUN4V bne,pt %icc, 1f nop ! Only manual fill if the restore instruction will cause a fill trap rdpr %canrestore, %g5 brnz %g5, 1f nop ! Do a manual fill wr %g0, ASI_AIUS, %asi rdpr %cwp, %g4 dec %g4 wrpr %g4, 0, %cwp rft_user_fault_start: FILL ldxa, %sp+BIAS, 8, %asi rft_user_fault_end: restored inc %g4 wrpr %g4, 0, %cwp 1: #endif restore 6: rdpr %canrestore, %g5 wrpr %g5, 0, %cleanwin ! Force cleanup of kernel windows #ifdef NOTDEF_DEBUG ldx [%g6 + CC64FSZ + STKB + TF_L + (0*8)], %g5! DEBUG -- get proper value for %l0 cmp %l0, %g5 be,a,pt %icc, 1f nop ! sir ! WATCHDOG set badregs, %g1 ! Save the suspect regs stw %l0, [%g1+(4*0)] stw %l1, [%g1+(4*1)] stw %l2, [%g1+(4*2)] stw %l3, [%g1+(4*3)] stw %l4, [%g1+(4*4)] stw %l5, [%g1+(4*5)] stw %l6, [%g1+(4*6)] stw %l7, [%g1+(4*7)] stw %i0, [%g1+(4*8)+(4*0)] stw %i1, [%g1+(4*8)+(4*1)] stw %i2, [%g1+(4*8)+(4*2)] stw %i3, [%g1+(4*8)+(4*3)] stw %i4, [%g1+(4*8)+(4*4)] stw %i5, [%g1+(4*8)+(4*5)] stw %i6, [%g1+(4*8)+(4*6)] stw %i7, [%g1+(4*8)+(4*7)] save inc %g7 wrpr %g7, 0, %otherwin wrpr %g0, 0, %canrestore wrpr %g0, WSTATE_KERN, %wstate ! Need to know where our sp points set rft_wcnt, %g4 ! Restore nsaved before trapping sethi %hi(CPCB), %g6 LDPTR [%g6 + %lo(CPCB)], %g6 lduw [%g4], %g4 stb %g4, [%g6 + PCB_NSAVED] ta 1 sir .data badregs: .space 16*4 .text 1: #endif rdpr %tstate, %g1 rdpr %cwp, %g7 ! Find our cur window andn %g1, CWP, %g1 ! Clear it from %tstate wrpr %g1, %g7, %tstate ! Set %tstate with %cwp mov CTX_SECONDARY, %g1 ! Restore the user context GET_MMU_CONTEXTID %g4, %g1, %g3 mov CTX_PRIMARY, %g2 SET_MMU_CONTEXTID %g4, %g2, %g3 sethi %hi(KERNBASE), %g7 ! Should not be needed due to retry membar #Sync ! Should not be needed due to retry flush %g7 ! Should not be needed due to retry CLRTT #ifdef TRAPSTATS set _C_LABEL(rftudone), %g1 lduw [%g1], %g2 inc %g2 stw %g2, [%g1] #endif #ifdef DEBUG sethi %hi(CPCB), %g5 LDPTR [%g5 + %lo(CPCB)], %g5 ldub [%g5 + PCB_NSAVED], %g5 ! Any saved reg windows? tst %g5 tnz %icc, 1; nop ! Debugger if we still have saved windows! #endif wrpr %g0, 0, %pil ! Enable all interrupts retry ! exported end marker for kernel gdb .globl _C_LABEL(endtrapcode) _C_LABEL(endtrapcode): /* * Kernel entry point. * * The contract between bootloader and kernel is: * * %o0 OpenFirmware entry point, to keep Sun's updaters happy * %o1 Address of boot information vector (see bootinfo.h) * %o2 Length of the vector, in bytes * %o3 OpenFirmware entry point, to mimic Sun bootloader behavior * %o4 OpenFirmware, to meet earlier NetBSD kernels expectations */ .align 8 start: dostart: /* * Startup. * * The Sun FCODE bootloader is nice and loads us where we want * to be. We have a full set of mappings already set up for us. * * I think we end up having an entire 16M allocated to us. * * We enter with the prom entry vector in %o0, dvec in %o1, * and the bootops vector in %o2. * * All we need to do is: * * 1: Save the prom vector * * 2: Create a decent stack for ourselves * * 3: Install the permanent 4MB kernel mapping * * 4: Call the C language initialization code * */ /* * Set the psr into a known state: * Set supervisor mode, interrupt level >= 13, traps enabled */ wrpr %g0, 13, %pil wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate wr %g0, FPRS_FEF, %fprs ! Turn on FPU /* * Step 2: Set up a v8-like stack if we need to */ #ifdef _LP64 btst 1, %sp bnz,pt %icc, 0f nop add %sp, -BIAS, %sp #else btst 1, %sp bz,pt %icc, 0f nop add %sp, BIAS, %sp #endif 0: call _C_LABEL(bootstrap) clr %g4 ! Clear data segment pointer /* * Initialize the boot CPU. Basically: * * Locate the cpu_info structure for this CPU. * Establish a locked mapping for interrupt stack. * Switch to the initial stack. * Call the routine passed in in cpu_info->ci_spinup */ #ifdef NO_VCACHE #define SUN4U_TTE_DATABITS SUN4U_TTE_L|SUN4U_TTE_CP|SUN4U_TTE_P|SUN4U_TTE_W #else #define SUN4U_TTE_DATABITS SUN4U_TTE_L|SUN4U_TTE_CP|SUN4U_TTE_CV|SUN4U_TTE_P|SUN4U_TTE_W #endif ENTRY_NOPROFILE(cpu_initialize) /* for cosmetic reasons - nicer backtrace */ /* Cache the cputyp in %l6 for later use below */ sethi %hi(cputyp), %l6 ld [%l6 + %lo(cputyp)], %l6 /* * Step 5: is no more. */ /* * Step 6: hunt through cpus list and find the one that matches our cpuid */ call _C_LABEL(cpu_myid) ! Retrieve cpuid in %o0 mov %g0, %o0 sethi %hi(_C_LABEL(cpus)), %l1 LDPTR [%l1 + %lo(_C_LABEL(cpus))], %l1 0: ld [%l1 + CI_CPUID], %l3 ! Load CPUID cmp %l3, %o0 ! Does it match? bne,a,pt %icc, 0b ! no LDPTR [%l1 + CI_NEXT], %l1 ! Load next cpu_info pointer /* * Get pointer to our cpu_info struct */ mov %l1, %l7 ! save cpu_info pointer ldx [%l1 + CI_PADDR], %l1 ! Load the interrupt stack's PA #ifdef SUN4V cmp %l6, CPU_SUN4V bne,pt %icc, 3f nop /* sun4v */ call _C_LABEL(pmap_setup_intstack_sun4v) ! Call nice C function for mapping INTSTACK mov %l1, %o0 ba 4f nop 3: #endif /* sun4u */ sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 sllx %l2, 32, %l2 ! Shift it into place mov -1, %l3 ! Create a nice mask sllx %l3, 43, %l4 ! Mask off high bits or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits andn %l1, %l4, %l1 ! Mask the phys page number or %l2, %l1, %l1 ! Now take care of the high bits or %l1, SUN4U_TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 !! !! Now, map in the interrupt stack as context==0 !! set TLB_TAG_ACCESS, %l5 set INTSTACK, %l0 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it membar #Sync 4: !! Setup kernel stack (we rely on curlwp on this cpu !! being lwp0 here and its uarea is mapped special !! and already accessible here) flushw LDPTR [%l7 + CI_CPCB], %l0 ! load PCB/uarea pointer set 2*USPACE - TF_SIZE - CC64FSZ, %l1 add %l1, %l0, %l0 #ifdef _LP64 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned sub %l0, BIAS, %l0 ! and biased #endif mov %l0, %sp flushw #ifdef DEBUG set _C_LABEL(pmapdebug), %o1 ld [%o1], %o1 sethi %hi(0x40000), %o2 btst %o2, %o1 bz 0f set 1f, %o0 ! Debug printf call _C_LABEL(prom_printf) nop .data 1: .asciz "Setting trap base...\n" _ALIGN .text 0: #endif /* * Step 7: change the trap base register, and install our TSB pointers */ /* * install our TSB pointers */ #ifdef SUN4V cmp %l6, CPU_SUN4V bne,pt %icc, 5f nop /* sun4v */ LDPTR [%l7 + CI_TSB_DESC], %o0 call _C_LABEL(pmap_setup_tsb_sun4v) nop ba 1f nop 5: #endif /* sun4u */ sethi %hi(_C_LABEL(tsbsize)), %l2 sethi %hi(0x1fff), %l3 sethi %hi(TSB), %l4 LDPTR [%l7 + CI_TSB_DMMU], %l0 LDPTR [%l7 + CI_TSB_IMMU], %l1 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 or %l3, %lo(0x1fff), %l3 or %l4, %lo(TSB), %l4 andn %l0, %l3, %l0 ! Mask off size and split bits or %l0, %l2, %l0 ! Make a TSB pointer stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer andn %l1, %l3, %l1 ! Mask off size and split bits or %l1, %l2, %l1 ! Make a TSB pointer stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer membar #Sync set 1f, %l1 flush %l1 1: /* set trap table */ #ifdef SUN4V cmp %l6, CPU_SUN4V bne,pt %icc, 6f nop /* sun4v */ set _C_LABEL(trapbase_sun4v), %l1 GET_MMFSA %o1 call _C_LABEL(prom_set_trap_table_sun4v) ! Now we should be running 100% from our handlers mov %l1, %o0 ba 7f nop 6: #endif /* sun4u */ set _C_LABEL(trapbase), %l1 call _C_LABEL(prom_set_trap_table_sun4u) ! Now we should be running 100% from our handlers mov %l1, %o0 7: wrpr %l1, 0, %tba ! Make sure the PROM didn't foul up. /* * Switch to the kernel mode and run away. */ wrpr %g0, WSTATE_KERN, %wstate #ifdef DEBUG wrpr %g0, 1, %tl ! Debug -- start at tl==3 so we'll watchdog wrpr %g0, 0x1ff, %tt ! Debug -- clear out unused trap regs wrpr %g0, 0, %tpc wrpr %g0, 0, %tnpc wrpr %g0, 0, %tstate wrpr %g0, 0, %tl #endif #ifdef DEBUG set _C_LABEL(pmapdebug), %o1 ld [%o1], %o1 sethi %hi(0x40000), %o2 btst %o2, %o1 bz 0f LDPTR [%l7 + CI_SPINUP], %o1 set 1f, %o0 ! Debug printf call _C_LABEL(prom_printf) mov %sp, %o2 .data 1: .asciz "Calling startup routine %p with stack at %p...\n" _ALIGN .text 0: #endif /* * Call our startup routine. */ LDPTR [%l7 + CI_SPINUP], %o1 call %o1 ! Call routine clr %o0 ! our frame arg is ignored set 1f, %o0 ! Main should never come back here call _C_LABEL(panic) nop .data 1: .asciz "main() returned\n" _ALIGN .text .align 8 ENTRY(get_romtba) retl rdpr %tba, %o0 ENTRY(setcputyp) sethi %hi(cputyp), %o1 ! Trash %o1 assuming this is ok st %o0, [%o1 + %lo(cputyp)] retl nop #ifdef MULTIPROCESSOR /* * cpu_mp_startup is called with: * * %g2 = cpu_args */ ENTRY(cpu_mp_startup) mov 1, %o0 sllx %o0, 63, %o0 wr %o0, TICK_CMPR ! XXXXXXX clear and disable %tick_cmpr for now wrpr %g0, 0, %cleanwin wrpr %g0, 0, %tl ! Make sure we're not in NUCLEUS mode wrpr %g0, WSTATE_KERN, %wstate wrpr %g0, PSTATE_KERN, %pstate flushw /* Cache the cputyp in %l6 for later use below */ sethi %hi(cputyp), %l6 ld [%l6 + %lo(cputyp)], %l6 /* * Get pointer to our cpu_info struct */ ldx [%g2 + CBA_CPUINFO], %l1 ! Load the interrupt stack's PA #ifdef SUN4V cmp %l6, CPU_SUN4V bne,pt %icc, 3f nop /* sun4v */ sethi %hi(0x80000000), %l2 ! V=1|NFO=0|SW=0 sllx %l2, 32, %l2 ! Shift it into place mov -1, %l3 ! Create a nice mask sllx %l3, 56, %l4 ! Mask off high 8 bits or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits andn %l1, %l4, %l1 ! Mask the phys page number into RA or %l2, %l1, %l1 ! Now take care of the 8 high bits V|NFO|SW or %l1, 0x0741, %l2 ! And low 13 bits IE=0|E=0|CP=1|CV=1|P=1| ! X=0|W=1|SW=00|SZ=0001 /* * Now, map in the interrupt stack & cpu_info as context==0 */ set INTSTACK, %o0 ! vaddr clr %o1 ! reserved mov %l2, %o2 ! tte mov MAP_DTLB, %o3 ! flags mov FT_MMU_MAP_PERM_ADDR, %o5 ! hv fast trap function ta ST_FAST_TRAP cmp %o0, 0 be,pt %icc, 5f nop sir ! crash if mapping fails 5: /* * Set 0 as primary context XXX */ mov CTX_PRIMARY, %o0 SET_MMU_CONTEXTID_SUN4V %g0, %o0 ba 4f nop 3: #endif /* sun4u */ sethi %hi(0xa0000000), %l2 ! V=1|SZ=01|NFO=0|IE=0 sllx %l2, 32, %l2 ! Shift it into place mov -1, %l3 ! Create a nice mask sllx %l3, 43, %l4 ! Mask off high bits or %l4, 0xfff, %l4 ! We can just load this in 12 (of 13) bits andn %l1, %l4, %l1 ! Mask the phys page number or %l2, %l1, %l1 ! Now take care of the high bits or %l1, SUN4U_TTE_DATABITS, %l2 ! And low bits: L=1|CP=1|CV=?|E=0|P=1|W=1|G=0 /* * Now, map in the interrupt stack & cpu_info as context==0 */ set TLB_TAG_ACCESS, %l5 set INTSTACK, %l0 stxa %l0, [%l5] ASI_DMMU ! Make DMMU point to it stxa %l2, [%g0] ASI_DMMU_DATA_IN ! Store it /* * Set 0 as primary context XXX */ mov CTX_PRIMARY, %o0 SET_MMU_CONTEXTID_SUN4U %g0, %o0 4: membar #Sync /* * Temporarily use the interrupt stack */ #ifdef _LP64 set ((EINTSTACK - CC64FSZ - TF_SIZE)) & ~0x0f - BIAS, %sp #else set EINTSTACK - CC64FSZ - TF_SIZE, %sp #endif set 1, %fp clr %i7 #ifdef SUN4V cmp %l6, CPU_SUN4V bne,pt %icc, 2f nop /* sun4v */ /* * install our TSB pointers */ set CPUINFO_VA, %o0 LDPTR [%o0 + CI_TSB_DESC], %o0 call _C_LABEL(pmap_setup_tsb_sun4v) nop /* set trap table */ set _C_LABEL(trapbase_sun4v), %l1 GET_MMFSA %o1 call _C_LABEL(prom_set_trap_table_sun4v) mov %l1, %o0 ! Now we should be running 100% from our handlers ba 3f nop 2: #endif /* sun4u */ /* * install our TSB pointers */ sethi %hi(CPUINFO_VA+CI_TSB_DMMU), %l0 sethi %hi(CPUINFO_VA+CI_TSB_IMMU), %l1 sethi %hi(_C_LABEL(tsbsize)), %l2 sethi %hi(0x1fff), %l3 sethi %hi(TSB), %l4 LDPTR [%l0 + %lo(CPUINFO_VA+CI_TSB_DMMU)], %l0 LDPTR [%l1 + %lo(CPUINFO_VA+CI_TSB_IMMU)], %l1 ld [%l2 + %lo(_C_LABEL(tsbsize))], %l2 or %l3, %lo(0x1fff), %l3 or %l4, %lo(TSB), %l4 andn %l0, %l3, %l0 ! Mask off size and split bits or %l0, %l2, %l0 ! Make a TSB pointer stxa %l0, [%l4] ASI_DMMU ! Install data TSB pointer membar #Sync andn %l1, %l3, %l1 ! Mask off size and split bits or %l1, %l2, %l1 ! Make a TSB pointer stxa %l1, [%l4] ASI_IMMU ! Install instruction TSB pointer membar #Sync set 1f, %o0 flush %o0 1: /* set trap table */ set _C_LABEL(trapbase), %l1 call _C_LABEL(prom_set_trap_table_sun4u) mov %l1, %o0 3: wrpr %l1, 0, %tba ! Make sure the PROM didn't ! foul up. /* * Use this CPUs idlelewp's uarea stack */ sethi %hi(CPUINFO_VA+CI_IDLELWP), %l0 LDPTR [%l0 + %lo(CPUINFO_VA+CI_IDLELWP)], %l0 set USPACE - TF_SIZE - CC64FSZ, %l1 LDPTR [%l0 + L_PCB], %l0 add %l0, %l1, %l0 #ifdef _LP64 andn %l0, 0x0f, %l0 ! Needs to be 16-byte aligned sub %l0, BIAS, %l0 ! and biased #endif mov %l0, %sp flushw /* * Switch to the kernel mode and run away. */ wrpr %g0, 13, %pil wrpr %g0, PSTATE_INTR|PSTATE_PEF, %pstate wr %g0, FPRS_FEF, %fprs ! Turn on FPU call _C_LABEL(cpu_hatch) clr %g4 b _C_LABEL(idle_loop) clr %o0 NOTREACHED .globl cpu_mp_startup_end cpu_mp_startup_end: #endif /* * openfirmware(cell* param); * * OpenFirmware entry point * * If we're running in 32-bit mode we need to convert to a 64-bit stack * and 64-bit cells. The cells we'll allocate off the stack for simplicity. */ .align 8 ENTRY(openfirmware) sethi %hi(romp), %o4 andcc %sp, 1, %g0 bz,pt %icc, 1f LDPTR [%o4+%lo(romp)], %o4 ! v9 stack, just load the addr and callit save %sp, -CC64FSZ, %sp rdpr %pil, %i2 mov PIL_HIGH, %i3 cmp %i3, %i2 movle %icc, %i2, %i3 wrpr %g0, %i3, %pil mov %i0, %o0 mov %g1, %l1 mov %g2, %l2 mov %g3, %l3 mov %g4, %l4 mov %g5, %l5 mov %g6, %l6 mov %g7, %l7 rdpr %pstate, %l0 jmpl %i4, %o7 #if !defined(_LP64) wrpr %g0, PSTATE_PROM, %pstate #else wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate #endif wrpr %l0, %g0, %pstate mov %l1, %g1 mov %l2, %g2 mov %l3, %g3 mov %l4, %g4 mov %l5, %g5 mov %l6, %g6 mov %l7, %g7 wrpr %i2, 0, %pil ret restore %o0, %g0, %o0 1: ! v8 -- need to screw with stack & params #ifdef NOTDEF_DEBUG mov %o7, %o5 call globreg_check nop mov %o5, %o7 #endif save %sp, -CC64FSZ, %sp ! Get a new 64-bit stack frame add %sp, -BIAS, %sp rdpr %pstate, %l0 srl %sp, 0, %sp rdpr %pil, %i2 ! s = splx(level) mov %i0, %o0 mov PIL_HIGH, %i3 mov %g1, %l1 mov %g2, %l2 cmp %i3, %i2 mov %g3, %l3 mov %g4, %l4 mov %g5, %l5 movle %icc, %i2, %i3 mov %g6, %l6 mov %g7, %l7 wrpr %i3, %g0, %pil jmpl %i4, %o7 ! Enable 64-bit addresses for the prom #if defined(_LP64) wrpr %g0, PSTATE_PROM, %pstate #else wrpr %g0, PSTATE_PROM|PSTATE_IE, %pstate #endif wrpr %l0, 0, %pstate wrpr %i2, 0, %pil mov %l1, %g1 mov %l2, %g2 mov %l3, %g3 mov %l4, %g4 mov %l5, %g5 mov %l6, %g6 mov %l7, %g7 ret restore %o0, %g0, %o0 /* * void ofw_exit(cell_t args[]) */ ENTRY(openfirmware_exit) STACKFRAME(-CC64FSZ) flushw ! Flush register windows wrpr %g0, PIL_HIGH, %pil ! Disable interrupts sethi %hi(romtba), %l5 LDPTR [%l5 + %lo(romtba)], %l5 wrpr %l5, 0, %tba ! restore the ofw trap table /* Arrange locked kernel stack as PROM stack */ set EINTSTACK - CC64FSZ, %l5 andn %l5, 0x0f, %l5 ! Needs to be 16-byte aligned sub %l5, BIAS, %l5 ! and biased mov %l5, %sp flushw sethi %hi(romp), %l6 LDPTR [%l6 + %lo(romp)], %l6 mov CTX_PRIMARY, %l3 ! set context 0 stxa %g0, [%l3] ASI_DMMU membar #Sync wrpr %g0, PSTATE_PROM, %pstate ! Disable interrupts ! and enable 64-bit addresses wrpr %g0, 0, %tl ! force trap level 0 call %l6 mov %i0, %o0 NOTREACHED /* * sp_tlb_flush_pte_us(vaddr_t va, int ctx) * sp_tlb_flush_pte_usiii(vaddr_t va, int ctx) * * Flush tte from both IMMU and DMMU. * * This uses %o0-%o5 */ .align 8 ENTRY(sp_tlb_flush_pte_us) #ifdef DEBUG set pmapdebug, %o3 lduw [%o3], %o3 ! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 btst 0x0020, %o3 bz,pt %icc, 2f nop save %sp, -CC64FSZ, %sp set 1f, %o0 mov %i1, %o1 andn %i0, 0xfff, %o3 or %o3, 0x010, %o3 call _C_LABEL(printf) mov %i0, %o2 restore .data 1: .asciz "sp_tlb_flush_pte_us: demap ctx=%x va=%08x res=%x\n" _ALIGN .text 2: #endif #ifdef MULTIPROCESSOR rdpr %pstate, %o3 andn %o3, PSTATE_IE, %o4 ! disable interrupts wrpr %o4, 0, %pstate #endif srlx %o0, PG_SHIFT4U, %o0 ! drop unused va bits mov CTX_SECONDARY, %o2 sllx %o0, PG_SHIFT4U, %o0 ldxa [%o2] ASI_DMMU, %o5 ! Save secondary context sethi %hi(KERNBASE), %o4 membar #LoadStore stxa %o1, [%o2] ASI_DMMU ! Insert context to demap membar #Sync or %o0, DEMAP_PAGE_SECONDARY, %o0 ! Demap page from secondary context only stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs #ifdef TLB_FLUSH_LOWVA srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap #endif flush %o4 stxa %o5, [%o2] ASI_DMMU ! Restore secondary context membar #Sync retl #ifdef MULTIPROCESSOR wrpr %o3, %pstate ! restore interrupts #else nop #endif ENTRY(sp_tlb_flush_pte_usiii) #ifdef DEBUG set pmapdebug, %o3 lduw [%o3], %o3 ! movrz %o1, -1, %o3 ! Print on either pmapdebug & PDB_DEMAP or ctx == 0 btst 0x0020, %o3 bz,pt %icc, 2f nop save %sp, -CC64FSZ, %sp set 1f, %o0 mov %i1, %o1 andn %i0, 0xfff, %o3 or %o3, 0x010, %o3 call _C_LABEL(printf) mov %i0, %o2 restore .data 1: .asciz "sp_tlb_flush_pte_usiii: demap ctx=%x va=%08x res=%x\n" _ALIGN .text 2: #endif ! %o0 = VA [in] ! %o1 = ctx value [in] / KERNBASE ! %o2 = CTX_PRIMARY ! %o3 = saved %tl ! %o4 = saved %pstate ! %o5 = saved primary ctx ! Need this for UP as well rdpr %pstate, %o4 andn %o4, PSTATE_IE, %o3 ! disable interrupts wrpr %o3, 0, %pstate !! !! Cheetahs do not support flushing the IMMU from secondary context !! rdpr %tl, %o3 mov CTX_PRIMARY, %o2 brnz,pt %o3, 1f andn %o0, 0xfff, %o0 ! drop unused va bits wrpr %g0, 1, %tl ! Make sure we're NUCLEUS 1: ldxa [%o2] ASI_DMMU, %o5 ! Save primary context membar #LoadStore stxa %o1, [%o2] ASI_DMMU ! Insert context to demap sethi %hi(KERNBASE), %o1 membar #Sync or %o0, DEMAP_PAGE_PRIMARY, %o0 stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap membar #Sync stxa %o0, [%o0] ASI_IMMU_DEMAP ! to both TLBs membar #Sync #ifdef TLB_FLUSH_LOWVA srl %o0, 0, %o0 ! and make sure it's both 32- and 64-bit entries stxa %o0, [%o0] ASI_DMMU_DEMAP ! Do the demap membar #Sync stxa %o0, [%o0] ASI_IMMU_DEMAP ! Do the demap membar #Sync #endif flush %o1 stxa %o5, [%o2] ASI_DMMU ! Restore primary context membar #Sync brnz,pt %o3, 1f flush %o1 wrpr %g0, %o3, %tl ! Return to kernel mode. 1: retl wrpr %o4, %pstate ! restore interrupts /* * sp_tlb_flush_all_us(void) * sp_tlb_flush_all_usiii(void) * * Flush all user TLB entries from both IMMU and DMMU. * We have both UltraSPARC I+II, and UltraSPARC >=III versions. */ .align 8 ENTRY(sp_tlb_flush_all_us) rdpr %pstate, %o3 andn %o3, PSTATE_IE, %o4 ! disable interrupts wrpr %o4, 0, %pstate set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 set CTX_SECONDARY, %o4 ldxa [%o4] ASI_DMMU, %o4 ! save secondary context set CTX_MASK, %o5 membar #Sync ! %o0 = loop counter ! %o1 = ctx value ! %o2 = TLB tag value ! %o3 = saved %pstate ! %o4 = saved primary ctx ! %o5 = CTX_MASK ! %xx = saved %tl 0: ldxa [%o0] ASI_DMMU_TLB_TAG, %o2 ! fetch the TLB tag andcc %o2, %o5, %o1 ! context 0? bz,pt %xcc, 1f ! if so, skip mov CTX_SECONDARY, %o2 stxa %o1, [%o2] ASI_DMMU ! set the context set DEMAP_CTX_SECONDARY, %o2 membar #Sync stxa %o2, [%o2] ASI_DMMU_DEMAP ! do the demap membar #Sync 1: dec 8, %o0 brgz,pt %o0, 0b ! loop over all entries nop /* * now do the IMMU */ set ((TLB_SIZE_SPITFIRE-1) * 8), %o0 0: ldxa [%o0] ASI_IMMU_TLB_TAG, %o2 ! fetch the TLB tag andcc %o2, %o5, %o1 ! context 0? bz,pt %xcc, 1f ! if so, skip mov CTX_SECONDARY, %o2 stxa %o1, [%o2] ASI_DMMU ! set the context set DEMAP_CTX_SECONDARY, %o2 membar #Sync stxa %o2, [%o2] ASI_IMMU_DEMAP ! do the demap membar #Sync 1: dec 8, %o0 brgz,pt %o0, 0b ! loop over all entries nop set CTX_SECONDARY, %o2 stxa %o4, [%o2] ASI_DMMU ! restore secondary ctx sethi %hi(KERNBASE), %o4 membar #Sync flush %o4 retl wrpr %o3, %pstate .align 8 ENTRY(sp_tlb_flush_all_usiii) rdpr %tl, %o5 brnz,pt %o5, 1f set DEMAP_ALL, %o2 wrpr 1, %tl 1: rdpr %pstate, %o3 andn %o3, PSTATE_IE, %o4 ! disable interrupts wrpr %o4, 0, %pstate stxa %o2, [%o2] ASI_IMMU_DEMAP membar #Sync stxa %o2, [%o2] ASI_DMMU_DEMAP sethi %hi(KERNBASE), %o4 membar #Sync flush %o4 wrpr %o5, %tl retl wrpr %o3, %pstate /* * sp_blast_dcache(int dcache_size, int dcache_line_size) * sp_blast_dcache_disabled(int dcache_size, int dcache_line_size) * * Clear out all of D$ regardless of contents. The latter one also * disables the D$ while doing so. */ .align 8 ENTRY(sp_blast_dcache) /* * We turn off interrupts for the duration to prevent RED exceptions. */ #ifdef PROF save %sp, -CC64FSZ, %sp #endif rdpr %pstate, %o3 sub %o0, %o1, %o0 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit wrpr %o4, 0, %pstate 1: stxa %g0, [%o0] ASI_DCACHE_TAG membar #Sync brnz,pt %o0, 1b sub %o0, %o1, %o0 sethi %hi(KERNBASE), %o2 flush %o2 membar #Sync #ifdef PROF wrpr %o3, %pstate ret restore #else retl wrpr %o3, %pstate #endif .align 8 ENTRY(sp_blast_dcache_disabled) /* * We turn off interrupts for the duration to prevent RED exceptions. */ #ifdef PROF save %sp, -CC64FSZ, %sp #endif rdpr %pstate, %o3 sub %o0, %o1, %o0 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit wrpr %o4, 0, %pstate ldxa [%g0] ASI_MCCR, %o5 andn %o5, MCCR_DCACHE_EN, %o4 ! Turn off the D$ stxa %o4, [%g0] ASI_MCCR flush %g0 1: stxa %g0, [%o0] ASI_DCACHE_TAG membar #Sync brnz,pt %o0, 1b sub %o0, %o1, %o0 sethi %hi(KERNBASE), %o2 flush %o2 membar #Sync stxa %o5, [%g0] ASI_MCCR ! Restore the D$ flush %g0 #ifdef PROF wrpr %o3, %pstate ret restore #else retl wrpr %o3, %pstate #endif #ifdef MULTIPROCESSOR /* * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size) * * Clear out all of D$ regardless of contents * * On entry: * %g2 = dcache_size * %g3 = dcache_line_size */ .align 8 ENTRY(sparc64_ipi_blast_dcache) sub %g2, %g3, %g2 1: stxa %g0, [%g2] ASI_DCACHE_TAG membar #Sync brnz,pt %g2, 1b sub %g2, %g3, %g2 sethi %hi(KERNBASE), %g5 flush %g5 membar #Sync ba,a ret_from_intr_vector nop #endif /* MULTIPROCESSOR */ /* * blast_icache_us() * blast_icache_usiii() * * Clear out all of I$ regardless of contents * Does not modify %o0 * * We turn off interrupts for the duration to prevent RED exceptions. * For the Cheetah version, we also have to to turn off the I$ during this as * ASI_ICACHE_TAG accesses interfere with coherency. */ .align 8 ENTRY(blast_icache_us) rdpr %pstate, %o3 sethi %hi(icache_size), %o1 ld [%o1 + %lo(icache_size)], %o1 sethi %hi(icache_line_size), %o2 ld [%o2 + %lo(icache_line_size)], %o2 sub %o1, %o2, %o1 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit wrpr %o4, 0, %pstate 1: stxa %g0, [%o1] ASI_ICACHE_TAG brnz,pt %o1, 1b sub %o1, %o2, %o1 sethi %hi(KERNBASE), %o5 flush %o5 membar #Sync retl wrpr %o3, %pstate .align 8 ENTRY(blast_icache_usiii) rdpr %pstate, %o3 sethi %hi(icache_size), %o1 ld [%o1 + %lo(icache_size)], %o1 sethi %hi(icache_line_size), %o2 ld [%o2 + %lo(icache_line_size)], %o2 sub %o1, %o2, %o1 andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit wrpr %o4, 0, %pstate ldxa [%g0] ASI_MCCR, %o5 andn %o5, MCCR_ICACHE_EN, %o4 ! Turn off the I$ stxa %o4, [%g0] ASI_MCCR flush %g0 1: stxa %g0, [%o1] ASI_ICACHE_TAG membar #Sync brnz,pt %o1, 1b sub %o1, %o2, %o1 stxa %o5, [%g0] ASI_MCCR ! Restore the I$ flush %g0 retl wrpr %o3, %pstate /* * dcache_flush_page_us(paddr_t pa) * dcache_flush_page_usiii(paddr_t pa) * * Clear one page from D$. * */ .align 8 ENTRY(dcache_flush_page_us) #ifndef _LP64 COMBINE(%o0, %o1, %o0) #endif mov -1, %o1 ! Generate mask for tag: bits [29..2] srlx %o0, 13-2, %o2 ! Tag is PA bits <40:13> in bits <29:2> clr %o4 srl %o1, 2, %o1 ! Now we have bits <29:0> set set (2*NBPG), %o5 ba,pt %icc, 1f andn %o1, 3, %o1 ! Now we have bits <29:2> set .align 8 1: ldxa [%o4] ASI_DCACHE_TAG, %o3 mov %o4, %o0 deccc 32, %o5 bl,pn %icc, 2f inc 32, %o4 xor %o3, %o2, %o3 andcc %o3, %o1, %g0 bne,pt %xcc, 1b membar #LoadStore stxa %g0, [%o0] ASI_DCACHE_TAG ba,pt %icc, 1b membar #StoreLoad 2: sethi %hi(KERNBASE), %o5 flush %o5 retl membar #Sync .align 8 ENTRY(dcache_flush_page_usiii) #ifndef _LP64 COMBINE(%o0, %o1, %o0) #endif set NBPG, %o1 sethi %hi(dcache_line_size), %o2 add %o0, %o1, %o1 ! end address ld [%o2 + %lo(dcache_line_size)], %o2 1: stxa %g0, [%o0] ASI_DCACHE_INVALIDATE add %o0, %o2, %o0 cmp %o0, %o1 bl,pt %xcc, 1b nop sethi %hi(KERNBASE), %o5 flush %o5 retl membar #Sync /* * cache_flush_phys_us(paddr_t, psize_t, int); * cache_flush_phys_usiii(paddr_t, psize_t, int); * * Clear a set of paddrs from the D$, I$ and if param3 is * non-zero, E$. (E$ is not supported yet). */ .align 8 ENTRY(cache_flush_phys_us) #ifndef _LP64 COMBINE(%o0, %o1, %o0) COMBINE(%o2, %o3, %o1) mov %o4, %o2 #endif #ifdef DEBUG tst %o2 ! Want to clear E$? tnz 1 ! Error! #endif add %o0, %o1, %o1 ! End PA dec %o1 !! !! Both D$ and I$ tags match pa bits 42-13, but !! they are shifted different amounts. So we'll !! generate a mask for bits 40-13. !! mov -1, %o2 ! Generate mask for tag: bits [40..13] srl %o2, 5, %o2 ! 32-5 = [27..0] sllx %o2, 13, %o2 ! 27+13 = [40..13] and %o2, %o0, %o0 ! Mask away uninteresting bits and %o2, %o1, %o1 ! (probably not necessary) set (2*NBPG), %o5 clr %o4 1: ldxa [%o4] ASI_DCACHE_TAG, %o3 sllx %o3, 40-29, %o3 ! Shift D$ tag into place and %o3, %o2, %o3 ! Mask out trash cmp %o0, %o3 blt,pt %xcc, 2f ! Too low cmp %o1, %o3 bgt,pt %xcc, 2f ! Too high nop membar #LoadStore stxa %g0, [%o4] ASI_DCACHE_TAG ! Just right membar #Sync 2: ldda [%o4] ASI_ICACHE_TAG, %g0 ! Tag goes in %g1 sllx %g1, 40-35, %g1 ! Shift I$ tag into place and %g1, %o2, %g1 ! Mask out trash cmp %o0, %g1 blt,pt %xcc, 3f cmp %o1, %g1 bgt,pt %xcc, 3f nop stxa %g0, [%o4] ASI_ICACHE_TAG 3: membar #StoreLoad dec 32, %o5 brgz,pt %o5, 1b inc 32, %o4 sethi %hi(KERNBASE), %o5 flush %o5 retl membar #Sync .align 8 ENTRY(cache_flush_phys_usiii) #ifndef _LP64 COMBINE(%o0, %o1, %o0) COMBINE(%o2, %o3, %o1) mov %o4, %o2 #endif #ifdef DEBUG tst %o2 ! Want to clear E$? tnz 1 ! Error! #endif add %o0, %o1, %o1 ! End PA sethi %hi(dcache_line_size), %o3 ld [%o3 + %lo(dcache_line_size)], %o3 sethi %hi(KERNBASE), %o5 1: stxa %g0, [%o0] ASI_DCACHE_INVALIDATE add %o0, %o3, %o0 cmp %o0, %o1 bl,pt %xcc, 1b nop /* don't need to flush the I$ on cheetah */ flush %o5 retl membar #Sync #ifdef COMPAT_16 #ifdef _LP64 /* * XXXXX Still needs lotsa cleanup after sendsig is complete and offsets are known * * The following code is copied to the top of the user stack when each * process is exec'ed, and signals are `trampolined' off it. * * When this code is run, the stack looks like: * [%sp] 128 bytes to which registers can be dumped * [%sp + 128] signal number (goes in %o0) * [%sp + 128 + 4] signal code (goes in %o1) * [%sp + 128 + 8] first word of saved state (sigcontext) * . * . * . * [%sp + NNN] last word of saved state * (followed by previous stack contents or top of signal stack). * The address of the function to call is in %g1; the old %g1 and %o0 * have already been saved in the sigcontext. We are running in a clean * window, all previous windows now being saved to the stack. * * Note that [%sp + 128 + 8] == %sp + 128 + 16. The copy at %sp+128+8 * will eventually be removed, with a hole left in its place, if things * work out. */ ENTRY_NOPROFILE(sigcode) /* * XXX the `save' and `restore' below are unnecessary: should * replace with simple arithmetic on %sp * * Make room on the stack for 64 %f registers + %fsr. This comes * out to 64*4+8 or 264 bytes, but this must be aligned to a multiple * of 64, or 320 bytes. */ save %sp, -CC64FSZ - 320, %sp mov %g2, %l2 ! save globals in %l registers mov %g3, %l3 mov %g4, %l4 mov %g5, %l5 mov %g6, %l6 mov %g7, %l7 /* * Saving the fpu registers is expensive, so do it iff it is * enabled and dirty. */ rd %fprs, %l0 btst FPRS_DL|FPRS_DU, %l0 ! All clean? bz,pt %icc, 2f btst FPRS_DL, %l0 ! test dl bz,pt %icc, 1f btst FPRS_DU, %l0 ! test du ! fpu is enabled, oh well stx %fsr, [%sp + CC64FSZ + BIAS + 0] add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can andn %l0, BLOCK_ALIGN, %l0 ! do a block store stda %f0, [%l0] ASI_BLK_P inc BLOCK_SIZE, %l0 stda %f16, [%l0] ASI_BLK_P 1: bz,pt %icc, 2f add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can andn %l0, BLOCK_ALIGN, %l0 ! do a block store add %l0, 2*BLOCK_SIZE, %l0 ! and skip what we already stored stda %f32, [%l0] ASI_BLK_P inc BLOCK_SIZE, %l0 stda %f48, [%l0] ASI_BLK_P 2: membar #Sync rd %fprs, %l0 ! reload fprs copy, for checking after rd %y, %l1 ! in any case, save %y lduw [%fp + BIAS + 128], %o0 ! sig lduw [%fp + BIAS + 128 + 4], %o1 ! code call %g1 ! (*sa->sa_handler)(sig,code,scp) add %fp, BIAS + 128 + 8, %o2 ! scp wr %l1, %g0, %y ! in any case, restore %y /* * Now that the handler has returned, re-establish all the state * we just saved above, then do a sigreturn. */ btst FPRS_DL|FPRS_DU, %l0 ! All clean? bz,pt %icc, 2f btst FPRS_DL, %l0 ! test dl bz,pt %icc, 1f btst FPRS_DU, %l0 ! test du ldx [%sp + CC64FSZ + BIAS + 0], %fsr add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can andn %l0, BLOCK_ALIGN, %l0 ! do a block load ldda [%l0] ASI_BLK_P, %f0 inc BLOCK_SIZE, %l0 ldda [%l0] ASI_BLK_P, %f16 1: bz,pt %icc, 2f nop add %sp, BIAS+CC64FSZ+BLOCK_SIZE, %l0 ! Generate a pointer so we can andn %l0, BLOCK_ALIGN, %l0 ! do a block load inc 2*BLOCK_SIZE, %l0 ! and skip what we already loaded ldda [%l0] ASI_BLK_P, %f32 inc BLOCK_SIZE, %l0 ldda [%l0] ASI_BLK_P, %f48 2: mov %l2, %g2 mov %l3, %g3 mov %l4, %g4 mov %l5, %g5 mov %l6, %g6 mov %l7, %g7 membar #Sync restore %g0, SYS_compat_16___sigreturn14, %g1 ! get registers back & set syscall # add %sp, BIAS + 128 + 8, %o0! compute scp ! andn %o0, 0x0f, %o0 t ST_SYSCALL ! sigreturn(scp) ! sigreturn does not return unless it fails mov SYS_exit, %g1 ! exit(errno) t ST_SYSCALL /* NOTREACHED */ .globl _C_LABEL(esigcode) _C_LABEL(esigcode): #endif #if !defined(_LP64) #define SIGCODE_NAME sigcode #define ESIGCODE_NAME esigcode #define SIGRETURN_NAME SYS_compat_16___sigreturn14 #define EXIT_NAME SYS_exit #include "sigcode32.s" #endif #endif /* * getfp() - get stack frame pointer */ ENTRY(getfp) retl mov %fp, %o0 /* * Call optional cpu_idle handler if provided */ ENTRY(cpu_idle) set CPUINFO_VA, %o0 LDPTR [%o0 + CI_IDLESPIN], %o1 tst %o1 bz 1f nop jmp %o1 nop 1: retl nop /* * cpu_switchto() switches to an lwp to run and runs it, saving the * current one away. * * struct lwp * cpu_switchto(struct lwp *current, struct lwp *next) * Switch to the specified next LWP * Arguments: * i0 'struct lwp *' of the current LWP * i1 'struct lwp *' of the LWP to switch to * i2 'bool' of the flag returning to a softint LWP or not * Returns: * the old lwp switched away from */ ENTRY(cpu_switchto) save %sp, -CC64FSZ, %sp /* * REGISTER USAGE AT THIS POINT: * %l1 = newpcb * %l3 = new trapframe * %l4 = new l->l_proc * %l5 = pcb of oldlwp * %l6 = %hi(CPCB) * %l7 = %hi(CURLWP) * %i0 = oldlwp * %i1 = lwp * %i2 = returning * %o0 = tmp 1 * %o1 = tmp 2 * %o2 = tmp 3 * %o3 = tmp 4 */ flushw ! save all register windows except this one wrpr %g0, PSTATE_KERN, %pstate ! make sure we're on normal globals ! with traps turned off sethi %hi(CPCB), %l6 rdpr %pstate, %o1 ! oldpstate = %pstate; LDPTR [%i0 + L_PCB], %l5 stx %i7, [%l5 + PCB_PC] stx %i6, [%l5 + PCB_SP] sth %o1, [%l5 + PCB_PSTATE] rdpr %cwp, %o2 ! Useless stb %o2, [%l5 + PCB_CWP] sethi %hi(CURLWP), %l7 LDPTR [%i1 + L_PCB], %l1 ! newpcb = l->l_pcb; /* * Load the new lwp. To load, we must change stacks and * alter cpcb and the window control registers, hence we must * keep interrupts disabled. * * Issue barriers to coordinate mutex_exit on this CPU with * mutex_vector_enter on another CPU. * * 1. Any prior mutex_exit by oldlwp must be visible to other * CPUs before we set ci_curlwp := newlwp on this one, * requiring a store-before-store barrier. * * 2. ci_curlwp := newlwp must be visible on all other CPUs * before any subsequent mutex_exit by newlwp can even test * whether there might be waiters, requiring a * store-before-load barrier. * * See kern_mutex.c for details -- this is necessary for * adaptive mutexes to detect whether the lwp is on the CPU in * order to safely block without requiring atomic r/m/w in * mutex_exit. */ membar #StoreStore STPTR %i1, [%l7 + %lo(CURLWP)] ! curlwp = l; membar #StoreLoad STPTR %l1, [%l6 + %lo(CPCB)] ! cpcb = newpcb; ldx [%l1 + PCB_SP], %i6 ldx [%l1 + PCB_PC], %i7 wrpr %g0, 0, %otherwin ! These two insns should be redundant wrpr %g0, 0, %canrestore GET_MAXCWP %o3 wrpr %g0, %o3, %cleanwin dec 1, %o3 ! CANSAVE + CANRESTORE + OTHERWIN = MAXCWP - 1 /* Skip the rest if returning to a interrupted LWP. */ brnz,pn %i2, Lsw_noras wrpr %o3, %cansave /* finally, enable traps */ wrpr %g0, PSTATE_INTR, %pstate !flushw !membar #Sync /* * Check for restartable atomic sequences (RAS) */ LDPTR [%i1 + L_PROC], %l4 ! now %l4 points to p mov %l4, %o0 ! p is first arg to ras_lookup LDPTR [%o0 + P_RASLIST], %o1 ! any RAS in p? brz,pt %o1, Lsw_noras ! no, skip RAS check LDPTR [%i1 + L_TF], %l3 ! pointer to trap frame call _C_LABEL(ras_lookup) ldx [%l3 + TF_PC], %o1 cmp %o0, -1 be,pt CCCR, Lsw_noras add %o0, 4, %o1 stx %o0, [%l3 + TF_PC] ! store rewound %pc stx %o1, [%l3 + TF_NPC] ! and %npc Lsw_noras: /* * We are resuming the process that was running at the * call to switch(). Just set psr ipl and return. */ ! wrpr %g0, 0, %cleanwin ! DEBUG clr %g4 ! This needs to point to the base of the data segment wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Restore default ASI !wrpr %g0, PSTATE_INTR, %pstate ret restore %i0, %g0, %o0 ! return old curlwp #ifdef __HAVE_FAST_SOFTINTS /* * Switch to the LWP assigned to handle interrupts from the given * source. We borrow the VM context from the interrupted LWP. * * int softint_fastintr(void *l) * * Arguments: * i0 softint lwp */ ENTRY(softint_fastintr) save %sp, -CC64FSZ, %sp set CPUINFO_VA, %l0 ! l0 = curcpu() rdpr %pil, %l7 ! l7 = splhigh() wrpr %g0, PIL_HIGH, %pil LDPTR [%l0 + CI_EINTSTACK], %l6 ! l6 = ci_eintstack add %sp, -CC64FSZ, %l2 ! ci_eintstack = sp - CC64FSZ STPTR %l2, [%l0 + CI_EINTSTACK] ! save intstack for nested intr mov %i0, %o0 ! o0/i0 = softint lwp mov %l7, %o1 ! o1/i1 = ipl save %sp, -CC64FSZ, %sp ! make one more register window flushw ! and save all sethi %hi(CURLWP), %l7 sethi %hi(CPCB), %l6 LDPTR [%l7 + %lo(CURLWP)], %l0 ! l0 = interrupted lwp (curlwp) /* save interrupted lwp/pcb info */ sethi %hi(softint_fastintr_ret - 8), %o0 ! trampoline function LDPTR [%l0 + L_PCB], %l5 ! l5 = interrupted pcb or %o0, %lo(softint_fastintr_ret - 8), %o0 stx %i6, [%l5 + PCB_SP] stx %o0, [%l5 + PCB_PC] rdpr %pstate, %o1 rdpr %cwp, %o2 sth %o1, [%l5 + PCB_PSTATE] stb %o2, [%l5 + PCB_CWP] /* switch to softint lwp */ sethi %hi(USPACE - TF_SIZE - CC64FSZ - STKB), %o3 LDPTR [%i0 + L_PCB], %l1 ! l1 = softint pcb or %o3, %lo(USPACE - TF_SIZE - CC64FSZ - STKB), %o3 membar #StoreStore /* for mutex_enter; see cpu_switchto */ STPTR %i0, [%l7 + %lo(CURLWP)] /* * No need for barrier after ci->ci_curlwp = softlwp -- when we * enter a softint lwp, it can't be holding any mutexes, so it * can't release any until after it has acquired them, so we * need not participate in the protocol with mutex_vector_enter * barriers here. */ add %l1, %o3, %i6 STPTR %l1, [%l6 + %lo(CPCB)] stx %i6, [%l1 + PCB_SP] add %i6, -CC64FSZ, %sp ! new stack /* now switched, then invoke MI dispatcher */ mov %i1, %o1 call _C_LABEL(softint_dispatch) mov %l0, %o0 /* switch back to interrupted lwp */ ldx [%l5 + PCB_SP], %i6 membar #StoreStore /* for mutex_enter; see cpu_switchto */ STPTR %l0, [%l7 + %lo(CURLWP)] membar #StoreLoad /* for mutex_enter; see cpu_switchto */ STPTR %l5, [%l6 + %lo(CPCB)] restore ! rewind register window STPTR %l6, [%l0 + CI_EINTSTACK] ! restore ci_eintstack wrpr %g0, %l7, %pil ! restore ipl ret restore %g0, 1, %o0 /* * Trampoline function that gets returned to by cpu_switchto() when * an interrupt handler blocks. * * Arguments: * o0 old lwp from cpu_switchto() * * from softint_fastintr(): * l0 CPUINFO_VA * l6 saved ci_eintstack * l7 saved ipl */ softint_fastintr_ret: /* re-adjust after mi_switch() */ ld [%l0 + CI_MTX_COUNT], %o1 inc %o1 ! ci_mtx_count++ st %o1, [%l0 + CI_MTX_COUNT] STPTR %l6, [%l0 + CI_EINTSTACK] ! restore ci_eintstack wrpr %g0, %l7, %pil ! restore ipl ret restore %g0, 1, %o0 #endif /* __HAVE_FAST_SOFTINTS */ /* * Snapshot the current process so that stack frames are up to date. * Only used just before a crash dump. */ ENTRY(snapshot) rdpr %pstate, %o1 ! save psr stx %o7, [%o0 + PCB_PC] ! save pc stx %o6, [%o0 + PCB_SP] ! save sp rdpr %pil, %o2 sth %o1, [%o0 + PCB_PSTATE] rdpr %cwp, %o3 stb %o2, [%o0 + PCB_PIL] stb %o3, [%o0 + PCB_CWP] flushw save %sp, -CC64FSZ, %sp flushw ret restore /* * cpu_lwp_fork() arranges for lwp_trampoline() to run when the * nascent lwp is selected by switch(). * * The switch frame will contain pointer to struct lwp of this lwp in * %l2, a pointer to the function to call in %l0, and an argument to * pass to it in %l1 (we abuse the callee-saved registers). * * We enter lwp_trampoline as if we are "returning" from * cpu_switchto(), so %o0 contains previous lwp (the one we are * switching from) that we pass to lwp_startup(). * * If the function *(%l0) returns, we arrange for an immediate return * to user mode. This happens in two known cases: after execve(2) of * init, and when returning a child to user mode after a fork(2). * * If were setting up a kernel thread, the function *(%l0) will not * return. */ ENTRY(lwp_trampoline) /* * Note: cpu_lwp_fork() has set up a stack frame for us to run * in, so we can call other functions from here without using * `save ... restore'. */ ! newlwp in %l2, oldlwp in %o0 call lwp_startup mov %l2, %o1 call %l0 ! re-use current frame mov %l1, %o0 /* * Here we finish up as in syscall, but simplified. */ b return_from_trap nop /* * pmap_zero_page_phys(pa) * * Zero one page physically addressed * * Block load/store ASIs do not exist for physical addresses, * so we won't use them. * * We will execute a flush at the end to sync the I$. * * This version expects to have the dcache_flush_page_all(pa) * to have been called before calling into here. */ ENTRY(pmap_zero_page_phys) #ifndef _LP64 COMBINE(%o0, %o1, %o0) #endif #ifdef DEBUG set pmapdebug, %o4 ld [%o4], %o4 btst 0x80, %o4 ! PDB_COPY bz,pt %icc, 3f nop save %sp, -CC64FSZ, %sp set 2f, %o0 call printf mov %i0, %o1 ! ta 1; nop restore .data 2: .asciz "pmap_zero_page(%p)\n" _ALIGN .text 3: #endif set NBPG, %o2 ! Loop count wr %g0, ASI_PHYS_CACHED, %asi 1: /* Unroll the loop 8 times */ stxa %g0, [%o0 + 0x00] %asi deccc 0x40, %o2 stxa %g0, [%o0 + 0x08] %asi stxa %g0, [%o0 + 0x10] %asi stxa %g0, [%o0 + 0x18] %asi stxa %g0, [%o0 + 0x20] %asi stxa %g0, [%o0 + 0x28] %asi stxa %g0, [%o0 + 0x30] %asi stxa %g0, [%o0 + 0x38] %asi bg,pt %icc, 1b inc 0x40, %o0 sethi %hi(KERNBASE), %o3 flush %o3 retl wr %g0, ASI_PRIMARY_NOFAULT, %asi ! Make C code happy /* * pmap_copy_page_phys(paddr_t src, paddr_t dst) * * Copy one page physically addressed * We need to use a global reg for ldxa/stxa * so the top 32-bits cannot be lost if we take * a trap and need to save our stack frame to a * 32-bit stack. We will unroll the loop by 4 to * improve performance. * * This version expects to have the dcache_flush_page_all(pa) * to have been called before calling into here. * */ ENTRY(pmap_copy_page_phys) #ifndef _LP64 COMBINE(%o0, %o1, %o0) COMBINE(%o2, %o3, %o1) #endif #ifdef DEBUG set pmapdebug, %o4 ld [%o4], %o4 btst 0x80, %o4 ! PDB_COPY bz,pt %icc, 3f nop save %sp, -CC64FSZ, %sp mov %i0, %o1 set 2f, %o0 call printf mov %i1, %o2 ! ta 1; nop restore .data 2: .asciz "pmap_copy_page(%p,%p)\n" _ALIGN .text 3: #endif #if 1 set NBPG, %o2 wr %g0, ASI_PHYS_CACHED, %asi 1: ldxa [%o0 + 0x00] %asi, %g1 ldxa [%o0 + 0x08] %asi, %o3 ldxa [%o0 + 0x10] %asi, %o4 ldxa [%o0 + 0x18] %asi, %o5 inc 0x20, %o0 deccc 0x20, %o2 stxa %g1, [%o1 + 0x00] %asi stxa %o3, [%o1 + 0x08] %asi stxa %o4, [%o1 + 0x10] %asi stxa %o5, [%o1 + 0x18] %asi bg,pt %icc, 1b ! We don't care about pages >4GB inc 0x20, %o1 retl wr %g0, ASI_PRIMARY_NOFAULT, %asi #else set NBPG, %o3 add %o3, %o0, %o3 mov %g1, %o4 ! Save g1 1: ldxa [%o0] ASI_PHYS_CACHED, %g1 inc 8, %o0 cmp %o0, %o3 stxa %g1, [%o1] ASI_PHYS_CACHED bl,pt %icc, 1b ! We don't care about pages >4GB inc 8, %o1 retl mov %o4, %g1 ! Restore g1 #endif /* * extern int64_t pseg_get_real(struct pmap *pm, vaddr_t addr); * * Return TTE at addr in pmap. Uses physical addressing only. * pmap->pm_physaddr must by the physical address of pm_segs * */ ENTRY(pseg_get_real) ! flushw ! Make sure we don't have stack probs & lose hibits of %o #ifndef _LP64 clruw %o1 ! Zero extend #endif ldx [%o0 + PM_PHYS], %o2 ! pmap->pm_segs srax %o1, HOLESHIFT, %o3 ! Check for valid address brz,pt %o3, 0f ! Should be zero or -1 inc %o3 ! Make -1 -> 0 brnz,pn %o3, 1f ! Error! In hole! 0: srlx %o1, STSHIFT, %o3 and %o3, STMASK, %o3 ! Index into pm_segs sll %o3, 3, %o3 add %o2, %o3, %o2 DLFLUSH(%o2,%o3) ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page directory pointer DLFLUSH2(%o3) srlx %o1, PDSHIFT, %o3 and %o3, PDMASK, %o3 sll %o3, 3, %o3 brz,pn %o2, 1f ! NULL entry? check somewhere else add %o2, %o3, %o2 DLFLUSH(%o2,%o3) ldxa [%o2] ASI_PHYS_CACHED, %o2 ! Load page table pointer DLFLUSH2(%o3) srlx %o1, PTSHIFT, %o3 ! Convert to ptab offset and %o3, PTMASK, %o3 sll %o3, 3, %o3 brz,pn %o2, 1f ! NULL entry? check somewhere else add %o2, %o3, %o2 DLFLUSH(%o2,%o3) ldxa [%o2] ASI_PHYS_CACHED, %o0 DLFLUSH2(%o3) brgez,pn %o0, 1f ! Entry invalid? Punt btst 1, %sp bz,pn %icc, 0f ! 64-bit mode? nop retl ! Yes, return full value nop 0: #if 1 srl %o0, 0, %o1 retl ! No, generate a %o0:%o1 double srlx %o0, 32, %o0 #else DLFLUSH(%o2,%o3) ldda [%o2] ASI_PHYS_CACHED, %o0 DLFLUSH2(%o3) retl ! No, generate a %o0:%o1 double nop #endif 1: #ifndef _LP64 clr %o1 #endif retl clr %o0 /* * In 32-bit mode: * * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, * int64_t tte %o2:%o3, paddr_t spare %o4:%o5); * * In 64-bit mode: * * extern int pseg_set_real(struct pmap* %o0, vaddr_t addr %o1, * int64_t tte %o2, paddr_t spare %o3); * * Set a pseg entry to a particular TTE value. Return values are: * * -2 addr in hole * 0 success (spare was not used if given) * 1 failure (spare was not given, but one is needed) * 2 success (spare was given, used for L2) * 3 failure (spare was given, used for L2, another is needed for L3) * 4 success (spare was given, used for L3) * * rv == 0 success, spare not used if one was given * rv & 4 spare was used for L3 * rv & 2 spare was used for L2 * rv & 1 failure, spare is needed * * (NB: nobody in pmap checks for the virtual hole, so the system will hang.) * The way to call this is: first just call it without a spare page. * If that fails, allocate a page and try again, passing the paddr of the * new page as the spare. * If spare is non-zero it is assumed to be the address of a zeroed physical * page that can be used to generate a directory table or page table if needed. * * We keep track of valid (A_TLB_V bit set) and wired (A_TLB_TSB_LOCK bit set) * mappings that are set here. We check both bits on the new data entered * and increment counts, as well as decrementing counts if the bits are set * in the value replaced by this call. * The counters are 32 bit or 64 bit wide, depending on the kernel type we are * running! */ ENTRY(pseg_set_real) #ifndef _LP64 clruw %o1 ! Zero extend COMBINE(%o2, %o3, %o2) COMBINE(%o4, %o5, %o3) #endif !! !! However we managed to get here we now have: !! !! %o0 = *pmap !! %o1 = addr !! %o2 = tte !! %o3 = paddr of spare page !! srax %o1, HOLESHIFT, %o4 ! Check for valid address brz,pt %o4, 0f ! Should be zero or -1 inc %o4 ! Make -1 -> 0 brz,pt %o4, 0f nop #ifdef DEBUG ta 1 ! Break into debugger #endif retl mov -2, %o0 ! Error -- in hole! 0: ldx [%o0 + PM_PHYS], %o4 ! pmap->pm_segs clr %g1 srlx %o1, STSHIFT, %o5 and %o5, STMASK, %o5 sll %o5, 3, %o5 add %o4, %o5, %o4 0: DLFLUSH(%o4,%g5) ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load page directory pointer DLFLUSH2(%g5) brnz,a,pt %o5, 0f ! Null pointer? mov %o5, %o4 brz,pn %o3, 9f ! Have a spare? mov %o3, %o5 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 brnz,pn %o5, 0b ! Something changed? DLFLUSH(%o4, %o5) mov %o3, %o4 mov 2, %g1 ! record spare used for L2 clr %o3 ! and not available for L3 0: srlx %o1, PDSHIFT, %o5 and %o5, PDMASK, %o5 sll %o5, 3, %o5 add %o4, %o5, %o4 0: DLFLUSH(%o4,%g5) ldxa [%o4] ASI_PHYS_CACHED, %o5 ! Load table directory pointer DLFLUSH2(%g5) brnz,a,pt %o5, 0f ! Null pointer? mov %o5, %o4 brz,pn %o3, 9f ! Have a spare? mov %o3, %o5 casxa [%o4] ASI_PHYS_CACHED, %g0, %o5 brnz,pn %o5, 0b ! Something changed? DLFLUSH(%o4, %o4) mov %o3, %o4 mov 4, %g1 ! record spare used for L3 0: srlx %o1, PTSHIFT, %o5 ! Convert to ptab offset and %o5, PTMASK, %o5 sll %o5, 3, %o5 add %o5, %o4, %o4 DLFLUSH(%o4,%g5) ldxa [%o4] ASI_PHYS_CACHED, %o5 ! save old value in %o5 stxa %o2, [%o4] ASI_PHYS_CACHED ! Easier than shift+or DLFLUSH2(%g5) !! at this point we have: !! %g1 = return value !! %o0 = struct pmap * (where the counts are) !! %o2 = new TTE !! %o5 = old TTE !! see if stats needs an update #ifdef SUN4V sethi %hi(cputyp), %g5 ld [%g5 + %lo(cputyp)], %g5 cmp %g5, CPU_SUN4V bne,pt %icc, 0f nop sethi %hh(SUN4V_TLB_TSB_LOCK), %g5 sllx %g5, 32, %g5 ba 1f nop 0: #endif set SUN4U_TLB_TSB_LOCK, %g5 1: xor %o2, %o5, %o3 ! %o3 - what changed brgez,pn %o3, 5f ! has resident changed? (we predict it has) btst %g5, %o3 ! has wired changed? LDPTR [%o0 + PM_RESIDENT], %o1 ! gonna update resident count brlz %o2, 0f mov 1, %o4 neg %o4 ! new is not resident -> decrement 0: add %o1, %o4, %o1 STPTR %o1, [%o0 + PM_RESIDENT] btst %g5, %o3 ! has wired changed? 5: bz,pt %xcc, 8f ! we predict it's not btst %g5, %o2 ! don't waste delay slot, check if new one is wired LDPTR [%o0 + PM_WIRED], %o1 ! gonna update wired count bnz,pt %xcc, 0f ! if wired changes, we predict it increments mov 1, %o4 neg %o4 ! new is not wired -> decrement 0: add %o1, %o4, %o1 STPTR %o1, [%o0 + PM_WIRED] 8: retl mov %g1, %o0 ! return %g1 9: retl or %g1, 1, %o0 ! spare needed, return flags + 1 /* * clearfpstate() * * Drops the current fpu state, without saving it. */ ENTRY(clearfpstate) rdpr %pstate, %o1 ! enable FPU wr %g0, FPRS_FEF, %fprs or %o1, PSTATE_PEF, %o1 retl wrpr %o1, 0, %pstate /* * savefpstate(f) struct fpstate *f; * * Store the current FPU state. * * Since the kernel may need to use the FPU and we have problems atomically * testing and enabling the FPU, we leave here with the FPRS_FEF bit set. * Normally this should be turned on in loadfpstate(). */ /* XXXXXXXXXX Assume caller created a proper stack frame */ ENTRY(savefpstate) ! flushw ! Make sure we don't have stack probs & lose hibits of %o rdpr %pstate, %o1 ! enable FP before we begin rd %fprs, %o5 wr %g0, FPRS_FEF, %fprs or %o1, PSTATE_PEF, %o1 wrpr %o1, 0, %pstate stx %fsr, [%o0 + FS_FSR] ! f->fs_fsr = getfsr(); rd %gsr, %o4 ! Save %gsr st %o4, [%o0 + FS_GSR] add %o0, FS_REGS, %o2 #ifdef DIAGNOSTIC btst BLOCK_ALIGN, %o2 ! Needs to be re-executed bnz,pn %icc, 6f ! Check alignment #endif st %g0, [%o0 + FS_QSIZE] ! f->fs_qsize = 0; btst FPRS_DL|FPRS_DU, %o5 ! Both FPU halves clean? bz,pt %icc, 5f ! Then skip it btst FPRS_DL, %o5 ! Lower FPU clean? membar #Sync bz,a,pt %icc, 1f ! Then skip it, but upper FPU not clean add %o2, 2*BLOCK_SIZE, %o2 ! Skip a block stda %f0, [%o2] ASI_BLK_P ! f->fs_f0 = etc; inc BLOCK_SIZE, %o2 stda %f16, [%o2] ASI_BLK_P btst FPRS_DU, %o5 ! Upper FPU clean? bz,pt %icc, 2f ! Then skip it inc BLOCK_SIZE, %o2 1: stda %f32, [%o2] ASI_BLK_P inc BLOCK_SIZE, %o2 stda %f48, [%o2] ASI_BLK_P 2: membar #Sync ! Finish operation so we can 5: retl wr %g0, FPRS_FEF, %fprs ! Mark FPU clean #ifdef DIAGNOSTIC !! !! Damn thing is *NOT* aligned on a 64-byte boundary !! 6: wr %g0, FPRS_FEF, %fprs ! XXX -- we should panic instead of silently entering debugger ta 1 retl nop #endif /* * Load FPU state. */ /* XXXXXXXXXX Should test to see if we only need to do a partial restore */ ENTRY(loadfpstate) flushw ! Make sure we don't have stack probs & lose hibits of %o rdpr %pstate, %o1 ! enable FP before we begin ld [%o0 + FS_GSR], %o4 ! Restore %gsr set PSTATE_PEF, %o2 wr %g0, FPRS_FEF, %fprs or %o1, %o2, %o1 wrpr %o1, 0, %pstate ldx [%o0 + FS_FSR], %fsr ! setfsr(f->fs_fsr); add %o0, FS_REGS, %o3 ! This is zero... #ifdef DIAGNOSTIC btst BLOCK_ALIGN, %o3 bne,pn %icc, 1f ! Only use block loads on aligned blocks #endif wr %o4, %g0, %gsr membar #Sync ldda [%o3] ASI_BLK_P, %f0 inc BLOCK_SIZE, %o3 ldda [%o3] ASI_BLK_P, %f16 inc BLOCK_SIZE, %o3 ldda [%o3] ASI_BLK_P, %f32 inc BLOCK_SIZE, %o3 ldda [%o3] ASI_BLK_P, %f48 membar #Sync ! Make sure loads are complete retl wr %g0, FPRS_FEF, %fprs ! Clear dirty bits #ifdef DIAGNOSTIC !! !! Damn thing is *NOT* aligned on a 64-byte boundary !! 1: wr %g0, FPRS_FEF, %fprs ! Clear dirty bits ! XXX -- we should panic instead of silently entering debugger ta 1 retl nop #endif /* * ienab_bis(bis) int bis; * ienab_bic(bic) int bic; * * Set and clear bits in the interrupt register. */ /* * sun4u has separate asr's for clearing/setting the interrupt mask. */ ENTRY(ienab_bis) retl wr %o0, 0, SET_SOFTINT ! SET_SOFTINT ENTRY(ienab_bic) retl wr %o0, 0, CLEAR_SOFTINT ! CLEAR_SOFTINT /* * send_softint(cpu, level, intrhand) * * Send a softint with an intrhand pointer so we can cause a vectored * interrupt instead of a polled interrupt. This does pretty much the same * as interrupt_vector. If cpu is -1 then send it to this CPU, if it's -2 * send it to any CPU, otherwise send it to a particular CPU. * * XXXX Dispatching to different CPUs is not implemented yet. */ ENTRY(send_softint) rdpr %pstate, %g1 andn %g1, PSTATE_IE, %g2 ! clear PSTATE.IE wrpr %g2, 0, %pstate sethi %hi(CPUINFO_VA+CI_INTRPENDING), %o3 LDPTR [%o2 + IH_PEND], %o5 or %o3, %lo(CPUINFO_VA+CI_INTRPENDING), %o3 brnz %o5, 1f sll %o1, PTRSHFT, %o5 ! Find start of table for this IPL add %o3, %o5, %o3 2: LDPTR [%o3], %o5 ! Load list head STPTR %o5, [%o2+IH_PEND] ! Link our intrhand node in mov %o2, %o4 CASPTRA [%o3] ASI_N, %o5, %o4 cmp %o4, %o5 ! Did it work? bne,pn CCCR, 2b ! No, try again .empty mov 1, %o4 ! Change from level to bitmask sllx %o4, %o1, %o4 wr %o4, 0, SET_SOFTINT ! SET_SOFTINT 1: retl wrpr %g1, 0, %pstate ! restore PSTATE.IE #define MICROPERSEC (1000000) /* * delay function * * void delay(N) -- delay N microseconds * * Register usage: %o0 = "N" number of usecs to go (counts down to zero) * %o1 = "timerblurb" (stays constant) * %o2 = counter for 1 usec (counts down from %o1 to zero) * * * ci_cpu_clockrate should be tuned during CPU probe to the CPU * clockrate in Hz * */ ENTRY(delay) ! %o0 = n #if 1 rdpr %tick, %o1 ! Take timer snapshot sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %o2 sethi %hi(MICROPERSEC), %o3 ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)], %o4 ! Get scale factor brnz,pt %o4, 0f or %o3, %lo(MICROPERSEC), %o3 !! Calculate ticks/usec ldx [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %o4 ! No, we need to calculate it udivx %o4, %o3, %o4 stx %o4, [%o2 + %lo(CPUINFO_VA + CI_CLOCKRATE + 8)] ! Save it so we don't need to divide again 0: mulx %o0, %o4, %o0 ! Convert usec -> ticks rdpr %tick, %o2 ! Top of next itr 1: sub %o2, %o1, %o3 ! How many ticks have gone by? sub %o0, %o3, %o4 ! Decrement count by that much movrgz %o3, %o4, %o0 ! But only if we're decrementing mov %o2, %o1 ! Remember last tick brgz,pt %o0, 1b ! Done? rdpr %tick, %o2 ! Get new tick retl nop #else /* This code only works if %tick does not wrap */ rdpr %tick, %g1 ! Take timer snapshot sethi %hi(CPUINFO_VA + CI_CLOCKRATE), %g2 sethi %hi(MICROPERSEC), %o2 ldx [%g2 + %lo(CPUINFO_VA + CI_CLOCKRATE)], %g2 ! Get scale factor or %o2, %lo(MICROPERSEC), %o2 ! sethi %hi(_C_LABEL(timerblurb), %o5 ! This is if we plan to tune the clock ! ld [%o5 + %lo(_C_LABEL(timerblurb))], %o5 ! with respect to the counter/timer mulx %o0, %g2, %g2 ! Scale it: (usec * Hz) / 1 x 10^6 = ticks udivx %g2, %o2, %g2 add %g1, %g2, %g2 ! add %o5, %g2, %g2 5, %g2, %g2 ! But this gets complicated rdpr %tick, %g1 ! Top of next itr mov %g1, %g1 ! Erratum 50 1: cmp %g1, %g2 bl,a,pn %xcc, 1b ! Done? rdpr %tick, %g1 retl nop #endif /* * If something's wrong with the standard setup do this stupid loop * calibrated for a 143MHz processor. */ Lstupid_delay: set 142857143/MICROPERSEC, %o1 Lstupid_loop: brnz,pt %o1, Lstupid_loop dec %o1 brnz,pt %o0, Lstupid_delay dec %o0 retl nop /* * next_tick(long increment) * * Sets the %tick_cmpr register to fire off in `increment' machine * cycles in the future. Also handles %tick wraparound. In 32-bit * mode we're limited to a 32-bit increment. */ ENTRY(next_tick) rd TICK_CMPR, %o2 rdpr %tick, %o1 mov 1, %o3 ! Mask off high bits of these registers sllx %o3, 63, %o3 andn %o1, %o3, %o1 andn %o2, %o3, %o2 cmp %o1, %o2 ! Did we wrap? (tick < tick_cmpr) bgt,pt %icc, 1f add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. /* * Handle the unlikely case of %tick wrapping. * * This should only happen every 10 years or more. * * We need to increment the time base by the size of %tick in * microseconds. This will require some divides and multiplies * which can take time. So we re-read %tick. * */ /* XXXXX NOT IMPLEMENTED */ 1: add %o2, %o0, %o2 andn %o2, %o3, %o4 brlz,pn %o4, Ltick_ovflw cmp %o2, %o1 ! Has this tick passed? blt,pn %xcc, 1b ! Yes nop #ifdef BB_ERRATA_1 ba,a 2f nop #else retl wr %o2, TICK_CMPR #endif Ltick_ovflw: /* * When we get here tick_cmpr has wrapped, but we don't know if %tick * has wrapped. If bit 62 is set then we have not wrapped and we can * use the current value of %o4 as %tick. Otherwise we need to return * to our loop with %o4 as %tick_cmpr (%o2). */ srlx %o3, 1, %o5 btst %o5, %o1 bz,pn %xcc, 1b mov %o4, %o2 #ifdef BB_ERRATA_1 ba,a 2f nop .align 64 2: wr %o2, TICK_CMPR rd TICK_CMPR, %g0 retl nop #else retl wr %o2, TICK_CMPR #endif /* * next_stick(long increment) * * Sets the %stick_cmpr register to fire off in `increment' machine * cycles in the future. Also handles %stick wraparound. In 32-bit * mode we're limited to a 32-bit increment. */ ENTRY(next_stick) rd STICK_CMPR, %o2 rd STICK, %o1 mov 1, %o3 ! Mask off high bits of these registers sllx %o3, 63, %o3 andn %o1, %o3, %o1 andn %o2, %o3, %o2 cmp %o1, %o2 ! Did we wrap? (stick < stick_cmpr) bgt,pt %xcc, 1f add %o1, 1000, %o1 ! Need some slack so we don't lose intrs. /* * Handle the unlikely case of %stick wrapping. * * This should only happen every 10 years or more. * * We need to increment the time base by the size of %stick in * microseconds. This will require some divides and multiplies * which can take time. So we re-read %stick. * */ /* XXXXX NOT IMPLEMENTED */ 1: add %o2, %o0, %o2 andn %o2, %o3, %o4 brlz,pn %o4, Lstick_ovflw cmp %o2, %o1 ! Has this stick passed? blt,pn %xcc, 1b ! Yes nop retl wr %o2, STICK_CMPR Lstick_ovflw: /* * When we get here tick_cmpr has wrapped, but we don't know if %stick * has wrapped. If bit 62 is set then we have not wrapped and we can * use the current value of %o4 as %stick. Otherwise we need to return * to our loop with %o4 as %stick_cmpr (%o2). */ srlx %o3, 1, %o5 btst %o5, %o1 bz,pn %xcc, 1b mov %o4, %o2 retl wr %o2, STICK_CMPR /* * next_stick_init() * * Sets the %stick_cmpr register to the value retrieved from %stick so * next_stick() does not spend too much time in the function when called * for the first time. * This has been observed on (at least) a SPARC-T5 (sun4v) system where * the %stick_cmpr ends up being less than the %stick value and then * the stickitr() interrupt is never triggered. */ ENTRY(next_stick_init) rd STICK, %o0 mov 1, %o1 ! Mask off high bits of the register sllx %o1, 63, %o1 andn %o0, %o1, %o0 retl wr %o0, STICK_CMPR ENTRY(setjmp) save %sp, -CC64FSZ, %sp ! Need a frame to return to. flushw stx %fp, [%i0+0] ! 64-bit stack pointer stx %i7, [%i0+8] ! 64-bit return pc ret restore %g0, 0, %o0 .data Lpanic_ljmp: .asciz "longjmp botch" _ALIGN .text ENTRY(longjmp) save %sp, -CC64FSZ, %sp ! prepare to restore to (old) frame flushw mov 1, %i2 ldx [%i0+0], %fp ! get return stack movrz %i1, %i1, %i2 ! compute v ? v : 1 ldx [%i0+8], %i7 ! get rpc ret restore %i2, 0, %o0 #if defined(DDB) || defined(KGDB) /* * Debug stuff. Dump the trap registers into buffer & set tl=0. * * %o0 = *ts */ ENTRY(savetstate) mov %o0, %o1 rdpr %tl, %o0 brz %o0, 2f mov %o0, %o2 1: rdpr %tstate, %o3 stx %o3, [%o1] deccc %o2 inc 8, %o1 rdpr %tpc, %o4 stx %o4, [%o1] inc 8, %o1 rdpr %tnpc, %o5 stx %o5, [%o1] inc 8, %o1 rdpr %tt, %o4 stx %o4, [%o1] inc 8, %o1 bnz 1b wrpr %o2, 0, %tl 2: retl nop /* * Debug stuff. Restore trap registers from buffer. * * %o0 = %tl * %o1 = *ts * * Maybe this should be re-written to increment tl instead of decrementing. */ ENTRY(restoretstate) flushw ! Make sure we don't have stack probs & lose hibits of %o brz,pn %o0, 2f mov %o0, %o2 wrpr %o0, 0, %tl 1: ldx [%o1], %o3 deccc %o2 inc 8, %o1 wrpr %o3, 0, %tstate ldx [%o1], %o4 inc 8, %o1 wrpr %o4, 0, %tpc ldx [%o1], %o5 inc 8, %o1 wrpr %o5, 0, %tnpc ldx [%o1], %o4 inc 8, %o1 wrpr %o4, 0, %tt bnz 1b wrpr %o2, 0, %tl 2: retl wrpr %o0, 0, %tl /* * Switch to context in abs(%o0) */ ENTRY(switchtoctx_us) set DEMAP_CTX_SECONDARY, %o3 stxa %o3, [%o3] ASI_DMMU_DEMAP mov CTX_SECONDARY, %o4 stxa %o3, [%o3] ASI_IMMU_DEMAP membar #Sync stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid sethi %hi(KERNBASE), %o2 membar #Sync flush %o2 retl nop ENTRY(switchtoctx_usiii) mov CTX_SECONDARY, %o4 ldxa [%o4] ASI_DMMU, %o2 ! Load secondary context mov CTX_PRIMARY, %o5 ldxa [%o5] ASI_DMMU, %o1 ! Save primary context membar #LoadStore stxa %o2, [%o5] ASI_DMMU ! Insert secondary for demap membar #Sync set DEMAP_CTX_PRIMARY, %o3 stxa %o3, [%o3] ASI_DMMU_DEMAP membar #Sync stxa %o0, [%o4] ASI_DMMU ! Maybe we should invalid membar #Sync stxa %o1, [%o5] ASI_DMMU ! Restore primary context sethi %hi(KERNBASE), %o2 membar #Sync flush %o2 retl nop #ifndef _LP64 /* * Convert to 32-bit stack then call OF_sym2val() */ ENTRY(OF_sym2val32) save %sp, -CC64FSZ, %sp btst 7, %i0 bnz,pn %icc, 1f add %sp, BIAS, %o1 btst 1, %sp movnz %icc, %o1, %sp call _C_LABEL(OF_sym2val) mov %i0, %o0 1: ret restore %o0, 0, %o0 /* * Convert to 32-bit stack then call OF_val2sym() */ ENTRY(OF_val2sym32) save %sp, -CC64FSZ, %sp btst 7, %i0 bnz,pn %icc, 1f add %sp, BIAS, %o1 btst 1, %sp movnz %icc, %o1, %sp call _C_LABEL(OF_val2sym) mov %i0, %o0 1: ret restore %o0, 0, %o0 #endif /* _LP64 */ #endif /* DDB */ #if defined(MULTIPROCESSOR) /* * IPI target function to setup a C compatible environment and call a MI function. * * On entry: * We are on one of the alternate set of globals * %g2 = function to call * %g3 = single argument to called function */ ENTRY(sparc64_ipi_ccall) #ifdef TRAPS_USE_IG wrpr %g0, PSTATE_KERN|PSTATE_IG, %pstate ! DEBUG #endif TRAP_SETUP(-CC64FSZ-TF_SIZE) #ifdef DEBUG rdpr %tt, %o1 ! debug sth %o1, [%sp + CC64FSZ + STKB + TF_TT]! debug #endif mov %g3, %o0 ! save argument of function to call mov %g2, %o5 ! save function pointer wrpr %g0, PSTATE_KERN, %pstate ! Get back to normal globals stx %g1, [%sp + CC64FSZ + STKB + TF_G + ( 1*8)] rdpr %tpc, %o2 ! (pc) stx %g2, [%sp + CC64FSZ + STKB + TF_G + ( 2*8)] rdpr %tstate, %g1 stx %g3, [%sp + CC64FSZ + STKB + TF_G + ( 3*8)] rdpr %tnpc, %o3 stx %g4, [%sp + CC64FSZ + STKB + TF_G + ( 4*8)] rd %y, %o4 stx %g5, [%sp + CC64FSZ + STKB + TF_G + ( 5*8)] stx %g6, [%sp + CC64FSZ + STKB + TF_G + ( 6*8)] stx %g7, [%sp + CC64FSZ + STKB + TF_G + ( 7*8)] stx %g1, [%sp + CC64FSZ + STKB + TF_TSTATE] stx %o2, [%sp + CC64FSZ + STKB + TF_PC] stx %o3, [%sp + CC64FSZ + STKB + TF_NPC] st %o4, [%sp + CC64FSZ + STKB + TF_Y] rdpr %pil, %g5 stb %g5, [%sp + CC64FSZ + STKB + TF_PIL] stb %g5, [%sp + CC64FSZ + STKB + TF_OLDPIL] rdpr %tl, %g7 dec %g7 movrlz %g7, %g0, %g7 wrpr %g0, %g7, %tl !! In the EMBEDANY memory model %g4 points to the start of the data segment. !! In our case we need to clear it before calling any C-code clr %g4 wr %g0, ASI_NUCLEUS, %asi ! default kernel ASI call %o5 ! call function nop b return_from_trap ! and return from IPI ldx [%sp + CC64FSZ + STKB + TF_TSTATE], %g1 ! Load this for return_from_trap #endif .data _ALIGN #if NKSYMS || defined(DDB) || defined(MODULAR) .globl _C_LABEL(esym) _C_LABEL(esym): POINTER 0 .globl _C_LABEL(ssym) _C_LABEL(ssym): POINTER 0 #endif .comm _C_LABEL(promvec), PTRSZ #ifdef DEBUG .comm _C_LABEL(trapdebug), 4 .comm _C_LABEL(pmapdebug), 4 #endif