/*	$NetBSD: ipifuncs.c,v 1.57 2022/05/03 20:52:31 andvar Exp $ */

/*-
 * Copyright (c) 2004 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.57 2022/05/03 20:52:31 andvar Exp $");

#include "opt_ddb.h"

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/xcall.h>
#include <sys/ipi.h>

#include <machine/db_machdep.h>

#include <machine/cpu.h>
#include <machine/cpu_counter.h>
#include <machine/ctlreg.h>
#include <machine/pmap.h>
#include <machine/sparc64.h>

#include <sparc64/sparc64/cache.h>

#if defined(DDB) || defined(KGDB)
#ifdef DDB
#include <ddb/db_command.h>
#include <ddb/db_output.h>
#endif
#endif

#define SPARC64_IPI_RETRIES	10000

/* CPU sets containing halted, paused and resumed cpus */
static volatile sparc64_cpuset_t cpus_halted;
static volatile sparc64_cpuset_t cpus_spinning;
static volatile sparc64_cpuset_t cpus_paused;
static volatile sparc64_cpuset_t cpus_resumed;

/* IPI handlers. */
static int	sparc64_ipi_wait(sparc64_cpuset_t volatile *, sparc64_cpuset_t);
static void	sparc64_ipi_error(const char *, sparc64_cpuset_t, sparc64_cpuset_t);

/* Send IPI functions for supported platforms */
static void	sparc64_send_ipi_sun4u(int, ipifunc_t, uint64_t, uint64_t);
static void	sparc64_send_ipi_sun4v(int, ipifunc_t, uint64_t, uint64_t);
void		(*sparc64_send_ipi)(int, ipifunc_t, uint64_t, uint64_t) = NULL;
 
/*
 * These are the "function" entry points in locore.s/mp_subr.s to handle IPI's.
 */
void	sparc64_ipi_halt(void *, void *);
void	sparc64_ipi_pause(void *, void *);
void	sparc64_ipi_flush_pte_us(void *, void *);
void	sparc64_ipi_flush_pte_usiii(void *, void *);
void	sparc64_ipi_flush_pte_sun4v(void *, void *);
void	sparc64_ipi_dcache_flush_page_us(void *, void *);
void	sparc64_ipi_dcache_flush_page_usiii(void *, void *);
void	sparc64_ipi_dcache_flush_page_sun4v(void *, void *);
void	sparc64_ipi_blast_dcache(void *, void *);
void	sparc64_ipi_ccall(void *, void *);

/* Function pointers to be setup in sparc64_ipi_init() */
static ipifunc_t smp_tlb_flush_pte_func = NULL;
static ipifunc_t sparc64_ipi_dcache_flush_page_func = NULL;

/*
 * Process cpu stop-self event.
 */
void
sparc64_ipi_halt_thiscpu(void *arg, void *arg2)
{
	extern void prom_printf(const char *fmt, ...);

	printf("cpu%d: shutting down\n", cpu_number());
	if (prom_has_stop_other() || !prom_has_stopself()) {
		/*
		 * just loop here, the final cpu will stop us later
		 */
		CPUSET_ADD(cpus_spinning, cpu_number());
		CPUSET_ADD(cpus_halted, cpu_number());
		spl0();
		while (1)
			/* nothing */;
	} else {
		CPUSET_ADD(cpus_halted, cpu_number());
		prom_stopself();
	}
}

void
sparc64_do_pause(void)
{
#if defined(DDB)
	extern bool ddb_running_on_this_cpu(void);
	extern void db_resume_others(void);
#endif

	CPUSET_ADD(cpus_paused, cpu_number());

	do {
		membar_Sync();
	} while(CPUSET_HAS(cpus_paused, cpu_number()));
	membar_Sync();
	CPUSET_ADD(cpus_resumed, cpu_number());

#if defined(DDB)
	if (ddb_running_on_this_cpu()) {
		db_command_loop();
		db_resume_others();
	}
#endif
}

/*
 * Pause cpu.  This is called from locore.s after setting up a trapframe.
 */
void
sparc64_ipi_pause_thiscpu(void *arg)
{
	int s;
#if defined(DDB)
	extern void fill_ddb_regs_from_tf(struct trapframe64 *tf);
	extern void ddb_restore_state(void);
	
	if (arg)
		fill_ddb_regs_from_tf(arg);
#endif

	s = intr_disable();
	sparc64_do_pause();

#if defined(DDB)
	if (arg) {
		ddb_restore_state();
		curcpu()->ci_ddb_regs = NULL;
	}
#endif

	intr_restore(s);
}

/*
 * Initialize IPI machinery.
 */
void
sparc64_ipi_init(void)
{

	/* Clear all cpu sets. */
	CPUSET_CLEAR(cpus_halted);
	CPUSET_CLEAR(cpus_spinning);
	CPUSET_CLEAR(cpus_paused);
	CPUSET_CLEAR(cpus_resumed);

	/*
	 * Prepare cpu type dependent function pointers
	 */

	if (CPU_ISSUN4V) {
		smp_tlb_flush_pte_func = sparc64_ipi_flush_pte_sun4v;
		sparc64_ipi_dcache_flush_page_func =
		    sparc64_ipi_dcache_flush_page_sun4v;
	}
	else if (CPU_IS_USIII_UP()) {
		smp_tlb_flush_pte_func = sparc64_ipi_flush_pte_usiii;
		sparc64_ipi_dcache_flush_page_func =
		    sparc64_ipi_dcache_flush_page_usiii;
	}
	else {
		smp_tlb_flush_pte_func = sparc64_ipi_flush_pte_us;
		sparc64_ipi_dcache_flush_page_func =
		    sparc64_ipi_dcache_flush_page_us;
	}

	if (CPU_ISSUN4V)
		sparc64_send_ipi = sparc64_send_ipi_sun4v;
	else
		sparc64_send_ipi = sparc64_send_ipi_sun4u;

}

/*
 * Send an IPI to all in the list but ourselves.
 */
void
sparc64_multicast_ipi(sparc64_cpuset_t cpuset, ipifunc_t func, uint64_t arg1,
		      uint64_t arg2)
{
	struct cpu_info *ci;

	CPUSET_DEL(cpuset, cpu_number());
	if (CPUSET_EMPTY(cpuset))
		return;

	for (ci = cpus; ci != NULL; ci = ci->ci_next) {
		if (CPUSET_HAS(cpuset, ci->ci_index)) {
			CPUSET_DEL(cpuset, ci->ci_index);
			sparc64_send_ipi(ci->ci_cpuid, func, arg1, arg2);
		}
	}
}

/*
 * Broadcast an IPI to all but ourselves.
 */
void
sparc64_broadcast_ipi(ipifunc_t func, uint64_t arg1, uint64_t arg2)
{

	sparc64_multicast_ipi(CPUSET_EXCEPT(cpus_active, cpu_number()), func,
		arg1, arg2);
}

/*
 * Send an interprocessor interrupt - sun4u.
 */
void
sparc64_send_ipi_sun4u(int upaid, ipifunc_t func, uint64_t arg1, uint64_t arg2)
{
	int i, ik, shift = 0;
	uint64_t intr_func;

	KASSERT(upaid != curcpu()->ci_cpuid);

	/*
	 * UltraSPARC-IIIi CPUs select the BUSY/NACK pair based on the
	 * lower two bits of the ITID.
	 */
	if (CPU_IS_USIIIi())
		shift = (upaid & 0x3) * 2;

	if (ldxa(0, ASI_IDSR) & (IDSR_BUSY << shift))
		panic("recursive IPI?");

	intr_func = (uint64_t)(u_long)func;

	/* Schedule an interrupt. */
	for (i = 0; i < 10000; i++) {
		int s = intr_disable();

		stxa(IDDR_0H, ASI_INTERRUPT_DISPATCH, intr_func);
		stxa(IDDR_1H, ASI_INTERRUPT_DISPATCH, arg1);
		stxa(IDDR_2H, ASI_INTERRUPT_DISPATCH, arg2);
		stxa(IDCR(upaid), ASI_INTERRUPT_DISPATCH, 0);
		membar_Sync();
		/* Workaround for SpitFire erratum #54, from FreeBSD */
		if (CPU_IS_SPITFIRE()) {
			(void)ldxa(P_DCR_0, ASI_INTERRUPT_RECEIVE_DATA);
			membar_Sync();
		}

		for (ik = 0; ik < 1000000; ik++) {
			if (ldxa(0, ASI_IDSR) & (IDSR_BUSY << shift))
				continue;
			else
				break;
		}
		intr_restore(s);

		if (ik == 1000000)
			break;

		if ((ldxa(0, ASI_IDSR) & (IDSR_NACK << shift)) == 0)
			return;
		/*
		 * Wait for a while with enabling interrupts to avoid
		 * deadlocks.  XXX - random value is better.
		 */
		DELAY(1);
	}

	if (panicstr == NULL)
		panic("cpu%d: ipi_send: couldn't send ipi to UPAID %u"
			" (tried %d times)", cpu_number(), upaid, i);
}

/*
 * Send an interprocessor interrupt - sun4v.
 */
void
sparc64_send_ipi_sun4v(int cpuid, ipifunc_t func, uint64_t arg1, uint64_t arg2)
{
	struct cpu_info *ci = curcpu();
	int err, i;
	
	stha(ci->ci_cpuset, ASI_PHYS_CACHED, cpuid);
	stxa(ci->ci_mondo, ASI_PHYS_CACHED, (vaddr_t)func);
	stxa(ci->ci_mondo + 8, ASI_PHYS_CACHED, arg1);
	stxa(ci->ci_mondo + 16, ASI_PHYS_CACHED, arg2);
	
	for (i = 0; i < SPARC64_IPI_RETRIES; i++) {
		err = hv_cpu_mondo_send(1, ci->ci_cpuset, ci->ci_mondo);
		if (err != H_EWOULDBLOCK)
			break;
		delay(10);
	}
	if (err != H_EOK)
		panic("Unable to send mondo %lx to cpu %d: %d",
		    (long unsigned int)func, cpuid, err);
}

/*
 * Wait for IPI operation to complete.
 * Return 0 on success.
 */
int
sparc64_ipi_wait(sparc64_cpuset_t volatile *cpus_watchset, sparc64_cpuset_t cpus_mask)
{
	uint64_t limit = gettick() + cpu_frequency(curcpu());

	while (gettick() < limit) {
		membar_Sync();
		if (CPUSET_EQUAL(*cpus_watchset, cpus_mask))
			return 0;
	}
	return 1;
}

/*
 * Halt all cpus but ourselves.
 */
void
mp_halt_cpus(void)
{
	sparc64_cpuset_t cpumask, cpuset;
	struct cpu_info *ci;

	CPUSET_ASSIGN(cpuset, cpus_active);
	CPUSET_DEL(cpuset, cpu_number());
	CPUSET_ASSIGN(cpumask, cpuset);
	CPUSET_SUB(cpuset, cpus_halted);

	if (CPUSET_EMPTY(cpuset))
		return;

	CPUSET_CLEAR(cpus_spinning);
	sparc64_multicast_ipi(cpuset, sparc64_ipi_halt, 0, 0);
	if (sparc64_ipi_wait(&cpus_halted, cpumask))
		sparc64_ipi_error("halt", cpumask, cpus_halted);

	/*
	 * Depending on available firmware methods, other cpus will
	 * either shut down themselves, or spin and wait for us to
	 * stop them.
	 */
	if (CPUSET_EMPTY(cpus_spinning)) {
		/* give other cpus a few cycles to actually power down */
		delay(10000);
		return;
	}
	/* there are cpus spinning - shut them down if we can */
	if (prom_has_stop_other()) {
		for (ci = cpus; ci != NULL; ci = ci->ci_next) {
			if (!CPUSET_HAS(cpus_spinning, ci->ci_index)) continue;
			prom_stop_other(ci->ci_cpuid);
		}
	}
}

/*
 * Pause all cpus but ourselves.
 */
void
mp_pause_cpus(void)
{
	int i = 3;
	sparc64_cpuset_t cpuset;

	CPUSET_ASSIGN(cpuset, cpus_active);
	CPUSET_DEL(cpuset, cpu_number());
	while (i-- > 0) {
		if (CPUSET_EMPTY(cpuset))
			return;

		sparc64_multicast_ipi(cpuset, sparc64_ipi_pause, 0, 0);
		if (!sparc64_ipi_wait(&cpus_paused, cpuset))
			return;
		CPUSET_SUB(cpuset, cpus_paused);
	}
	sparc64_ipi_error("pause", cpus_paused, cpuset);
}

/*
 * Resume a single cpu
 */
void
mp_resume_cpu(int cno)
{
	CPUSET_DEL(cpus_paused, cno);
	membar_Sync();
}

/*
 * Resume all paused cpus.
 */
void
mp_resume_cpus(void)
{
	int i = 3;
	sparc64_cpuset_t cpuset;

	CPUSET_CLEAR(cpuset);	/* XXX: gcc -Wuninitialized */

	while (i-- > 0) {
		CPUSET_CLEAR(cpus_resumed);
		CPUSET_ASSIGN(cpuset, cpus_paused);
		membar_Sync();
		CPUSET_CLEAR(cpus_paused);

		/* CPUs awake on cpus_paused clear */
		if (!sparc64_ipi_wait(&cpus_resumed, cpuset))
			return;
	}
	sparc64_ipi_error("resume", cpus_resumed, cpuset);
}

int
mp_cpu_is_paused(sparc64_cpuset_t cpunum)
{

	return CPUSET_HAS(cpus_paused, cpunum);
}

/*
 * Flush pte on all active processors.
 */
void
smp_tlb_flush_pte(vaddr_t va, struct pmap * pm)
{
	sparc64_cpuset_t cpuset;
	struct cpu_info *ci;
	int ctx;
	bool kpm = (pm == pmap_kernel());
	/* Flush our own TLB */
	ctx = pm->pm_ctx[cpu_number()];
	KASSERT(ctx >= 0);
	if (kpm || ctx > 0)
		sp_tlb_flush_pte(va, ctx);

	CPUSET_ASSIGN(cpuset, cpus_active);
	CPUSET_DEL(cpuset, cpu_number());
	if (CPUSET_EMPTY(cpuset))
		return;

	/* Flush others */
	for (ci = cpus; ci != NULL; ci = ci->ci_next) {
		if (CPUSET_HAS(cpuset, ci->ci_index)) {
			CPUSET_DEL(cpuset, ci->ci_index);
			ctx = pm->pm_ctx[ci->ci_index];
			KASSERT(ctx >= 0);
			if (!kpm && ctx == 0)
				continue;
			sparc64_send_ipi(ci->ci_cpuid, smp_tlb_flush_pte_func, va, ctx);
		}
	}
}

/*
 * Make sure this page is flushed from all/some CPUs.
 */
void
smp_dcache_flush_page_cpuset(paddr_t pa, sparc64_cpuset_t activecpus)
{
	sparc64_multicast_ipi(activecpus, sparc64_ipi_dcache_flush_page_func, pa, dcache_line_size);
	sp_dcache_flush_page(pa);
}

void
smp_dcache_flush_page_allcpu(paddr_t pa)
{

	smp_dcache_flush_page_cpuset(pa, cpus_active);
}

/*
 * Flush the D$ on all CPUs.
 */
void
smp_blast_dcache(void)
{

	sparc64_multicast_ipi(cpus_active, sparc64_ipi_blast_dcache,
			      dcache_size, dcache_line_size);
	sp_blast_dcache(dcache_size, dcache_line_size);
}

/*
 * Print an error message.
 */
void
sparc64_ipi_error(const char *s, sparc64_cpuset_t cpus_succeeded,
	sparc64_cpuset_t cpus_expected)
{
	int cpuid;

	CPUSET_DEL(cpus_expected, cpus_succeeded);
	if (!CPUSET_EMPTY(cpus_expected)) {
		printf("Failed to %s:", s);
		do {
			cpuid = CPUSET_NEXT(cpus_expected);
			CPUSET_DEL(cpus_expected, cpuid);
			printf(" cpu%d", cpuid);
		} while(!CPUSET_EMPTY(cpus_expected));
	}

	printf("\n");
}

/*
 * MD support for xcall(9) interface.
 */

void
sparc64_generic_xcall(struct cpu_info *target, ipi_c_call_func_t func,
	void *arg)
{
	/* if target == NULL broadcast to everything but curcpu */
	if (target)
		sparc64_send_ipi(target->ci_cpuid, sparc64_ipi_ccall,
		    (uint64_t)(uintptr_t)func, (uint64_t)(uintptr_t)arg);
	else {
		
		sparc64_multicast_ipi(cpus_active, sparc64_ipi_ccall,
		    (uint64_t)(uintptr_t)func, (uint64_t)(uintptr_t)arg);
	}
}

void
xc_send_ipi(struct cpu_info *target)
{

	sparc64_generic_xcall(target, (ipi_c_call_func_t)xc_ipi_handler, NULL);
}

void
cpu_ipi(struct cpu_info *target)
{

	sparc64_generic_xcall(target, (ipi_c_call_func_t)ipi_cpu_handler, NULL);
}