/* $NetBSD: tprof_x86_amd.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $ */ /* * Copyright (c) 2018 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Maxime Villard. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* * Copyright (c)2008,2009 YAMAMOTO Takashi, * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __KERNEL_RCSID(0, "$NetBSD: tprof_x86_amd.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $"); #include #include #include #include #include #include #include #include #include /* VM_MIN_KERNEL_ADDRESS */ #include #include #include /* CPUVENDOR_* */ #include /* cpu_vendor */ #include #include #define NCTRS 4 #define COUNTER_BITWIDTH 48 #define PERFEVTSEL(i) (0xc0010000 + (i)) #define PERFCTR(i) (0xc0010004 + (i)) #define PESR_EVENT_MASK_LO __BITS(0, 7) #define PESR_UNIT_MASK __BITS(8, 15) #define PESR_USR __BIT(16) #define PESR_OS __BIT(17) #define PESR_E __BIT(18) #define PESR_PC __BIT(19) #define PESR_INT __BIT(20) /* bit 21 reserved */ #define PESR_EN __BIT(22) #define PESR_INV __BIT(23) #define PESR_COUNTER_MASK __BITS(24, 31) #define PESR_EVENT_MASK_HI __BITS(32, 35) /* bit 36-39 reserved */ #define PESR_GO __BIT(40) #define PESR_HO __BIT(41) /* bit 42-63 reserved */ /* * Documents: * http://support.amd.com/TechDocs/32559.pdf * http://developer.amd.com/wordpress/media/2012/10/Basic_Performance_Measurements.pdf */ static uint32_t amd_lapic_saved[MAXCPUS]; static nmi_handler_t *amd_nmi_handle; static uint32_t tprof_amd_ncounters(void) { return NCTRS; } static u_int tprof_amd_counter_bitwidth(u_int counter) { return COUNTER_BITWIDTH; } static inline void tprof_amd_counter_write(u_int counter, uint64_t val) { wrmsr(PERFCTR(counter), val); } static inline uint64_t tprof_amd_counter_read(u_int counter) { return rdmsr(PERFCTR(counter)); } static void tprof_amd_configure_event(u_int counter, const tprof_param_t *param) { uint64_t pesr; uint64_t event_lo; uint64_t event_hi; event_hi = param->p_event >> 8; event_lo = param->p_event & 0xff; pesr = ((param->p_flags & TPROF_PARAM_USER) ? PESR_USR : 0) | ((param->p_flags & TPROF_PARAM_KERN) ? PESR_OS : 0) | PESR_INT | __SHIFTIN(event_lo, PESR_EVENT_MASK_LO) | __SHIFTIN(event_hi, PESR_EVENT_MASK_HI) | __SHIFTIN(0, PESR_COUNTER_MASK) | __SHIFTIN(param->p_unit, PESR_UNIT_MASK); wrmsr(PERFEVTSEL(counter), pesr); /* Reset the counter */ tprof_amd_counter_write(counter, param->p_value); } static void tprof_amd_start(tprof_countermask_t runmask) { int bit; while ((bit = ffs(runmask)) != 0) { bit--; CLR(runmask, __BIT(bit)); wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) | PESR_EN); } } static void tprof_amd_stop(tprof_countermask_t stopmask) { int bit; while ((bit = ffs(stopmask)) != 0) { bit--; CLR(stopmask, __BIT(bit)); wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) & ~PESR_EN); } } static int tprof_amd_nmi(const struct trapframe *tf, void *arg) { tprof_backend_softc_t *sc = arg; tprof_frame_info_t tfi; int bit; uint64_t *counters_offset = percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu()); tprof_countermask_t mask = sc->sc_ctr_ovf_mask; while ((bit = ffs(mask)) != 0) { bit--; CLR(mask, __BIT(bit)); /* If the highest bit is non zero, then it's not for us. */ uint64_t ctr = tprof_amd_counter_read(bit); if ((ctr & __BIT(COUNTER_BITWIDTH - 1)) != 0) continue; /* not overflowed */ if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) { /* Account for the counter, and reset */ tprof_amd_counter_write(bit, sc->sc_count[bit].ctr_counter_reset_val); counters_offset[bit] += sc->sc_count[bit].ctr_counter_val + ctr; /* Record a sample */ #if defined(__x86_64__) tfi.tfi_pc = tf->tf_rip; #else tfi.tfi_pc = tf->tf_eip; #endif tfi.tfi_counter = bit; tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS; tprof_sample(NULL, &tfi); } else { /* Not profiled, but require to consider overflow */ counters_offset[bit] += __BIT(COUNTER_BITWIDTH); } } return 1; } static uint64_t tprof_amd_counter_estimate_freq(u_int counter) { return curcpu()->ci_data.cpu_cc_freq; } static uint32_t tprof_amd_ident(void) { struct cpu_info *ci = curcpu(); if (cpu_vendor != CPUVENDOR_AMD) return TPROF_IDENT_NONE; switch (CPUID_TO_FAMILY(ci->ci_signature)) { case 0x10: case 0x15: case 0x17: case 0x19: return TPROF_IDENT_AMD_GENERIC; } return TPROF_IDENT_NONE; } static void tprof_amd_establish_cpu(void *arg1, void *arg2) { struct cpu_info * const ci = curcpu(); amd_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT); lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI); } static void tprof_amd_disestablish_cpu(void *arg1, void *arg2) { struct cpu_info * const ci = curcpu(); lapic_writereg(LAPIC_LVT_PCINT, amd_lapic_saved[cpu_index(ci)]); } static int tprof_amd_establish(tprof_backend_softc_t *sc) { uint64_t xc; if (tprof_amd_ident() == TPROF_IDENT_NONE) return ENOTSUP; KASSERT(amd_nmi_handle == NULL); amd_nmi_handle = nmi_establish(tprof_amd_nmi, sc); xc = xc_broadcast(0, tprof_amd_establish_cpu, sc, NULL); xc_wait(xc); return 0; } static void tprof_amd_disestablish(tprof_backend_softc_t *sc) { uint64_t xc; xc = xc_broadcast(0, tprof_amd_disestablish_cpu, sc, NULL); xc_wait(xc); KASSERT(amd_nmi_handle != NULL); nmi_disestablish(amd_nmi_handle); amd_nmi_handle = NULL; } const tprof_backend_ops_t tprof_amd_ops = { .tbo_ident = tprof_amd_ident, .tbo_ncounters = tprof_amd_ncounters, .tbo_counter_bitwidth = tprof_amd_counter_bitwidth, .tbo_counter_read = tprof_amd_counter_read, .tbo_counter_estimate_freq = tprof_amd_counter_estimate_freq, .tbo_valid_event = NULL, .tbo_configure_event = tprof_amd_configure_event, .tbo_start = tprof_amd_start, .tbo_stop = tprof_amd_stop, .tbo_establish = tprof_amd_establish, .tbo_disestablish = tprof_amd_disestablish, };