Date: Thu, 17 Feb 2005 20:23:14 GMT From: Peter Wemm <peter@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 71205 for review Message-ID: <200502172023.j1HKNEus040553@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=71205 Change 71205 by peter@peter_daintree on 2005/02/17 20:23:00 integ -b i386_hammer First pass at integrating local apic timer stuff. Affected files ... .. //depot/projects/hammer/sys/amd64/amd64/apic_vector.S#25 integrate .. //depot/projects/hammer/sys/amd64/amd64/io_apic.c#34 integrate .. //depot/projects/hammer/sys/amd64/amd64/legacy.c#20 integrate .. //depot/projects/hammer/sys/amd64/amd64/local_apic.c#43 integrate .. //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#81 integrate .. //depot/projects/hammer/sys/amd64/amd64/mp_watchdog.c#4 integrate .. //depot/projects/hammer/sys/amd64/amd64/mptable.c#38 integrate .. //depot/projects/hammer/sys/amd64/amd64/vm_machdep.c#66 integrate .. //depot/projects/hammer/sys/amd64/conf/NOTES#55 integrate .. //depot/projects/hammer/sys/amd64/include/apicvar.h#29 integrate .. //depot/projects/hammer/sys/amd64/include/sf_buf.h#5 integrate .. //depot/projects/hammer/sys/amd64/include/smp.h#22 integrate .. //depot/projects/hammer/sys/amd64/isa/clock.c#32 integrate Differences ... ==== //depot/projects/hammer/sys/amd64/amd64/apic_vector.S#25 (text+ko) ==== @@ -137,6 +137,23 @@ ISR_VEC(6, apic_isr6) ISR_VEC(7, apic_isr7) +/* + * Local APIC periodic timer handler. + */ + .text + SUPERALIGN_TEXT +IDTVEC(timerint) + PUSH_FRAME + + movq lapic, %rdx + movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */ + + FAKE_MCOUNT(TF_EIP(%esp)) + + call lapic_handle_timer + MEXITCOUNT + jmp doreti + #ifdef SMP /* * Global address space TLB shootdown. ==== //depot/projects/hammer/sys/amd64/amd64/io_apic.c#34 (text+ko) ==== @@ -66,7 +66,7 @@ #define TODO printf("%s: not implemented!\n", __func__) -MALLOC_DEFINE(M_IOAPIC, "I/O APIC", "I/O APIC structures"); +static MALLOC_DEFINE(M_IOAPIC, "I/O APIC", "I/O APIC structures"); /* * New interrupt support code.. ==== //depot/projects/hammer/sys/amd64/amd64/legacy.c#20 (text+ko) ==== ==== //depot/projects/hammer/sys/amd64/amd64/local_apic.c#43 (text+ko) ==== @@ -39,6 +39,7 @@ #include <sys/bus.h> #include <sys/kernel.h> #include <sys/pcpu.h> +#include <sys/smp.h> #include <sys/proc.h> #include <vm/vm.h> @@ -66,6 +67,10 @@ CTASSERT(APIC_LOCAL_INTS == 240); CTASSERT(IPI_STOP < APIC_SPURIOUS_INT); +#define LAPIC_TIMER_HZ_DIVIDER 3 +#define LAPIC_TIMER_STATHZ_DIVIDER 23 +#define LAPIC_TIMER_PROFHZ_DIVIDER 2 + /* * Support for local APICs. Local APICs manage interrupts on each * individual processor as opposed to I/O APICs which receive interrupts @@ -90,6 +95,10 @@ u_int la_cluster:4; u_int la_cluster_id:2; u_int la_present:1; + u_long *la_timer_count; + u_long la_hard_ticks; + u_long la_stat_ticks; + u_long la_prof_ticks; } static lapics[MAX_APICID]; /* XXX: should thermal be an NMI? */ @@ -115,9 +124,21 @@ IDTVEC(apic_isr7), /* 224 - 255 */ }; +static u_int32_t lapic_timer_divisors[] = { + APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, + APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 +}; + volatile lapic_t *lapic; +static u_long lapic_timer_divisor, lapic_timer_period, lapic_timer_hz; +static u_long *lapic_virtual_hardclock, *lapic_virtual_statclock, + *lapic_virtual_profclock; static void lapic_enable(void); +static void lapic_timer_enable_intr(void); +static void lapic_timer_oneshot(u_int count); +static void lapic_timer_periodic(u_int count); +static void lapic_timer_set_divisor(u_int divisor); static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); static uint32_t @@ -180,7 +201,10 @@ /* Set BSP's per-CPU local APIC ID. */ PCPU_SET(apic_id, lapic_id()); - /* XXX: timer/error/thermal interrupts */ + /* Local APIC timer interrupt. */ + setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYSIGT, SEL_KPL, 0); + + /* XXX: error/thermal interrupts */ } /* @@ -251,6 +275,7 @@ struct lapic *la; u_int32_t value, maxlvt; register_t eflags; + char buf[MAXCOMLEN + 1]; la = &lapics[lapic_id()]; KASSERT(la->la_present, ("missing APIC structure")); @@ -280,11 +305,84 @@ lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0); lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1); - /* XXX: more LVT entries */ + /* Program timer LVT and setup handler. */ + lapic->lvt_timer = lvt_mode(la, LVT_TIMER, lapic->lvt_timer); + snprintf(buf, sizeof(buf), "lapic%d: timer", lapic_id()); + intrcnt_add(buf, &la->la_timer_count); + if (PCPU_GET(cpuid) != 0) { + KASSERT(lapic_timer_period != 0, ("lapic%u: zero divisor", + lapic_id())); + lapic_timer_set_divisor(lapic_timer_divisor); + lapic_timer_periodic(lapic_timer_period); + lapic_timer_enable_intr(); + } + + /* XXX: Performance counter, error, and thermal LVTs */ intr_restore(eflags); } +/* + * Called by cpu_initclocks() on the BSP to setup the local APIC timer so + * that it can drive hardclock, statclock, and profclock. This function + * returns true if it is able to use the local APIC timer to drive the + * clocks and false if it is not able. + */ +int +lapic_setup_clock(void) +{ + u_long value; + + /* Can't drive the timer without a local APIC. */ + if (lapic == NULL) + return (0); + + /* If we've only got one CPU, then use the RTC and ISA timer instead. */ + if (mp_ncpus == 1) + return (0); + + /* Start off with a divisor of 2 (power on reset default). */ + lapic_timer_divisor = 2; + + /* Try to calibrate the local APIC timer. */ + do { + lapic_timer_set_divisor(lapic_timer_divisor); + lapic_timer_oneshot(APIC_TIMER_MAX_COUNT); + DELAY(2000000); + value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer; + if (value != APIC_TIMER_MAX_COUNT) + break; + lapic_timer_divisor <<= 1; + } while (lapic_timer_divisor <= 128); + if (lapic_timer_divisor > 128) + panic("lapic: Divisor too big"); + value /= 2; + if (bootverbose) + printf("lapic: Divisor %lu, Frequency %lu hz\n", + lapic_timer_divisor, value); + + /* + * We will drive the timer at a small multiple of hz and drive + * both of the other timers with similarly small but relatively + * prime divisors. + */ + lapic_timer_hz = hz * LAPIC_TIMER_HZ_DIVIDER; + stathz = lapic_timer_hz / LAPIC_TIMER_STATHZ_DIVIDER; + profhz = lapic_timer_hz / LAPIC_TIMER_PROFHZ_DIVIDER; + lapic_timer_period = value / lapic_timer_hz; + intrcnt_add("lapic: hardclock", &lapic_virtual_hardclock); + intrcnt_add("lapic: statclock", &lapic_virtual_statclock); + intrcnt_add("lapic: profclock", &lapic_virtual_profclock); + + /* + * Start up the timer on the BSP. The APs will kick off their + * timer during lapic_setup(). + */ + lapic_timer_periodic(lapic_timer_period); + lapic_timer_enable_intr(); + return (1); +} + void lapic_disable(void) { @@ -516,6 +614,91 @@ intr_execute_handlers(isrc, &frame); } +void +lapic_handle_timer(struct clockframe frame) +{ + struct lapic *la; + + la = &lapics[PCPU_GET(apic_id)]; + (*la->la_timer_count)++; + critical_enter(); + + /* Fire hardclock at hz. */ + la->la_hard_ticks += hz; + if (la->la_hard_ticks >= lapic_timer_hz) { + la->la_hard_ticks -= lapic_timer_hz; + if (PCPU_GET(cpuid) == 0) { + (*lapic_virtual_hardclock)++; + hardclock(&frame); + } else + hardclock_process(&frame); + } + + /* Fire statclock at stathz. */ + la->la_stat_ticks += stathz; + if (la->la_stat_ticks >= lapic_timer_hz) { + la->la_stat_ticks -= lapic_timer_hz; + if (PCPU_GET(cpuid) == 0) + (*lapic_virtual_statclock)++; + statclock(&frame); + } + + /* Fire profclock at profhz, but only when needed. */ + la->la_prof_ticks += profhz; + if (la->la_prof_ticks >= lapic_timer_hz) { + la->la_prof_ticks -= lapic_timer_hz; + if (PCPU_GET(cpuid) == 0) + (*lapic_virtual_profclock)++; + if (profprocs != 0) + profclock(&frame); + } + critical_exit(); +} + +static void +lapic_timer_set_divisor(u_int divisor) +{ + + KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); + KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) / + sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor)); + lapic->dcr_timer = lapic_timer_divisors[ffs(divisor) - 1]; +} + +static void +lapic_timer_oneshot(u_int count) +{ + u_int32_t value; + + value = lapic->lvt_timer; + value &= ~APIC_LVTT_TM; + value |= APIC_LVTT_TM_ONE_SHOT; + lapic->lvt_timer = value; + lapic->icr_timer = count; +} + +static void +lapic_timer_periodic(u_int count) +{ + u_int32_t value; + + value = lapic->lvt_timer; + value &= ~APIC_LVTT_TM; + value |= APIC_LVTT_TM_PERIODIC; + lapic->lvt_timer = value; + lapic->icr_timer = count; +} + +static void +lapic_timer_enable_intr(void) +{ + u_int32_t value; + + value = lapic->lvt_timer; + value &= ~APIC_LVT_M; + lapic->lvt_timer = value; +} + /* Translate between IDT vectors and IRQ vectors. */ u_int apic_irq_to_idt(u_int irq) ==== //depot/projects/hammer/sys/amd64/amd64/mp_machdep.c#81 (text+ko) ==== @@ -872,82 +872,15 @@ } -/* - * For statclock, we send an IPI to all CPU's to have them call this - * function. - */ - void -forward_statclock(void) -{ - int map; - - CTR0(KTR_SMP, "forward_statclock"); - - if (!smp_started || cold || panicstr) - return; - - map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); - if (map != 0) - ipi_selected(map, IPI_STATCLOCK); -} - -/* - * For each hardclock(), we send an IPI to all other CPU's to have them - * execute this function. It would be nice to reduce contention on - * sched_lock if we could simply peek at the CPU to determine the user/kernel - * state and call hardclock_process() on the CPU receiving the clock interrupt - * and then just use a simple IPI to handle any ast's if needed. - */ - -void -forward_hardclock(void) -{ - u_int map; - - CTR0(KTR_SMP, "forward_hardclock"); - - if (!smp_started || cold || panicstr) - return; - - map = PCPU_GET(other_cpus) & ~(stopped_cpus|hlt_cpus_mask); - if (map != 0) - ipi_selected(map, IPI_HARDCLOCK); -} - -void ipi_bitmap_handler(struct clockframe frame) { int cpu = PCPU_GET(cpuid); u_int ipi_bitmap; - struct thread *td; ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]); - critical_enter(); - /* Nothing to do for AST */ - - if (ipi_bitmap & (1 << IPI_HARDCLOCK)) { - td = curthread; - td->td_intr_nesting_level++; - hardclock_process(&frame); - td->td_intr_nesting_level--; - } - - if (ipi_bitmap & (1 << IPI_STATCLOCK)) { - CTR0(KTR_SMP, "forwarded_statclock"); - - td = curthread; - td->td_intr_nesting_level++; - if (profprocs != 0) - profclock(&frame); - if (pscnt == psdiv) - statclock(&frame); - td->td_intr_nesting_level--; - } - - critical_exit(); } /* ==== //depot/projects/hammer/sys/amd64/amd64/mp_watchdog.c#4 (text+ko) ==== @@ -49,11 +49,11 @@ #include <machine/mp_watchdog.h> /* - * mp_swatchdog hijacks the idle thread on a specified CPU, prevents new work + * mp_watchdog hijacks the idle thread on a specified CPU, prevents new work * from being scheduled there, and uses it as a "watchdog" to detect kernel * failure on other CPUs. This is made reasonable by inclusion of logical * processors in Xeon hardware. The watchdog is configured by setting the - * debug.watchdog_cpu sysctl to the CPU of interest. A callout will then + * debug.watchdog sysctl/tunable to the CPU of interest. A callout will then * begin executing reseting a timer that is gradually lowered by the watching * thread. If the timer reaches 0, the watchdog fires by ether dropping * directly to the debugger, or by sending an NMI IPI to the boot processor. @@ -68,11 +68,14 @@ static int watchdog_timer = -1; static int watchdog_nmi = 1; +TUNABLE_INT("debug.watchdog", &watchdog_cpu); SYSCTL_INT(_debug, OID_AUTO, watchdog_nmi, CTLFLAG_RW, &watchdog_nmi, 0, "IPI the boot processor with an NMI to enter the debugger"); static struct callout watchdog_callout; +static void watchdog_change(int wdcpu); + /* * Number of seconds before the watchdog will fire if the callout fails to * reset the timer. @@ -84,6 +87,8 @@ { callout_init(&watchdog_callout, CALLOUT_MPSAFE); + if (watchdog_cpu != -1) + watchdog_change(watchdog_cpu); } /* @@ -108,6 +113,27 @@ } SYSINIT(watchdog_init, SI_SUB_DRIVERS, SI_ORDER_ANY, watchdog_init, NULL); +static void +watchdog_change(int wdcpu) +{ + + if (wdcpu == -1 || wdcpu == 0xffffffff) { + /* + * Disable the watcdog. + */ + watchdog_cpu = -1; + watchdog_dontfire = 1; + callout_stop(&watchdog_callout); + printf("watchdog stopped\n"); + } else { + watchdog_timer = WATCHDOG_THRESHOLD; + watchdog_dontfire = 0; + watchdog_cpu = wdcpu; + callout_reset(&watchdog_callout, 1 * hz, watchdog_function, + NULL); + } +} + /* * This sysctl sets which CPU is the watchdog CPU. Set to -1 or 0xffffffff * to disable the watchdog. @@ -122,27 +148,12 @@ if (error) return (error); - if (req->newptr != NULL) { - if (temp == -1 || temp == 0xffffffff) { - /* - * Disable the watcdog. - */ - watchdog_cpu = -1; - watchdog_dontfire = 1; - callout_stop(&watchdog_callout); - printf("watchdog stopped\n"); - } else { - watchdog_timer = WATCHDOG_THRESHOLD; - watchdog_dontfire = 0; - watchdog_cpu = temp; - callout_reset(&watchdog_callout, 1 * hz, - watchdog_function, NULL); - } - } + if (req->newptr != NULL) + watchdog_change(temp); return (0); } SYSCTL_PROC(_debug, OID_AUTO, watchdog, CTLTYPE_INT|CTLFLAG_RW, 0, 0, - sysctl_watchdog, "IU", ""); + sysctl_watchdog, "I", ""); /* * A badly behaved sysctl that leaks the sched lock when written to. Then ==== //depot/projects/hammer/sys/amd64/amd64/mptable.c#38 (text+ko) ==== @@ -141,7 +141,7 @@ static int mptable_nioapics, mptable_nbusses, mptable_maxbusid; static int pci0 = -1; -MALLOC_DEFINE(M_MPTABLE, "MP Table", "MP Table Items"); +static MALLOC_DEFINE(M_MPTABLE, "MP Table", "MP Table Items"); static enum intr_polarity conforming_polarity(u_char src_bus, u_char src_bus_irq); ==== //depot/projects/hammer/sys/amd64/amd64/vm_machdep.c#66 (text+ko) ==== ==== //depot/projects/hammer/sys/amd64/conf/NOTES#55 (text+ko) ==== @@ -4,7 +4,7 @@ # This file contains machine dependent kernel configuration notes. For # machine independent notes, look in /sys/conf/NOTES. # -# (XXX from i386:NOTES,v 1.1179) +# (XXX from i386:NOTES,v 1.1180) # $FreeBSD: src/sys/amd64/conf/NOTES,v 1.21 2005/02/05 21:01:09 njl Exp $ # @@ -244,6 +244,9 @@ #XXX# still calls MD i386 kvtop function instead of vtophys etc #XXX#device ed +#XXX#options ED_3C503 +#XXX#options ED_HPP +#XXX#options ED_SIC #XXX#device lnc device ath @@ -253,8 +256,6 @@ device wlan # 802.11 layer # -# ATA raid adapters -# #XXX this stores pointers in a 32bit field that is defined by the hardware #device pst ==== //depot/projects/hammer/sys/amd64/include/apicvar.h#29 (text+ko) ==== @@ -123,9 +123,7 @@ /* IPIs handled by IPI_BITMAPED_VECTOR (XXX ups is there a better place?) */ #define IPI_AST 0 /* Generate software trap. */ -#define IPI_HARDCLOCK 1 /* Inter-CPU clock handling. */ -#define IPI_STATCLOCK 2 -#define IPI_BITMAP_LAST IPI_STATCLOCK +#define IPI_BITMAP_LAST IPI_AST #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST) #define IPI_STOP (APIC_IPI_INTS + 6) /* Stop CPU until restarted. */ @@ -172,7 +170,7 @@ inthand_t IDTVEC(apic_isr1), IDTVEC(apic_isr2), IDTVEC(apic_isr3), IDTVEC(apic_isr4), IDTVEC(apic_isr5), IDTVEC(apic_isr6), - IDTVEC(apic_isr7), IDTVEC(spuriousint); + IDTVEC(apic_isr7), IDTVEC(spuriousint), IDTVEC(timerint); u_int apic_irq_to_idt(u_int irq); u_int apic_idt_to_irq(u_int vector); @@ -203,6 +201,7 @@ void lapic_ipi_vectored(u_int vector, int dest); int lapic_ipi_wait(int delay); void lapic_handle_intr(void *cookie, struct intrframe frame); +void lapic_handle_timer(struct clockframe frame); void lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id); int lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked); int lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode); @@ -212,6 +211,7 @@ enum intr_trigger trigger); void lapic_set_tpr(u_int vector); void lapic_setup(void); +int lapic_setup_clock(void); #endif /* !LOCORE */ #endif /* _MACHINE_APICVAR_H_ */ ==== //depot/projects/hammer/sys/amd64/include/sf_buf.h#5 (text+ko) ==== @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2003 Alan L. Cox <alc@cs.rice.edu> + * Copyright (c) 2003, 2005 Alan L. Cox <alc@cs.rice.edu> * All rights reserved. * * Redistribution and use in source and binary forms, with or without ==== //depot/projects/hammer/sys/amd64/include/smp.h#22 (text+ko) ==== @@ -51,8 +51,6 @@ void ipi_all(u_int ipi); void ipi_all_but_self(u_int ipi); void ipi_self(u_int ipi); -void forward_statclock(void); -void forward_hardclock(void); void ipi_bitmap_handler(struct clockframe frame); u_int mp_bootaddress(u_int); int mp_grab_cpu_hlt(void); ==== //depot/projects/hammer/sys/amd64/isa/clock.c#32 (text+ko) ==== @@ -70,9 +70,7 @@ #include <machine/intr_machdep.h> #include <machine/md_var.h> #include <machine/psl.h> -#ifdef SMP -#include <machine/smp.h> -#endif +#include <machine/apicvar.h> #include <machine/specialreg.h> #include <amd64/isa/isa.h> @@ -113,6 +111,7 @@ static u_int32_t i8254_offset; static int (*i8254_pending)(struct intsrc *); static int i8254_ticked; +static int using_lapic_timer; static u_char rtc_statusa = RTCSA_DIVIDER | RTCSA_NOPROF; static u_char rtc_statusb = RTCSB_24HR | RTCSB_PINTR; @@ -151,10 +150,8 @@ clkintr_pending = 0; mtx_unlock_spin(&clock_lock); } - hardclock(frame); -#ifdef SMP - forward_hardclock(); -#endif + if (!using_lapic_timer) + hardclock(frame); } int @@ -221,9 +218,6 @@ } if (pscnt == psdiv) statclock(frame); -#ifdef SMP - forward_statclock(); -#endif } } @@ -730,7 +724,8 @@ { int diag; - if (statclock_disable) { + using_lapic_timer = lapic_setup_clock(); + if (statclock_disable || using_lapic_timer) { /* * The stat interrupt mask is different without the * statistics clock. Also, don't set the interrupt @@ -756,7 +751,7 @@ writertc(RTC_STATUSB, RTCSB_24HR); /* Don't bother enabling the statistics clock. */ - if (!statclock_disable) { + if (!statclock_disable && !using_lapic_timer) { diag = rtcin(RTC_DIAG); if (diag != 0) printf("RTC BIOS diagnostic error %b\n", diag, RTCDG_BITS); @@ -775,6 +770,8 @@ cpu_startprofclock(void) { + if (using_lapic_timer) + return; rtc_statusa = RTCSA_DIVIDER | RTCSA_PROF; writertc(RTC_STATUSA, rtc_statusa); psdiv = pscnt = psratio; @@ -784,6 +781,8 @@ cpu_stopprofclock(void) { + if (using_lapic_timer) + return; rtc_statusa = RTCSA_DIVIDER | RTCSA_NOPROF; writertc(RTC_STATUSA, rtc_statusa); psdiv = pscnt = 1;
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200502172023.j1HKNEus040553>