Date: Sun, 6 Jan 2013 04:58:48 +0000 (UTC) From: Neel Natu <neel@FreeBSD.org> To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r245099 - in projects/x2apic/sys: amd64/amd64 amd64/include x86/x86 Message-ID: <201301060458.r064wmKb004372@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: neel Date: Sun Jan 6 04:58:47 2013 New Revision: 245099 URL: http://svnweb.freebsd.org/changeset/base/245099 Log: Add support for x2apic mode of accessing the local apic. The default setting is to enable the x2apic only if executing inside a virtual machine. However this may be controlled by using the 'machdep.x2apic_desired' tunable. The sysctl 'machdep.x2apic' can be inspected to determine whether or not the kernel has selected x2apic mode. Obtained from: NetApp (by way of projects/bhyve) Modified: projects/x2apic/sys/amd64/amd64/apic_vector.S projects/x2apic/sys/amd64/amd64/mp_machdep.c projects/x2apic/sys/amd64/include/apicvar.h projects/x2apic/sys/x86/x86/local_apic.c Modified: projects/x2apic/sys/amd64/amd64/apic_vector.S ============================================================================== --- projects/x2apic/sys/amd64/amd64/apic_vector.S Sun Jan 6 04:40:07 2013 (r245098) +++ projects/x2apic/sys/amd64/amd64/apic_vector.S Sun Jan 6 04:58:47 2013 (r245099) @@ -57,8 +57,15 @@ IDTVEC(vec_name) ; \ PUSH_FRAME ; \ FAKE_MCOUNT(TF_RIP(%rsp)) ; \ movq lapic, %rdx ; /* pointer to local APIC */ \ + testq %rdx, %rdx; \ + jnz 3f; \ + movl $MSR_APIC_ISR ## index, %ecx; \ + rdmsr; \ + jmp 4f; \ +3: ; \ movl LA_ISR + 16 * (index)(%rdx), %eax ; /* load ISR */ \ - bsrl %eax, %eax ; /* index of highest set bit in ISR */ \ +4: ; \ + bsrl %eax, %eax ; /* index of highset set bit in ISR */ \ jz 1f ; \ addl $(32 * index),%eax ; \ movq %rsp, %rsi ; \ @@ -129,6 +136,26 @@ IDTVEC(errorint) jmp doreti #ifdef SMP + +/* + * We assume that %rax is being saved/restored outside of this macro + */ +#define DO_EOI \ + movq lapic, %rax; \ + testq %rax, %rax; \ + jz 8f; \ + movl $0, LA_EOI(%rax); \ + jmp 9f; \ +8:; \ + pushq %rcx; \ + pushq %rdx; \ + xorl %edx, %edx; /* eax is already zero */ \ + movl $MSR_APIC_EOI, %ecx; \ + wrmsr; \ + popq %rdx; \ + popq %rcx; \ +9: + /* * Global address space TLB shootdown. */ @@ -153,8 +180,7 @@ IDTVEC(invltlb) movq %cr3, %rax /* invalidate the TLB */ movq %rax, %cr3 - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI lock incl smp_tlb_wait @@ -186,8 +212,7 @@ IDTVEC(invlpg) movq smp_tlb_addr1, %rax invlpg (%rax) /* invalidate single page */ - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI lock incl smp_tlb_wait @@ -224,8 +249,7 @@ IDTVEC(invlrng) cmpq %rax, %rdx jb 1b - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI lock incl smp_tlb_wait @@ -252,8 +276,7 @@ IDTVEC(invlcache) wbinvd - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI lock incl smp_tlb_wait @@ -269,9 +292,8 @@ IDTVEC(invlcache) IDTVEC(ipi_intr_bitmap_handler) PUSH_FRAME - movq lapic, %rdx - movl $0, LA_EOI(%rdx) /* End Of Interrupt to APIC */ - + DO_EOI + FAKE_MCOUNT(TF_RIP(%rsp)) call ipi_bitmap_handler @@ -286,8 +308,7 @@ IDTVEC(ipi_intr_bitmap_handler) IDTVEC(cpustop) PUSH_FRAME - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI call cpustop_handler jmp doreti @@ -301,8 +322,7 @@ IDTVEC(cpususpend) PUSH_FRAME call cpususpend_handler - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI jmp doreti /* @@ -320,7 +340,6 @@ IDTVEC(rendezvous) incq (%rax) #endif call smp_rendezvous_action - movq lapic, %rax - movl $0, LA_EOI(%rax) /* End Of Interrupt to APIC */ + DO_EOI jmp doreti #endif /* SMP */ Modified: projects/x2apic/sys/amd64/amd64/mp_machdep.c ============================================================================== --- projects/x2apic/sys/amd64/amd64/mp_machdep.c Sun Jan 6 04:40:07 2013 (r245098) +++ projects/x2apic/sys/amd64/amd64/mp_machdep.c Sun Jan 6 04:58:47 2013 (r245099) @@ -688,6 +688,8 @@ init_secondary(void) wrmsr(MSR_STAR, msr); wrmsr(MSR_SF_MASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D); + lapic_init_ap(); + /* Disable local APIC just to be sure. */ lapic_disable(); Modified: projects/x2apic/sys/amd64/include/apicvar.h ============================================================================== --- projects/x2apic/sys/amd64/include/apicvar.h Sun Jan 6 04:40:07 2013 (r245098) +++ projects/x2apic/sys/amd64/include/apicvar.h Sun Jan 6 04:58:47 2013 (r245099) @@ -209,6 +209,7 @@ int lapic_enable_pmc(void); void lapic_eoi(void); int lapic_id(void); void lapic_init(vm_paddr_t addr); +void lapic_init_ap(void); int lapic_intr_pending(u_int vector); void lapic_ipi_raw(register_t icrlo, u_int dest); void lapic_ipi_vectored(u_int vector, int dest); Modified: projects/x2apic/sys/x86/x86/local_apic.c ============================================================================== --- projects/x2apic/sys/x86/x86/local_apic.c Sun Jan 6 04:40:07 2013 (r245098) +++ projects/x2apic/sys/x86/x86/local_apic.c Sun Jan 6 04:58:47 2013 (r245099) @@ -50,12 +50,14 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/sched.h> #include <sys/smp.h> +#include <sys/sysctl.h> #include <sys/timeet.h> #include <vm/vm.h> #include <vm/pmap.h> #include <x86/apicreg.h> +#include <machine/atomic.h> #include <machine/cpu.h> #include <machine/cputypes.h> #include <machine/frame.h> @@ -159,6 +161,15 @@ vm_paddr_t lapic_paddr; static u_long lapic_timer_divisor; static struct eventtimer lapic_et; +static int x2apic; +SYSCTL_INT(_machdep, OID_AUTO, x2apic, CTLFLAG_RD, &x2apic, 0, "x2apic mode"); + +static int x2apic_desired = -1; /* enable only if running in a VM */ +TUNABLE_INT("machdep.x2apic_desired", &x2apic_desired); +SYSCTL_INT(_machdep, OID_AUTO, x2apic_desired, CTLFLAG_RDTUN, + &x2apic_desired, 0, + "0 (disable), 1 (enable), -1 (leave it up to the kernel)"); + static void lapic_enable(void); static void lapic_resume(struct pic *pic); static void lapic_timer_oneshot(struct lapic *, @@ -171,6 +182,37 @@ static uint32_t lvt_mode(struct lapic *l static int lapic_et_start(struct eventtimer *et, struct bintime *first, struct bintime *period); static int lapic_et_stop(struct eventtimer *et); +static uint32_t lapic_version(void); +static uint32_t lapic_ldr(void); +static uint32_t lapic_dfr(void); +static uint32_t lapic_lvt_lint0(void); +static void lapic_set_lvt_lint0(uint32_t value); +static uint32_t lapic_lvt_lint1(void); +static void lapic_set_lvt_lint1(uint32_t value); +static uint32_t lapic_tpr(void); +static uint32_t lapic_svr(void); +static void lapic_set_svr(uint32_t value); +static uint32_t lapic_lvt_timer(void); +static void lapic_set_lvt_timer(uint32_t value); +static uint32_t lapic_lvt_thermal(void); +static uint32_t lapic_lvt_error(void); +static void lapic_set_lvt_error(uint32_t value); +static uint32_t lapic_lvt_pcint(void); +static void lapic_set_lvt_pcint(uint32_t value); +static uint32_t lapic_lvt_cmci(void); +static void lapic_set_lvt_cmci(uint32_t value); +static uint32_t lapic_esr(void); +static void lapic_set_esr(uint32_t value); +static uint32_t lapic_ccr_timer(void); +static void lapic_set_dcr_timer(uint32_t value); +static void lapic_set_icr_timer(uint32_t value); +uint32_t lapic_irr(int num); +uint32_t lapic_tmr(int num); +uint32_t lapic_isr(int num); +static uint32_t lapic_icr_lo(void); +static uint32_t lapic_icr_hi(void); +static void lapic_set_icr(uint64_t value); +static boolean_t lapic_missing(void); struct pic lapic_pic = { .pic_resume = lapic_resume }; @@ -215,6 +257,17 @@ lvt_mode(struct lapic *la, u_int pin, ui return (value); } +static void +x2apic_init(void) +{ + uint64_t apic_base; + + apic_base = rdmsr(MSR_APICBASE); + + if ((apic_base & APICBASE_X2APIC) == 0) + wrmsr(MSR_APICBASE, apic_base | APICBASE_X2APIC); +} + /* * Map the local APIC and setup necessary interrupt vectors. */ @@ -224,11 +277,29 @@ lapic_init(vm_paddr_t addr) u_int regs[4]; int i, arat; - /* Map the local APIC and setup the spurious interrupt handler. */ - KASSERT(trunc_page(addr) == addr, - ("local APIC not aligned on a page boundary")); - lapic = pmap_mapdev(addr, sizeof(lapic_t)); - lapic_paddr = addr; + if ((cpu_feature2 & CPUID2_X2APIC) != 0) { + if (rdmsr(MSR_APICBASE) & APICBASE_X2APIC) + x2apic = 1; + else if (x2apic_desired != 0) { + /* + * The default behavior is to enable x2apic only if + * the kernel is executing inside a virtual machine. + */ + if (vm_guest != VM_GUEST_NO || x2apic_desired == 1) + x2apic = 1; + } + } + + if (x2apic) { + x2apic_init(); + if (bootverbose) + printf("Local APIC access using x2APIC MSRs\n"); + } else { + KASSERT(trunc_page(addr) == addr, + ("local APIC not aligned on a page boundary")); + lapic = pmap_mapdev(addr, sizeof(lapic_t)); + lapic_paddr = addr; + } setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL, GSEL_APIC); @@ -279,6 +350,14 @@ lapic_init(vm_paddr_t addr) } } +void +lapic_init_ap(void) +{ + + if (x2apic) + x2apic_init(); +} + /* * Create a local APIC instance. */ @@ -330,19 +409,19 @@ lapic_dump(const char* str) { uint32_t maxlvt; - maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + maxlvt = (lapic_version() & APIC_VER_MAXLVT) >> MAXLVTSHIFT; printf("cpu%d %s:\n", PCPU_GET(cpuid), str); printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n", - lapic->id, lapic->version, lapic->ldr, lapic->dfr); + lapic_id(), lapic_version(), lapic_ldr(), lapic_dfr()); printf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", - lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr); + lapic_lvt_lint0(), lapic_lvt_lint1(), lapic_tpr(), lapic_svr()); printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x", - lapic->lvt_timer, lapic->lvt_thermal, lapic->lvt_error); + lapic_lvt_timer(), lapic_lvt_thermal(), lapic_lvt_error()); if (maxlvt >= LVT_PMC) - printf(" pmc: 0x%08x", lapic->lvt_pcint); + printf(" pmc: 0x%08x", lapic_lvt_pcint()); printf("\n"); if (maxlvt >= LVT_CMCI) - printf(" cmci: 0x%08x\n", lapic->lvt_cmci); + printf(" cmci: 0x%08x\n", lapic_lvt_cmci()); } void @@ -356,7 +435,7 @@ lapic_setup(int boot) la = &lapics[lapic_id()]; KASSERT(la->la_present, ("missing APIC structure")); saveintr = intr_disable(); - maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + maxlvt = (lapic_version() & APIC_VER_MAXLVT) >> MAXLVTSHIFT; /* Initialize the TPR to allow all interrupts. */ lapic_set_tpr(0); @@ -365,16 +444,16 @@ lapic_setup(int boot) lapic_enable(); /* Program LINT[01] LVT entries. */ - lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0); - lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1); + lapic_set_lvt_lint0(lvt_mode(la, LVT_LINT0, lapic_lvt_lint0())); + lapic_set_lvt_lint1(lvt_mode(la, LVT_LINT1, lapic_lvt_lint1())); /* Program the PMC LVT entry if present. */ if (maxlvt >= LVT_PMC) - lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); + lapic_set_lvt_pcint(lvt_mode(la, LVT_PMC, lapic_lvt_pcint())); /* Program timer LVT and setup handler. */ - la->lvt_timer_cache = lapic->lvt_timer = - lvt_mode(la, LVT_TIMER, lapic->lvt_timer); + la->lvt_timer_cache = lvt_mode(la, LVT_TIMER, lapic_lvt_timer()); + lapic_set_lvt_timer(la->lvt_timer_cache); if (boot) { snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid)); intrcnt_add(buf, &la->la_timer_count); @@ -392,14 +471,14 @@ lapic_setup(int boot) } /* Program error LVT and clear any existing errors. */ - lapic->lvt_error = lvt_mode(la, LVT_ERROR, lapic->lvt_error); - lapic->esr = 0; + lapic_set_lvt_error(lvt_mode(la, LVT_ERROR, lapic_lvt_error())); + lapic_set_esr(0); /* XXX: Thermal LVT */ /* Program the CMCI LVT entry if present. */ if (maxlvt >= LVT_CMCI) - lapic->lvt_cmci = lvt_mode(la, LVT_CMCI, lapic->lvt_cmci); + lapic_set_lvt_cmci(lvt_mode(la, LVT_CMCI, lapic_lvt_cmci())); intr_restore(saveintr); } @@ -410,9 +489,9 @@ lapic_reenable_pmc(void) #ifdef HWPMC_HOOKS uint32_t value; - value = lapic->lvt_pcint; + value = lapic_lvt_pcint(); value &= ~APIC_LVT_M; - lapic->lvt_pcint = value; + lapic_set_lvt_pcint(value); #endif } @@ -423,7 +502,7 @@ lapic_update_pmc(void *dummy) struct lapic *la; la = &lapics[lapic_id()]; - lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); + lapic_set_lvt_pcint(lvt_mode(la, LVT_PMC, lapic_lvt_pcint())); } #endif @@ -434,11 +513,11 @@ lapic_enable_pmc(void) u_int32_t maxlvt; /* Fail if the local APIC is not present. */ - if (lapic == NULL) + if (lapic_missing()) return (0); /* Fail if the PMC LVT is not present. */ - maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + maxlvt = (lapic_version() & APIC_VER_MAXLVT) >> MAXLVTSHIFT; if (maxlvt < LVT_PMC) return (0); @@ -468,11 +547,11 @@ lapic_disable_pmc(void) u_int32_t maxlvt; /* Fail if the local APIC is not present. */ - if (lapic == NULL) + if (lapic_missing()) return; /* Fail if the PMC LVT is not present. */ - maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; + maxlvt = (lapic_version() & APIC_VER_MAXLVT) >> MAXLVTSHIFT; if (maxlvt < LVT_PMC) return; @@ -502,7 +581,7 @@ lapic_et_start(struct eventtimer *et, lapic_timer_set_divisor(lapic_timer_divisor); lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0); DELAY(1000000); - value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer; + value = APIC_TIMER_MAX_COUNT - lapic_ccr_timer(); if (value != APIC_TIMER_MAX_COUNT) break; lapic_timer_divisor <<= 1; @@ -556,9 +635,9 @@ lapic_disable(void) uint32_t value; /* Software disable the local APIC. */ - value = lapic->svr; + value = lapic_svr(); value &= ~APIC_SVR_SWEN; - lapic->svr = value; + lapic_set_svr(value); } static void @@ -567,10 +646,10 @@ lapic_enable(void) u_int32_t value; /* Program the spurious vector to enable the local APIC. */ - value = lapic->svr; + value = lapic_svr(); value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT); - lapic->svr = value; + lapic_set_svr(value); } /* Reset the local APIC on the BSP during resume. */ @@ -581,19 +660,369 @@ lapic_resume(struct pic *pic) lapic_setup(0); } +static uint32_t +lapic_version(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_VERSION)); + else + return (lapic->version); +} + +static uint32_t +lapic_ldr(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LDR)); + else + return (lapic->ldr); +} + +static uint32_t +lapic_dfr(void) +{ + + if (x2apic) + return (0xffffffff); /* DFR not available in x2APIC mode */ + else + return (lapic->dfr); +} + +static uint32_t +lapic_lvt_lint0(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_LINT0)); + else + return (lapic->lvt_lint0); +} + +static void +lapic_set_lvt_lint0(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_LVT_LINT0, value); + else + lapic->lvt_lint0 = value; +} + +static uint32_t +lapic_lvt_lint1(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_LINT1)); + else + return (lapic->lvt_lint1); +} + +static void +lapic_set_lvt_lint1(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_LVT_LINT1, value); + else + lapic->lvt_lint1 = value; +} + +static uint32_t +lapic_tpr(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_TPR)); + else + return (lapic->tpr); +} + +static uint32_t +lapic_svr(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_SVR)); + else + return (lapic->svr); +} + +static void +lapic_set_svr(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_SVR, value); + else + lapic->svr = value; +} + +static uint32_t +lapic_lvt_timer(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_TIMER)); + else + return (lapic->lvt_timer); +} + +static void +lapic_set_lvt_timer(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_LVT_TIMER, value); + else + lapic->lvt_timer = value; +} + +static uint32_t +lapic_lvt_thermal(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_THERMAL)); + else + return (lapic->lvt_thermal); +} + +static uint32_t +lapic_lvt_error(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_ERROR)); + else + return (lapic->lvt_error); +} + +static void +lapic_set_lvt_error(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_LVT_ERROR, value); + else + lapic->lvt_error = value; +} + +static uint32_t +lapic_lvt_pcint(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_PCINT)); + else + return (lapic->lvt_pcint); +} + +static void +lapic_set_lvt_pcint(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_LVT_PCINT, value); + else + lapic->lvt_pcint = value; +} + +static uint32_t +lapic_lvt_cmci(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_LVT_CMCI)); + else + return (lapic->lvt_cmci); +} + +static void +lapic_set_lvt_cmci(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_LVT_CMCI, value); + else + lapic->lvt_cmci = value; +} + +static uint32_t +lapic_esr(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_ESR)); + else + return (lapic->esr); +} + +static void +lapic_set_esr(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_ESR, value); + else + lapic->esr = value; +} + +static uint32_t +lapic_ccr_timer(void) +{ + + if (x2apic) + return (rdmsr(MSR_APIC_CCR_TIMER)); + else + return (lapic->ccr_timer); +} + +static void +lapic_set_dcr_timer(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_DCR_TIMER, value); + else + lapic->dcr_timer = value; +} + +static void +lapic_set_icr_timer(uint32_t value) +{ + + if (x2apic) + wrmsr(MSR_APIC_ICR_TIMER, value); + else + lapic->icr_timer = value; +} + +uint32_t +lapic_tmr(int num) +{ + int msr; + volatile uint32_t *regptr; + + KASSERT(num >= 0 && num < 8, ("lapic_tmr: invalid num %d", num)); + + if (x2apic) { + msr = MSR_APIC_TMR0 + num; + return (rdmsr(msr)); + } else { + regptr = &lapic->tmr0; + return (regptr[num * 4]); + } +} + +uint32_t +lapic_irr(int num) +{ + int msr; + volatile uint32_t *regptr; + + KASSERT(num >= 0 && num < 8, ("lapic_irr: invalid num %d", num)); + + if (x2apic) { + msr = MSR_APIC_IRR0 + num; + return (rdmsr(msr)); + } else { + regptr = &lapic->irr0; + return (regptr[num * 4]); + } +} + +uint32_t +lapic_isr(int num) +{ + int msr; + volatile uint32_t *regptr; + + KASSERT(num >= 0 && num < 8, ("lapic_isr: invalid num %d", num)); + + if (x2apic) { + msr = MSR_APIC_ISR0 + num; + return (rdmsr(msr)); + } else { + regptr = &lapic->isr0; + return (regptr[num * 4]); + } +} + +static uint32_t +lapic_icr_lo(void) +{ + + if (x2apic) + return (0); + else + return (lapic->icr_lo); +} + +static uint32_t +lapic_icr_hi(void) +{ + + if (x2apic) + return (0); + else + return (lapic->icr_hi); +} + +static void +lapic_set_icr(uint64_t value) +{ + + /* + * Access to x2apic MSR registers is not a serializing condition. + * + * A number of IPI handlers (e.g. rendezvous, tlb shootdown) + * depend on shared state in memory between the cpu that + * originated the IPI and the cpus that are the target. + * + * Insert a memory barrier to ensure that changes to memory + * are globally visible to the other cpus. + */ + if (x2apic) { + /* + * XXX + * Intel's architecture spec seems to suggest that an + * "sfence" should be sufficient here but empirically + * an "mfence" is required to do the job. + */ + mb(); + wrmsr(MSR_APIC_ICR, value); + } else { + lapic->icr_hi = value >> 32; + lapic->icr_lo = value; + } +} + +static boolean_t +lapic_missing(void) +{ + + if (x2apic == 0 && lapic == NULL) + return (TRUE); + else + return (FALSE); +} + int lapic_id(void) { - KASSERT(lapic != NULL, ("local APIC is not mapped")); - return (lapic->id >> APIC_ID_SHIFT); + if (x2apic) + return (rdmsr(MSR_APIC_ID)); + else + return (lapic->id >> APIC_ID_SHIFT); } int lapic_intr_pending(u_int vector) { - volatile u_int32_t *irr; - /* * The IRR registers are an array of 128-bit registers each of * which only describes 32 interrupts in the low 32 bits.. Thus, @@ -603,8 +1032,7 @@ lapic_intr_pending(u_int vector) * modulus the vector by 32 to determine the individual bit to * test. */ - irr = &lapic->irr0; - return (irr[(vector / 32) * 4] & 1 << (vector % 32)); + return (lapic_irr(vector / 32) & 1 << (vector % 32)); } void @@ -760,13 +1188,19 @@ void lapic_set_tpr(u_int vector) { #ifdef CHEAP_TPR - lapic->tpr = vector; + if (x2apic) + wrmsr(MSR_APIC_TPR, vector); + else + lapic->tpr = vector; #else u_int32_t tpr; - tpr = lapic->tpr & ~APIC_TPR_PRIO; + tpr = lapic_tpr() & ~APIC_TPR_PRIO; tpr |= vector; - lapic->tpr = tpr; + if (x2apic) + wrmsr(MSR_APIC_TPR, tpr); + else + lapic->tpr = tpr; #endif } @@ -774,7 +1208,10 @@ void lapic_eoi(void) { - lapic->eoi = 0; + if (x2apic) + wrmsr(MSR_APIC_EOI, 0); + else + lapic->eoi = 0; } void @@ -836,7 +1273,7 @@ lapic_timer_set_divisor(u_int divisor) KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) / sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor)); - lapic->dcr_timer = lapic_timer_divisors[ffs(divisor) - 1]; + lapic_set_dcr_timer(lapic_timer_divisors[ffs(divisor) - 1]); } static void @@ -849,8 +1286,8 @@ lapic_timer_oneshot(struct lapic *la, u_ value |= APIC_LVTT_TM_ONE_SHOT; if (enable_int) value &= ~APIC_LVT_M; - lapic->lvt_timer = value; - lapic->icr_timer = count; + lapic_set_lvt_timer(value); + lapic_set_icr_timer(count); } static void @@ -863,8 +1300,8 @@ lapic_timer_periodic(struct lapic *la, u value |= APIC_LVTT_TM_PERIODIC; if (enable_int) value &= ~APIC_LVT_M; - lapic->lvt_timer = value; - lapic->icr_timer = count; + lapic_set_lvt_timer(value); + lapic_set_icr_timer(count); } static void @@ -875,7 +1312,7 @@ lapic_timer_stop(struct lapic *la) value = la->lvt_timer_cache; value &= ~APIC_LVTT_TM; value |= APIC_LVT_M; - lapic->lvt_timer = value; + lapic_set_lvt_timer(value); } void @@ -921,8 +1358,8 @@ lapic_handle_error(void) * to update its value to indicate any errors that have * occurred since the previous write to the register. */ - lapic->esr = 0; - esr = lapic->esr; + lapic_set_esr(0); + esr = lapic_esr(); printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); lapic_eoi(); @@ -1190,17 +1627,17 @@ DB_SHOW_COMMAND(lapic, db_show_lapic) uint32_t v; db_printf("lapic ID = %d\n", lapic_id()); - v = lapic->version; + v = lapic_version(); db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, v & 0xf); db_printf("max LVT = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT); - v = lapic->svr; + v = lapic_svr(); db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, v & APIC_SVR_ENABLE ? "enabled" : "disabled"); - db_printf("TPR = %02x\n", lapic->tpr); + db_printf("TPR = %02x\n", lapic_tpr()); #define dump_field(prefix, index) \ - dump_mask(__XSTRING(prefix ## index), lapic->prefix ## index, \ + dump_mask(__XSTRING(prefix ## index), lapic_ ## prefix(index), \ index * 32) db_printf("In-service Interrupts:\n"); @@ -1404,7 +1841,7 @@ lapic_ipi_wait(int delay) } else incr = 1; for (x = 0; x < delay; x += incr) { - if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE) + if ((lapic_icr_lo() & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE) return (1); ia32_pause(); } @@ -1414,10 +1851,11 @@ lapic_ipi_wait(int delay) void lapic_ipi_raw(register_t icrlo, u_int dest) { - register_t value, saveintr; + register_t saveintr; + uint32_t hi, lo; /* XXX: Need more sanity checking of icrlo? */ - KASSERT(lapic != NULL, ("%s called too early", __func__)); + KASSERT(!lapic_missing(), ("%s called too early", __func__)); KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, ("%s: invalid dest field", __func__)); KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, @@ -1426,17 +1864,21 @@ lapic_ipi_raw(register_t icrlo, u_int de /* Set destination in ICR HI register if it is being used. */ saveintr = intr_disable(); if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { - value = lapic->icr_hi; - value &= ~APIC_ID_MASK; - value |= dest << APIC_ID_SHIFT; - lapic->icr_hi = value; - } + if (x2apic) { + hi = dest; + } else { + hi = lapic_icr_hi(); + hi &= ~APIC_ID_MASK; + hi |= dest << APIC_ID_SHIFT; + } + } else + hi = 0; /* Program the contents of the IPI and dispatch it. */ - value = lapic->icr_lo; - value &= APIC_ICRLO_RESV_MASK; - value |= icrlo; - lapic->icr_lo = value; + lo = lapic_icr_lo(); + lo &= APIC_ICRLO_RESV_MASK; + lo |= icrlo; + lapic_set_icr((uint64_t)hi << 32 | lo); intr_restore(saveintr); } @@ -1513,7 +1955,7 @@ lapic_ipi_vectored(u_int vector, int des printf("APIC: IPI might be stuck\n"); #else /* !needsattention */ /* Wait until mesage is sent without a timeout. */ - while (lapic->icr_lo & APIC_DELSTAT_PEND) + while (lapic_icr_lo() & APIC_DELSTAT_PEND) ia32_pause(); #endif /* needsattention */ } *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201301060458.r064wmKb004372>