Date: Wed, 8 Aug 2007 05:11:14 GMT From: Kip Macy <kmacy@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 124874 for review Message-ID: <200708080511.l785BEAc094980@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=124874 Change 124874 by kmacy@kmacy_home:ethng on 2007/08/08 05:10:52 add intr_bind to i386 and amd64 to bind an interrupt to caller-specified cpu rather than the default round robin Submitted by: jhb Affected files ... .. //depot/projects/ethng/src/sys/amd64/amd64/intr_machdep.c#2 edit .. //depot/projects/ethng/src/sys/amd64/amd64/sys_machdep.c#2 edit .. //depot/projects/ethng/src/sys/amd64/include/intr_machdep.h#2 edit .. //depot/projects/ethng/src/sys/amd64/include/sysarch.h#2 edit .. //depot/projects/ethng/src/sys/conf/options#3 edit .. //depot/projects/ethng/src/sys/i386/i386/intr_machdep.c#2 edit .. //depot/projects/ethng/src/sys/i386/i386/sys_machdep.c#2 edit .. //depot/projects/ethng/src/sys/i386/include/intr_machdep.h#2 edit .. //depot/projects/ethng/src/sys/i386/include/sysarch.h#2 edit .. //depot/projects/ethng/src/sys/kern/kern_intr.c#2 edit .. //depot/projects/ethng/src/sys/sys/interrupt.h#2 edit Differences ... ==== //depot/projects/ethng/src/sys/amd64/amd64/intr_machdep.c#2 (text+ko) ==== @@ -51,6 +51,7 @@ #include <sys/syslog.h> #include <sys/systm.h> #include <sys/sx.h> +#include <sys/smp.h> #include <machine/clock.h> #include <machine/intr_machdep.h> #include <machine/smp.h> @@ -87,7 +88,7 @@ static void intr_assign_next_cpu(struct intsrc *isrc); #endif - +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); @@ -145,10 +146,10 @@ #ifdef INTR_FILTER error = intr_event_create(&isrc->is_event, isrc, 0, (mask_fn)isrc->is_pic->pic_enable_source, - intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector); + intr_eoi_src, intr_disab_eoi_src, intr_assign_cpu, "irq%d:", vector); #else error = intr_event_create(&isrc->is_event, isrc, 0, - (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", vector); #endif if (error) return (error); @@ -430,6 +431,33 @@ sx_xunlock(&intr_table_lock); } +static int +intr_assign_cpu(void *arg, u_char cpu) +{ +#ifdef SMP + struct intsrc *isrc; + + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu) { + isrc = arg; + + if (bootverbose) + printf("assigning vector=%d to cpu=%d\n", isrc->is_pic->pic_vector(isrc), cpu); + + sx_xlock(&intr_table_lock); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); + sx_xunlock(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif +} + + static void intrcnt_setname(const char *name, int index) { @@ -477,12 +505,16 @@ static void intr_init(void *dummy __unused) { - + int flags = 0; + intrcnt_setname("???", 0); intrcnt_index = 1; STAILQ_INIT(&pics); - sx_init(&intr_table_lock, "intr sources"); - mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); +#ifdef BIND_ALL + flags = SX_RECURSE; +#endif + sx_init_flags(&intr_table_lock, "intr sources", flags); + mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) @@ -541,15 +573,14 @@ static void intr_assign_next_cpu(struct intsrc *isrc) { - struct pic *pic; - u_int apic_id; - /* * Assign this source to a local APIC in a round-robin fashion. */ - pic = isrc->is_pic; - apic_id = cpu_apic_ids[current_cpu]; - pic->pic_assign_cpu(isrc, apic_id); +#ifdef BIND_ALL + intr_event_bind(isrc->is_event, current_cpu); +#else + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]); +#endif do { current_cpu++; if (current_cpu >= num_cpus) @@ -557,6 +588,20 @@ } while (!(intr_cpus & (1 << current_cpu))); } +/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(u_int vector, u_char cpu) +{ + struct intsrc *isrc; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + isrc->is_event->ie_source = isrc; + + return (intr_event_bind(isrc->is_event, cpu)); +} + /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. @@ -585,6 +630,10 @@ struct intsrc *isrc; int i; + /* + * XXX should be mp_ncpus but this causes a crash + */ + /* Don't bother on UP. */ if (num_cpus <= 1) return; @@ -594,8 +643,19 @@ assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; - if (isrc != NULL && isrc->is_handlers > 0) - intr_assign_next_cpu(isrc); + if (isrc != NULL && isrc->is_handlers > 0) { + /* + * If this event is already bound to a CPU, + * then assign the source to that CPU instead + * of picking one via round-robin. + */ + if (isrc->is_event->ie_cpu != NOCPU) + isrc->is_pic->pic_assign_cpu(isrc, + cpu_apic_ids[isrc->is_event->ie_cpu]); + else + intr_assign_next_cpu(isrc); + + } } sx_xunlock(&intr_table_lock); } ==== //depot/projects/ethng/src/sys/amd64/amd64/sys_machdep.c#2 (text+ko) ==== @@ -35,9 +35,11 @@ #include <sys/param.h> #include <sys/systm.h> -#include <sys/lock.h> +#include <sys/bus.h> +#include <sys/interrupt.h> #include <sys/proc.h> #include <sys/sysproto.h> +#include <machine/intr_machdep.h> #include <machine/specialreg.h> #include <machine/sysarch.h> #include <machine/pcb.h> @@ -62,7 +64,10 @@ struct pcb *pcb = curthread->td_pcb; uint32_t i386base; uint64_t a64base; - +#ifdef SMP + struct amd64_intr_bind_args bargs; +#endif + switch(uap->op) { case I386_GET_FSBASE: i386base = pcb->pcb_fsbase; @@ -126,6 +131,15 @@ } break; +#ifdef SMP +/* ABI and API compatible with I386_INTR_BIND. */ + case AMD64_INTR_BIND: + error = copyin(uap->parms, &bargs, + sizeof(struct amd64_intr_bind_args)); + if (error == 0) + error = intr_bind(bargs.vector, bargs.cpu); + break; +#endif default: error = EINVAL; break; ==== //depot/projects/ethng/src/sys/amd64/include/intr_machdep.h#2 (text+ko) ==== @@ -133,6 +133,7 @@ void elcr_write_trigger(u_int irq, enum intr_trigger trigger); #ifdef SMP void intr_add_cpu(u_int cpu); +int intr_bind(u_int vector, u_char cpu); #endif int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, ==== //depot/projects/ethng/src/sys/amd64/include/sysarch.h#2 (text+ko) ==== @@ -39,6 +39,7 @@ #define I386_SET_FSBASE 8 #define I386_GET_GSBASE 9 #define I386_SET_GSBASE 10 +#define AMD64_INTR_BIND 11 /* Leave space for 0-127 for to avoid translating syscalls */ #define AMD64_GET_FSBASE 128 @@ -46,6 +47,11 @@ #define AMD64_GET_GSBASE 130 #define AMD64_SET_GSBASE 131 +struct amd64_intr_bind_args { + unsigned int vector; + unsigned int cpu; +}; + #ifndef _KERNEL #include <sys/cdefs.h> @@ -54,6 +60,7 @@ int amd64_get_gsbase(void **); int amd64_set_fsbase(void *); int amd64_set_gsbase(void *); +int amd64_intr_bind(unsigned int, unsigned int); int sysarch(int, void *); __END_DECLS #endif ==== //depot/projects/ethng/src/sys/conf/options#3 (text+ko) ==== @@ -547,6 +547,7 @@ SX_NOINLINE opt_global.h VFS_BIO_DEBUG opt_global.h IFNET_MULTIQUEUE opt_global.h +BIND_ALL opt_global.h # These are VM related options VM_KMEM_SIZE opt_vm.h ==== //depot/projects/ethng/src/sys/i386/i386/intr_machdep.c#2 (text+ko) ==== @@ -79,6 +79,7 @@ static void intr_assign_next_cpu(struct intsrc *isrc); #endif +static int intr_assign_cpu(void *arg, u_char cpu); static void intr_init(void *__dummy); static int intr_pic_registered(struct pic *pic); static void intrcnt_setname(const char *name, int index); @@ -136,10 +137,10 @@ #ifdef INTR_FILTER error = intr_event_create(&isrc->is_event, isrc, 0, (mask_fn)isrc->is_pic->pic_enable_source, - intr_eoi_src, intr_disab_eoi_src, "irq%d:", vector); + intr_eoi_src, intr_disab_eoi_src, intr_assign_cpu,, "irq%d:", vector); #else error = intr_event_create(&isrc->is_event, isrc, 0, - (mask_fn)isrc->is_pic->pic_enable_source, "irq%d:", vector); + (mask_fn)isrc->is_pic->pic_enable_source, intr_assign_cpu, "irq%d:", vector); #endif if (error) return (error); @@ -428,6 +429,28 @@ sx_xunlock(&intr_table_lock); } +static int +intr_assign_cpu(void *arg, u_char cpu) +{ +#ifdef SMP + struct intsrc *isrc; + + /* + * Don't do anything during early boot. We will pick up the + * assignment once the APs are started. + */ + if (assign_cpu) { + isrc = arg; + sx_xlock(&intr_table_lock); + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]); + sx_xunlock(&intr_table_lock); + } + return (0); +#else + return (EOPNOTSUPP); +#endif +} + static void intrcnt_setname(const char *name, int index) { @@ -475,11 +498,15 @@ static void intr_init(void *dummy __unused) { - + int flags = 0; + intrcnt_setname("???", 0); intrcnt_index = 1; STAILQ_INIT(&pics); - sx_init(&intr_table_lock, "intr sources"); +#ifdef BIND_ALL + flags = SX_RECURSE; +#endif + sx_init_flags(&intr_table_lock, "intr sources", flags); mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); } SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL) @@ -517,15 +544,15 @@ static void intr_assign_next_cpu(struct intsrc *isrc) { - struct pic *pic; - u_int apic_id; /* * Assign this source to a local APIC in a round-robin fashion. */ - pic = isrc->is_pic; - apic_id = cpu_apic_ids[current_cpu]; - pic->pic_assign_cpu(isrc, apic_id); +#ifdef BIND_ALL + intr_event_bind(isrc->is_event, current_cpu); +#else + isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]); +#endif do { current_cpu++; if (current_cpu >= num_cpus) @@ -533,6 +560,20 @@ } while (!(intr_cpus & (1 << current_cpu))); } ++/* Attempt to bind the specified IRQ to the specified CPU. */ +int +intr_bind(u_int vector, u_char cpu) +{ + struct intsrc *isrc; + + isrc = intr_lookup_source(vector); + if (isrc == NULL) + return (EINVAL); + isrc->is_event->ie_source = isrc; + + return (intr_event_bind(isrc->is_event, cpu)); +} + /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. @@ -562,7 +603,7 @@ int i; /* Don't bother on UP. */ - if (num_cpus <= 1) + if (mp_ncpus <= 1) return; /* Round-robin assign a CPU to each enabled source. */ @@ -570,8 +611,14 @@ assign_cpu = 1; for (i = 0; i < NUM_IO_INTS; i++) { isrc = interrupt_sources[i]; - if (isrc != NULL && isrc->is_handlers > 0) - intr_assign_next_cpu(isrc); + if (isrc != NULL && isrc->is_handlers > 0) { + if (isrc->is_event->ie_cpu != NOCPU) + isrc->is_pic->pic_assign_cpu(isrc, + cpu_apic_ids[isrc->is_event->ie_cpu]); + else + intr_assign_next_cpu(isrc); + + } } sx_xunlock(&intr_table_lock); } ==== //depot/projects/ethng/src/sys/i386/i386/sys_machdep.c#2 (text+ko) ==== @@ -37,6 +37,8 @@ #include <sys/param.h> #include <sys/systm.h> +#include <sys/bus.h> +#include <sys/interrupt.h> #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mutex.h> @@ -44,6 +46,7 @@ #include <sys/proc.h> #include <sys/smp.h> #include <sys/sysproto.h> +#include <machine/intr_machdep.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -90,6 +93,9 @@ union { struct i386_ldt_args largs; struct i386_ioperm_args iargs; +#ifdef SMP + struct i386_intr_bind_args bargs; +#endif } kargs; uint32_t base; struct segment_descriptor sd, *sdp; @@ -209,6 +215,14 @@ load_gs(GSEL(GUGS_SEL, SEL_UPL)); } break; +#ifdef SMP + case I386_INTR_BIND: + error = copyin(uap->parms, &kargs.bargs, + sizeof(struct i386_intr_bind_args)); + if (error == 0) + error = intr_bind(kargs.bargs.vector, kargs.bargs.cpu); + break; +#endif default: error = EINVAL; break; ==== //depot/projects/ethng/src/sys/i386/include/intr_machdep.h#2 (text+ko) ==== @@ -130,6 +130,7 @@ void elcr_write_trigger(u_int irq, enum intr_trigger trigger); #ifdef SMP void intr_add_cpu(u_int cpu); +int intr_bind(int vector, u_int cpu); #endif int intr_add_handler(const char *name, int vector, driver_filter_t filter, driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep); ==== //depot/projects/ethng/src/sys/i386/include/sysarch.h#2 (text+ko) ==== @@ -47,6 +47,7 @@ #define I386_SET_FSBASE 8 #define I386_GET_GSBASE 9 #define I386_SET_GSBASE 10 +#define I386_INTR_BIND 11 /* These four only exist when running an i386 binary on amd64 */ #define _AMD64_GET_FSBASE 128 @@ -71,6 +72,11 @@ char *sub_args; /* args */ }; +struct i386_intr_bind_args { + unsigned int vector; + unsigned int cpu; +}; + #ifndef _KERNEL #include <sys/cdefs.h> ==== //depot/projects/ethng/src/sys/kern/kern_intr.c#2 (text+ko) ==== @@ -46,6 +46,7 @@ #include <sys/random.h> #include <sys/resourcevar.h> #include <sys/sched.h> +#include <sys/smp.h> #include <sys/sysctl.h> #include <sys/unistd.h> #include <sys/vmmeter.h> @@ -240,7 +241,7 @@ #ifndef INTR_FILTER int intr_event_create(struct intr_event **event, void *source, int flags, - void (*enable)(void *), const char *fmt, ...) + void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...) { struct intr_event *ie; va_list ap; @@ -251,6 +252,8 @@ ie = malloc(sizeof(struct intr_event), M_ITHREAD, M_WAITOK | M_ZERO); ie->ie_source = source; ie->ie_enable = enable; + ie->ie_assign_cpu = assign_cpu; + ie->ie_cpu = NOCPU; ie->ie_flags = flags; TAILQ_INIT(&ie->ie_handlers); mtx_init(&ie->ie_lock, "intr event", NULL, MTX_DEF); @@ -271,7 +274,7 @@ int intr_event_create(struct intr_event **event, void *source, int flags, void (*enable)(void *), void (*eoi)(void *), void (*disab)(void *), - const char *fmt, ...) + int (*assign_cpu)(void *, u_char), const char *fmt, ...) { struct intr_event *ie; va_list ap; @@ -302,6 +305,32 @@ } #endif +/* + * Bind an interrupt event to the specified CPU. + */ +int +intr_event_bind(struct intr_event *ie, u_char cpu) +{ + int error; + struct thread *td = curthread; + + /* Need a CPU to bind to. */ + if (cpu != NOCPU && CPU_ABSENT(cpu)) + return (EINVAL); + + if (ie->ie_assign_cpu == NULL) + return (EOPNOTSUPP); + + error = ie->ie_assign_cpu(ie->ie_source, cpu); + + thread_lock(td); + ie->ie_cpu = cpu; + thread_unlock(td); + + + return (0); +} + int intr_event_destroy(struct intr_event *ie) { @@ -895,10 +924,10 @@ } else { #ifdef INTR_FILTER error = intr_event_create(&ie, NULL, IE_SOFT, - NULL, NULL, NULL, "swi%d:", pri); + NULL, NULL, NULL, NULL, "swi%d:", pri); #else error = intr_event_create(&ie, NULL, IE_SOFT, - NULL, "swi%d:", pri); + NULL, NULL, "swi%d:", pri); #endif if (error) return (error); @@ -1081,6 +1110,7 @@ struct intr_event *ie; struct thread *td; struct proc *p; + u_char cpu; td = curthread; p = td->td_proc; @@ -1089,7 +1119,8 @@ ("%s: ithread and proc linkage out of sync", __func__)); ie = ithd->it_event; ie->ie_count = 0; - + cpu = NOCPU; + /* * As long as we have interrupts outstanding, go through the * list of handlers, giving each one a go at it. @@ -1134,6 +1165,21 @@ ie->ie_count = 0; mi_switch(SW_VOL, NULL); } + +#ifdef SMP + /* + * Ensure we are bound to the correct CPU. We can't + * move ithreads until SMP is running however, so just + * leave interrupts on the boor CPU during boot. + */ + if (ie->ie_cpu != cpu && smp_started) { + cpu = ie->ie_cpu; + if (cpu == NOCPU) + sched_unbind(td); + else + sched_bind(td, cpu); + } +#endif thread_unlock(td); } } @@ -1463,6 +1509,8 @@ db_printf("ADDING_THREAD"); comma = 1; } + if (ie->ie_cpu != NOCPU) + db_printf(" (CPU %d)", ie->ie_cpu); if (it != NULL && it->it_need) { if (comma) db_printf(", "); ==== //depot/projects/ethng/src/sys/sys/interrupt.h#2 (text+ko) ==== @@ -73,6 +73,7 @@ void *ie_source; /* Cookie used by MD code. */ struct intr_thread *ie_thread; /* Thread we are connected to. */ void (*ie_enable)(void *); + int (*ie_assign_cpu)(void *, u_char); #ifdef INTR_FILTER void (*ie_eoi)(void *); void (*ie_disab)(void *); @@ -81,6 +82,7 @@ int ie_count; /* Loop counter. */ int ie_warncnt; /* Rate-check interrupt storm warns. */ struct timeval ie_warntm; + u_char ie_cpu; }; /* Interrupt event flags kept in ie_flags. */ @@ -127,15 +129,16 @@ int intr_event_add_handler(struct intr_event *ie, const char *name, driver_filter_t filter, driver_intr_t handler, void *arg, u_char pri, enum intr_type flags, void **cookiep); +int intr_event_bind(struct intr_event *ie, u_char cpu); #ifndef INTR_FILTER int intr_event_create(struct intr_event **event, void *source, - int flags, void (*enable)(void *), const char *fmt, ...) - __printflike(5, 6); + int flags, void (*enable)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...) + __printflike(6, 7); #else int intr_event_create(struct intr_event **event, void *source, int flags, void (*enable)(void *), void (*eoi)(void *), - void (*disab)(void *), const char *fmt, ...) - __printflike(7, 8); + void (*disab)(void *), int (*assign_cpu)(void *, u_char), const char *fmt, ...) + __printflike(8, 9); #endif int intr_event_destroy(struct intr_event *ie); int intr_event_remove_handler(void *cookie);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200708080511.l785BEAc094980>