Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 25 Jun 2009 18:13:46 +0000 (UTC)
From:      John Baldwin <jhb@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r194985 - in head/sys: amd64/amd64 amd64/include i386/i386 i386/include
Message-ID:  <200906251813.n5PIDkfl060004@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: jhb
Date: Thu Jun 25 18:13:46 2009
New Revision: 194985
URL: http://svn.freebsd.org/changeset/base/194985

Log:
  - Restore the behavior of pre-allocating IDT vectors for MSI interrupts.
    This is mostly important for the multiple MSI message case where the
    IDT vectors for the entire group need to be allocated together.  This
    also restores the assumptions made by the PCI bus code that it could
    invoke PCIB_MAP_MSI() once MSI vectors were allocated.
  - To avoid whiplash with CPU assignments, change the way that CPUs are
    assigned to interrupt sources on activation.  Instead of assigning the
    CPU via pic_assign_cpu() before calling enable_intr(), allow the
    different interrupt source drivers to ask the MD interrupt code which
    CPU to use when they allocate an IDT vector.  I/O APIC interrupt pins
    do this in their pic_enable_intr() routines giving the same behavior as
    before.  MSI sources do it when the IDT vectors are allocated during
    msi_alloc() and msix_alloc().
  - Change the intr_table_lock from an sx lock to a mutex.
  
  Tested by:	rnoland

Modified:
  head/sys/amd64/amd64/intr_machdep.c
  head/sys/amd64/amd64/io_apic.c
  head/sys/amd64/amd64/msi.c
  head/sys/amd64/include/intr_machdep.h
  head/sys/i386/i386/intr_machdep.c
  head/sys/i386/i386/io_apic.c
  head/sys/i386/i386/msi.c
  head/sys/i386/include/intr_machdep.h

Modified: head/sys/amd64/amd64/intr_machdep.c
==============================================================================
--- head/sys/amd64/amd64/intr_machdep.c	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/amd64/amd64/intr_machdep.c	Thu Jun 25 18:13:46 2009	(r194985)
@@ -51,7 +51,6 @@
 #include <sys/smp.h>
 #include <sys/syslog.h>
 #include <sys/systm.h>
-#include <sys/sx.h>
 #include <machine/clock.h>
 #include <machine/intr_machdep.h>
 #include <machine/smp.h>
@@ -73,14 +72,12 @@ typedef void (*mask_fn)(void *);
 
 static int intrcnt_index;
 static struct intsrc *interrupt_sources[NUM_IO_INTS];
-static struct sx intr_table_lock;
+static struct mtx intr_table_lock;
 static struct mtx intrcnt_lock;
 static STAILQ_HEAD(, pic) pics;
 
 #ifdef SMP
 static int assign_cpu;
-
-static void	intr_assign_next_cpu(struct intsrc *isrc);
 #endif
 
 static int	intr_assign_cpu(void *arg, u_char cpu);
@@ -114,14 +111,14 @@ intr_register_pic(struct pic *pic)
 {
 	int error;
 
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	if (intr_pic_registered(pic))
 		error = EBUSY;
 	else {
 		STAILQ_INSERT_TAIL(&pics, pic, pics);
 		error = 0;
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 	return (error);
 }
 
@@ -145,16 +142,16 @@ intr_register_source(struct intsrc *isrc
 	    vector);
 	if (error)
 		return (error);
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	if (interrupt_sources[vector] != NULL) {
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 		intr_event_destroy(isrc->is_event);
 		return (EEXIST);
 	}
 	intrcnt_register(isrc);
 	interrupt_sources[vector] = isrc;
 	isrc->is_handlers = 0;
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 	return (0);
 }
 
@@ -178,18 +175,14 @@ intr_add_handler(const char *name, int v
 	error = intr_event_add_handler(isrc->is_event, name, filter, handler,
 	    arg, intr_priority(flags), flags, cookiep);
 	if (error == 0) {
-		sx_xlock(&intr_table_lock);
+		mtx_lock(&intr_table_lock);
 		intrcnt_updatename(isrc);
 		isrc->is_handlers++;
 		if (isrc->is_handlers == 1) {
-#ifdef SMP
-			if (assign_cpu)
-				intr_assign_next_cpu(isrc);
-#endif
 			isrc->is_pic->pic_enable_intr(isrc);
 			isrc->is_pic->pic_enable_source(isrc);
 		}
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 	}
 	return (error);
 }
@@ -203,14 +196,14 @@ intr_remove_handler(void *cookie)
 	isrc = intr_handler_source(cookie);
 	error = intr_event_remove_handler(cookie);
 	if (error == 0) {
-		sx_xlock(&intr_table_lock);
+		mtx_lock(&intr_table_lock);
 		isrc->is_handlers--;
 		if (isrc->is_handlers == 0) {
 			isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI);
 			isrc->is_pic->pic_disable_intr(isrc);
 		}
 		intrcnt_updatename(isrc);
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 	}
 	return (error);
 }
@@ -284,12 +277,12 @@ intr_resume(void)
 #ifndef DEV_ATPIC
 	atpic_reset();
 #endif
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	STAILQ_FOREACH(pic, &pics, pics) {
 		if (pic->pic_resume != NULL)
 			pic->pic_resume(pic);
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 }
 
 void
@@ -297,12 +290,12 @@ intr_suspend(void)
 {
 	struct pic *pic;
 
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	STAILQ_FOREACH(pic, &pics, pics) {
 		if (pic->pic_suspend != NULL)
 			pic->pic_suspend(pic);
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 }
 
 static int
@@ -317,9 +310,9 @@ intr_assign_cpu(void *arg, u_char cpu)
 	 */
 	if (assign_cpu && cpu != NOCPU) {
 		isrc = arg;
-		sx_xlock(&intr_table_lock);
+		mtx_lock(&intr_table_lock);
 		isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]);
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 	}
 	return (0);
 #else
@@ -378,7 +371,7 @@ intr_init(void *dummy __unused)
 	intrcnt_setname("???", 0);
 	intrcnt_index = 1;
 	STAILQ_INIT(&pics);
-	sx_init(&intr_table_lock, "intr sources");
+	mtx_init(&intr_table_lock, "intr sources", NULL, MTX_DEF | MTX_RECURSE);
 	mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
 }
 SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL);
@@ -435,19 +428,28 @@ DB_SHOW_COMMAND(irqs, db_show_irqs)
 static cpumask_t intr_cpus = (1 << 0);
 static int current_cpu;
 
-static void
-intr_assign_next_cpu(struct intsrc *isrc)
+/*
+ * Return the CPU that the next interrupt source should use.  For now
+ * this just returns the next local APIC according to round-robin.
+ */
+u_int
+intr_next_cpu(void)
 {
+	u_int apic_id;
 
-	/*
-	 * Assign this source to a local APIC in a round-robin fashion.
-	 */
-	isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]);
+	/* Leave all interrupts on the BSP during boot. */
+	if (!assign_cpu)
+		return (cpu_apic_ids[0]);
+
+	mtx_lock(&intr_table_lock);
+	apic_id = cpu_apic_ids[current_cpu];
 	do {
 		current_cpu++;
 		if (current_cpu > mp_maxid)
 			current_cpu = 0;
 	} while (!(intr_cpus & (1 << current_cpu)));
+	mtx_unlock(&intr_table_lock);
+	return (apic_id);
 }
 
 /* Attempt to bind the specified IRQ to the specified CPU. */
@@ -487,6 +489,7 @@ static void
 intr_shuffle_irqs(void *arg __unused)
 {
 	struct intsrc *isrc;
+	u_int apic_id;
 	int i;
 
 	/* Don't bother on UP. */
@@ -494,7 +497,7 @@ intr_shuffle_irqs(void *arg __unused)
 		return;
 
 	/* Round-robin assign a CPU to each enabled source. */
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	assign_cpu = 1;
 	for (i = 0; i < NUM_IO_INTS; i++) {
 		isrc = interrupt_sources[i];
@@ -505,13 +508,13 @@ intr_shuffle_irqs(void *arg __unused)
 			 * of picking one via round-robin.
 			 */
 			if (isrc->is_event->ie_cpu != NOCPU)
-				isrc->is_pic->pic_assign_cpu(isrc,
-				    cpu_apic_ids[isrc->is_event->ie_cpu]);
+				apic_id = isrc->is_event->ie_cpu;
 			else
-				intr_assign_next_cpu(isrc);
+				apic_id = intr_next_cpu();
+			isrc->is_pic->pic_assign_cpu(isrc, apic_id);
 		}
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 }
 SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs,
     NULL);

Modified: head/sys/amd64/amd64/io_apic.c
==============================================================================
--- head/sys/amd64/amd64/io_apic.c	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/amd64/amd64/io_apic.c	Thu Jun 25 18:13:46 2009	(r194985)
@@ -372,7 +372,7 @@ ioapic_enable_intr(struct intsrc *isrc)
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
 
 	if (intpin->io_vector == 0)
-		ioapic_assign_cpu(isrc, pcpu_find(0)->pc_apic_id);
+		ioapic_assign_cpu(isrc, intr_next_cpu());
 	apic_enable_vector(intpin->io_cpu, intpin->io_vector);
 }
 

Modified: head/sys/amd64/amd64/msi.c
==============================================================================
--- head/sys/amd64/amd64/msi.c	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/amd64/amd64/msi.c	Thu Jun 25 18:13:46 2009	(r194985)
@@ -161,8 +161,6 @@ msi_enable_intr(struct intsrc *isrc)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
 
-	if (msi->msi_vector == 0)
-		msi_assign_cpu(isrc, 0);
 	apic_enable_vector(msi->msi_cpu, msi->msi_vector);
 }
 
@@ -208,10 +206,11 @@ msi_assign_cpu(struct intsrc *isrc, u_in
 	/* Store information to free existing irq. */
 	old_vector = msi->msi_vector;
 	old_id = msi->msi_cpu;
-	if (old_vector && old_id == apic_id)
+	if (old_id == apic_id)
 		return;
-	if (old_vector && !msi->msi_msix && msi->msi_first->msi_count > 1)
+	if (!msi->msi_msix && msi->msi_first->msi_count > 1)
 		return;
+
 	/* Allocate IDT vector on this cpu. */
 	vector = apic_alloc_vector(apic_id, msi->msi_irq);
 	if (vector == 0)
@@ -223,15 +222,14 @@ msi_assign_cpu(struct intsrc *isrc, u_in
 		    msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
 		    msi->msi_cpu, msi->msi_vector);
 	pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
+
 	/*
 	 * Free the old vector after the new one is established.  This is done
 	 * to prevent races where we could miss an interrupt.
 	 */
-	if (old_vector)
-		apic_free_vector(old_id, old_vector, msi->msi_irq);
+	apic_free_vector(old_id, old_vector, msi->msi_irq);
 }
 
-
 void
 msi_init(void)
 {
@@ -287,7 +285,8 @@ int
 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
 {
 	struct msi_intsrc *msi, *fsrc;
-	int cnt, i;
+	u_int cpu;
+	int cnt, i, vector;
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -333,12 +332,25 @@ again:
 	/* Ok, we now have the IRQs allocated. */
 	KASSERT(cnt == count, ("count mismatch"));
 
+	/* Allocate 'count' IDT vectors. */
+	cpu = intr_next_cpu();
+	vector = apic_alloc_vectors(cpu, irqs, count, maxcount);
+	if (vector == 0) {
+		mtx_unlock(&msi_lock);
+		return (ENOSPC);
+	}
+
 	/* Assign IDT vectors and make these messages owned by 'dev'. */
 	fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
 	for (i = 0; i < count; i++) {
 		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
+		msi->msi_cpu = cpu;
 		msi->msi_dev = dev;
-		msi->msi_vector = 0;
+		msi->msi_vector = vector + i;
+		if (bootverbose)
+			printf(
+		    "msi: routing MSI IRQ %d to local APIC %u vector %u\n",
+			    msi->msi_irq, msi->msi_cpu, msi->msi_vector);
 		msi->msi_first = fsrc;
 		KASSERT(msi->msi_intsrc.is_handlers == 0,
 		    ("dead MSI has handlers"));
@@ -391,18 +403,14 @@ msi_release(int *irqs, int count)
 		KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
 		msi->msi_first = NULL;
 		msi->msi_dev = NULL;
-		if (msi->msi_vector)
-			apic_free_vector(msi->msi_cpu, msi->msi_vector,
-			    msi->msi_irq);
+		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
 		msi->msi_vector = 0;
 	}
 
 	/* Clear out the first message. */
 	first->msi_first = NULL;
 	first->msi_dev = NULL;
-	if (first->msi_vector)
-		apic_free_vector(first->msi_cpu, first->msi_vector,
-		    first->msi_irq);
+	apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
 	first->msi_vector = 0;
 	first->msi_count = 0;
 
@@ -451,7 +459,8 @@ int
 msix_alloc(device_t dev, int *irq)
 {
 	struct msi_intsrc *msi;
-	int i;
+	u_int cpu;
+	int i, vector;
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -486,9 +495,17 @@ again:
 		goto again;
 	}
 
+	/* Allocate an IDT vector. */
+	cpu = intr_next_cpu();
+	vector = apic_alloc_vector(cpu, i);
+	if (bootverbose)
+		printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
+		    msi->msi_irq, cpu, vector);
+	
 	/* Setup source. */
+	msi->msi_cpu = cpu;
 	msi->msi_dev = dev;
-	msi->msi_vector = 0;
+	msi->msi_vector = vector;
 	msi->msi_msix = 1;
 
 	KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
@@ -520,8 +537,7 @@ msix_release(int irq)
 
 	/* Clear out the message. */
 	msi->msi_dev = NULL;
-	if (msi->msi_vector)
-		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
+	apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
 	msi->msi_vector = 0;
 	msi->msi_msix = 0;
 

Modified: head/sys/amd64/include/intr_machdep.h
==============================================================================
--- head/sys/amd64/include/intr_machdep.h	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/amd64/include/intr_machdep.h	Thu Jun 25 18:13:46 2009	(r194985)
@@ -152,6 +152,9 @@ int	intr_bind(u_int vector, u_char cpu);
 int	intr_config_intr(int vector, enum intr_trigger trig,
     enum intr_polarity pol);
 void	intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame);
+#ifdef SMP
+u_int	intr_next_cpu(void);
+#endif
 struct intsrc *intr_lookup_source(int vector);
 int	intr_register_pic(struct pic *pic);
 int	intr_register_source(struct intsrc *isrc);

Modified: head/sys/i386/i386/intr_machdep.c
==============================================================================
--- head/sys/i386/i386/intr_machdep.c	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/i386/i386/intr_machdep.c	Thu Jun 25 18:13:46 2009	(r194985)
@@ -50,7 +50,6 @@
 #include <sys/smp.h>
 #include <sys/syslog.h>
 #include <sys/systm.h>
-#include <sys/sx.h>
 #include <machine/clock.h>
 #include <machine/intr_machdep.h>
 #include <machine/smp.h>
@@ -64,14 +63,12 @@ typedef void (*mask_fn)(void *);
 
 static int intrcnt_index;
 static struct intsrc *interrupt_sources[NUM_IO_INTS];
-static struct sx intr_table_lock;
+static struct mtx intr_table_lock;
 static struct mtx intrcnt_lock;
 static STAILQ_HEAD(, pic) pics;
 
 #ifdef SMP
 static int assign_cpu;
-
-static void	intr_assign_next_cpu(struct intsrc *isrc);
 #endif
 
 static int	intr_assign_cpu(void *arg, u_char cpu);
@@ -105,14 +102,14 @@ intr_register_pic(struct pic *pic)
 {
 	int error;
 
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	if (intr_pic_registered(pic))
 		error = EBUSY;
 	else {
 		STAILQ_INSERT_TAIL(&pics, pic, pics);
 		error = 0;
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 	return (error);
 }
 
@@ -136,16 +133,16 @@ intr_register_source(struct intsrc *isrc
 	    vector);
 	if (error)
 		return (error);
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	if (interrupt_sources[vector] != NULL) {
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 		intr_event_destroy(isrc->is_event);
 		return (EEXIST);
 	}
 	intrcnt_register(isrc);
 	interrupt_sources[vector] = isrc;
 	isrc->is_handlers = 0;
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 	return (0);
 }
 
@@ -169,18 +166,14 @@ intr_add_handler(const char *name, int v
 	error = intr_event_add_handler(isrc->is_event, name, filter, handler,
 	    arg, intr_priority(flags), flags, cookiep);
 	if (error == 0) {
-		sx_xlock(&intr_table_lock);
+		mtx_lock(&intr_table_lock);
 		intrcnt_updatename(isrc);
 		isrc->is_handlers++;
 		if (isrc->is_handlers == 1) {
-#ifdef SMP
-			if (assign_cpu)
-				intr_assign_next_cpu(isrc);
-#endif
 			isrc->is_pic->pic_enable_intr(isrc);
 			isrc->is_pic->pic_enable_source(isrc);
 		}
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 	}
 	return (error);
 }
@@ -194,14 +187,14 @@ intr_remove_handler(void *cookie)
 	isrc = intr_handler_source(cookie);
 	error = intr_event_remove_handler(cookie);
 	if (error == 0) {
-		sx_xlock(&intr_table_lock);
+		mtx_lock(&intr_table_lock);
 		isrc->is_handlers--;
 		if (isrc->is_handlers == 0) {
 			isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI);
 			isrc->is_pic->pic_disable_intr(isrc);
 		}
 		intrcnt_updatename(isrc);
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 	}
 	return (error);
 }
@@ -272,12 +265,12 @@ intr_resume(void)
 {
 	struct pic *pic;
 
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	STAILQ_FOREACH(pic, &pics, pics) {
 		if (pic->pic_resume != NULL)
 			pic->pic_resume(pic);
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 }
 
 void
@@ -285,12 +278,12 @@ intr_suspend(void)
 {
 	struct pic *pic;
 
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	STAILQ_FOREACH(pic, &pics, pics) {
 		if (pic->pic_suspend != NULL)
 			pic->pic_suspend(pic);
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 }
 
 static int
@@ -305,9 +298,9 @@ intr_assign_cpu(void *arg, u_char cpu)
 	 */
 	if (assign_cpu && cpu != NOCPU) {
 		isrc = arg;
-		sx_xlock(&intr_table_lock);
+		mtx_lock(&intr_table_lock);
 		isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[cpu]);
-		sx_xunlock(&intr_table_lock);
+		mtx_unlock(&intr_table_lock);
 	}
 	return (0);
 #else
@@ -366,7 +359,7 @@ intr_init(void *dummy __unused)
 	intrcnt_setname("???", 0);
 	intrcnt_index = 1;
 	STAILQ_INIT(&pics);
-	sx_init(&intr_table_lock, "intr sources");
+	mtx_init(&intr_table_lock, "intr sources", NULL, MTX_DEF | MTX_RECURSE);
 	mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
 }
 SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL);
@@ -401,19 +394,28 @@ DB_SHOW_COMMAND(irqs, db_show_irqs)
 static cpumask_t intr_cpus = (1 << 0);
 static int current_cpu;
 
-static void
-intr_assign_next_cpu(struct intsrc *isrc)
+/*
+ * Return the CPU that the next interrupt source should use.  For now
+ * this just returns the next local APIC according to round-robin.
+ */
+u_int
+intr_next_cpu(void)
 {
+	u_int apic_id;
 
-	/*
-	 * Assign this source to a local APIC in a round-robin fashion.
-	 */
-	isrc->is_pic->pic_assign_cpu(isrc, cpu_apic_ids[current_cpu]);
+	/* Leave all interrupts on the BSP during boot. */
+	if (!assign_cpu)
+		return (cpu_apic_ids[0]);
+
+	mtx_lock(&intr_table_lock);
+	apic_id = cpu_apic_ids[current_cpu];
 	do {
 		current_cpu++;
 		if (current_cpu > mp_maxid)
 			current_cpu = 0;
 	} while (!(intr_cpus & (1 << current_cpu)));
+	mtx_unlock(&intr_table_lock);
+	return (apic_id);
 }
 
 /* Attempt to bind the specified IRQ to the specified CPU. */
@@ -453,6 +455,7 @@ static void
 intr_shuffle_irqs(void *arg __unused)
 {
 	struct intsrc *isrc;
+	u_int apic_id;
 	int i;
 
 #ifdef XEN
@@ -467,7 +470,7 @@ intr_shuffle_irqs(void *arg __unused)
 		return;
 
 	/* Round-robin assign a CPU to each enabled source. */
-	sx_xlock(&intr_table_lock);
+	mtx_lock(&intr_table_lock);
 	assign_cpu = 1;
 	for (i = 0; i < NUM_IO_INTS; i++) {
 		isrc = interrupt_sources[i];
@@ -478,13 +481,13 @@ intr_shuffle_irqs(void *arg __unused)
 			 * of picking one via round-robin.
 			 */
 			if (isrc->is_event->ie_cpu != NOCPU)
-				isrc->is_pic->pic_assign_cpu(isrc,
-				    cpu_apic_ids[isrc->is_event->ie_cpu]);
+				apic_id = isrc->is_event->ie_cpu;
 			else
-				intr_assign_next_cpu(isrc);
+				apic_id = intr_next_cpu();
+			isrc->is_pic->pic_assign_cpu(isrc, apic_id);
 		}
 	}
-	sx_xunlock(&intr_table_lock);
+	mtx_unlock(&intr_table_lock);
 }
 SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs,
     NULL);

Modified: head/sys/i386/i386/io_apic.c
==============================================================================
--- head/sys/i386/i386/io_apic.c	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/i386/i386/io_apic.c	Thu Jun 25 18:13:46 2009	(r194985)
@@ -372,7 +372,7 @@ ioapic_enable_intr(struct intsrc *isrc)
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
 
 	if (intpin->io_vector == 0)
-		ioapic_assign_cpu(isrc, pcpu_find(0)->pc_apic_id);
+		ioapic_assign_cpu(isrc, intr_next_cpu());
 	apic_enable_vector(intpin->io_cpu, intpin->io_vector);
 }
 

Modified: head/sys/i386/i386/msi.c
==============================================================================
--- head/sys/i386/i386/msi.c	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/i386/i386/msi.c	Thu Jun 25 18:13:46 2009	(r194985)
@@ -161,8 +161,6 @@ msi_enable_intr(struct intsrc *isrc)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
 
-	if (msi->msi_vector == 0)
-		msi_assign_cpu(isrc, 0);
 	apic_enable_vector(msi->msi_cpu, msi->msi_vector);
 }
 
@@ -208,10 +206,11 @@ msi_assign_cpu(struct intsrc *isrc, u_in
 	/* Store information to free existing irq. */
 	old_vector = msi->msi_vector;
 	old_id = msi->msi_cpu;
-	if (old_vector && old_id == apic_id)
+	if (old_id == apic_id)
 		return;
-	if (old_vector && !msi->msi_msix && msi->msi_first->msi_count > 1)
+	if (!msi->msi_msix && msi->msi_first->msi_count > 1)
 		return;
+
 	/* Allocate IDT vector on this cpu. */
 	vector = apic_alloc_vector(apic_id, msi->msi_irq);
 	if (vector == 0)
@@ -223,15 +222,14 @@ msi_assign_cpu(struct intsrc *isrc, u_in
 		    msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
 		    msi->msi_cpu, msi->msi_vector);
 	pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
+
 	/*
 	 * Free the old vector after the new one is established.  This is done
 	 * to prevent races where we could miss an interrupt.
 	 */
-	if (old_vector)
-		apic_free_vector(old_id, old_vector, msi->msi_irq);
+	apic_free_vector(old_id, old_vector, msi->msi_irq);
 }
 
-
 void
 msi_init(void)
 {
@@ -287,7 +285,8 @@ int
 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
 {
 	struct msi_intsrc *msi, *fsrc;
-	int cnt, i;
+	u_int cpu;
+	int cnt, i, vector;
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -333,12 +332,25 @@ again:
 	/* Ok, we now have the IRQs allocated. */
 	KASSERT(cnt == count, ("count mismatch"));
 
+	/* Allocate 'count' IDT vectors. */
+	cpu = intr_next_cpu();
+	vector = apic_alloc_vectors(cpu, irqs, count, maxcount);
+	if (vector == 0) {
+		mtx_unlock(&msi_lock);
+		return (ENOSPC);
+	}
+
 	/* Assign IDT vectors and make these messages owned by 'dev'. */
 	fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
 	for (i = 0; i < count; i++) {
 		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
+		msi->msi_cpu = cpu;
 		msi->msi_dev = dev;
-		msi->msi_vector = 0;
+		msi->msi_vector = vector + i;
+		if (bootverbose)
+			printf(
+		    "msi: routing MSI IRQ %d to local APIC %u vector %u\n",
+			    msi->msi_irq, msi->msi_cpu, msi->msi_vector);
 		msi->msi_first = fsrc;
 		KASSERT(msi->msi_intsrc.is_handlers == 0,
 		    ("dead MSI has handlers"));
@@ -391,18 +403,14 @@ msi_release(int *irqs, int count)
 		KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
 		msi->msi_first = NULL;
 		msi->msi_dev = NULL;
-		if (msi->msi_vector)
-			apic_free_vector(msi->msi_cpu, msi->msi_vector,
-			    msi->msi_irq);
+		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
 		msi->msi_vector = 0;
 	}
 
 	/* Clear out the first message. */
 	first->msi_first = NULL;
 	first->msi_dev = NULL;
-	if (first->msi_vector)
-		apic_free_vector(first->msi_cpu, first->msi_vector,
-		    first->msi_irq);
+	apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
 	first->msi_vector = 0;
 	first->msi_count = 0;
 
@@ -451,7 +459,8 @@ int
 msix_alloc(device_t dev, int *irq)
 {
 	struct msi_intsrc *msi;
-	int i;
+	u_int cpu;
+	int i, vector;
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -486,9 +495,17 @@ again:
 		goto again;
 	}
 
+	/* Allocate an IDT vector. */
+	cpu = intr_next_cpu();
+	vector = apic_alloc_vector(cpu, i);
+	if (bootverbose)
+		printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
+		    msi->msi_irq, cpu, vector);
+	
 	/* Setup source. */
+	msi->msi_cpu = cpu;
 	msi->msi_dev = dev;
-	msi->msi_vector = 0;
+	msi->msi_vector = vector;
 	msi->msi_msix = 1;
 
 	KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
@@ -520,8 +537,7 @@ msix_release(int irq)
 
 	/* Clear out the message. */
 	msi->msi_dev = NULL;
-	if (msi->msi_vector)
-		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
+	apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
 	msi->msi_vector = 0;
 	msi->msi_msix = 0;
 

Modified: head/sys/i386/include/intr_machdep.h
==============================================================================
--- head/sys/i386/include/intr_machdep.h	Thu Jun 25 18:09:23 2009	(r194984)
+++ head/sys/i386/include/intr_machdep.h	Thu Jun 25 18:13:46 2009	(r194985)
@@ -139,6 +139,9 @@ int	intr_bind(u_int vector, u_char cpu);
 int	intr_config_intr(int vector, enum intr_trigger trig,
     enum intr_polarity pol);
 void	intr_execute_handlers(struct intsrc *isrc, struct trapframe *frame);
+#ifdef SMP
+u_int	intr_next_cpu(void);
+#endif
 struct intsrc *intr_lookup_source(int vector);
 int	intr_register_pic(struct pic *pic);
 int	intr_register_source(struct intsrc *isrc);



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200906251813.n5PIDkfl060004>