Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 5 Aug 2016 17:13:26 +0000 (UTC)
From:      John Baldwin <jhb@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org
Subject:   svn commit: r303776 - in stable/10/sys: amd64/include i386/include x86/x86 x86/xen
Message-ID:  <201608051713.u75HDQe8060409@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: jhb
Date: Fri Aug  5 17:13:25 2016
New Revision: 303776
URL: https://svnweb.freebsd.org/changeset/base/303776

Log:
  MFC 302181,302635: Disable MSI-X migration on older Xen hypervisors.
  
  302181:
  Add a tunable to disable migration of MSI-X interrupts.
  
  The new 'machdep.disable_msix_migration' tunable can be set to 1 to
  disable migration of MSI-X interrupts.
  
  Xen versions prior to 4.6.0 do not properly handle updates to MSI-X
  table entries after the initial write.  In particular, the operation
  to unmask a table entry after updating it during migration is not
  propagated to the "real" table for passthrough devices causing the
  interrupt to remain masked.  At least some systems in EC2 are
  affected by this bug when using SRIOV.  The tunable can be set in
  loader.conf as a workaround.
  
  302635:
  xen: automatically disable MSI-X interrupt migration
  
  If the hypervisor version is smaller than 4.6.0. Xen commits 74fd00 and
  70a3cb are required on the hypervisor side for this to be fixed, and those
  are only included in 4.6.0, so stay on the safe side and disable MSI-X
  interrupt migration on anything older than 4.6.0.
  
  It should not cause major performance degradation unless a lot of MSI-X
  interrupts are allocated.

Modified:
  stable/10/sys/amd64/include/intr_machdep.h
  stable/10/sys/i386/include/intr_machdep.h
  stable/10/sys/x86/x86/msi.c
  stable/10/sys/x86/xen/hvm.c
Directory Properties:
  stable/10/   (props changed)

Modified: stable/10/sys/amd64/include/intr_machdep.h
==============================================================================
--- stable/10/sys/amd64/include/intr_machdep.h	Fri Aug  5 16:53:24 2016	(r303775)
+++ stable/10/sys/amd64/include/intr_machdep.h	Fri Aug  5 17:13:25 2016	(r303776)
@@ -144,6 +144,8 @@ struct nmi_pcpu {
 extern struct mtx icu_lock;
 extern int elcr_found;
 
+extern int msix_disable_migration;
+
 #ifndef DEV_ATPIC
 void	atpic_reset(void);
 #endif

Modified: stable/10/sys/i386/include/intr_machdep.h
==============================================================================
--- stable/10/sys/i386/include/intr_machdep.h	Fri Aug  5 16:53:24 2016	(r303775)
+++ stable/10/sys/i386/include/intr_machdep.h	Fri Aug  5 17:13:25 2016	(r303776)
@@ -141,6 +141,8 @@ struct trapframe;
 extern struct mtx icu_lock;
 extern int elcr_found;
 
+extern int msix_disable_migration;
+
 #ifndef DEV_ATPIC
 void	atpic_reset(void);
 #endif

Modified: stable/10/sys/x86/x86/msi.c
==============================================================================
--- stable/10/sys/x86/x86/msi.c	Fri Aug  5 16:53:24 2016	(r303775)
+++ stable/10/sys/x86/x86/msi.c	Fri Aug  5 17:13:25 2016	(r303776)
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/sx.h>
+#include <sys/sysctl.h>
 #include <sys/systm.h>
 #include <x86/apicreg.h>
 #include <machine/cputypes.h>
@@ -134,6 +135,20 @@ struct pic msi_pic = { msi_enable_source
 		       msi_source_pending, NULL, NULL, msi_config_intr,
 		       msi_assign_cpu };
 
+/**
+ * Xen hypervisors prior to 4.6.0 do not properly handle updates to
+ * enabled MSI-X table entries.  Allow migration of MSI-X interrupts
+ * to be disabled via a tunable. Values have the following meaning:
+ *
+ * -1: automatic detection by FreeBSD
+ *  0: enable migration
+ *  1: disable migration
+ */
+int msix_disable_migration = -1;
+SYSCTL_INT(_machdep, OID_AUTO, disable_msix_migration, CTLFLAG_RDTUN,
+    &msix_disable_migration, 0,
+    "Disable migration of MSI-X interrupts between CPUs");
+
 static int msi_enabled;
 static int msi_last_irq;
 static struct mtx msi_lock;
@@ -212,6 +227,9 @@ msi_assign_cpu(struct intsrc *isrc, u_in
 	if (msi->msi_first != msi)
 		return (EINVAL);
 
+	if (msix_disable_migration && msi->msi_msix)
+		return (EINVAL);
+
 	/* Store information to free existing irq. */
 	old_vector = msi->msi_vector;
 	old_id = msi->msi_cpu;
@@ -284,6 +302,11 @@ msi_init(void)
 		return;
 	}
 
+	if (msix_disable_migration == -1) {
+		/* The default is to allow migration of MSI-X interrupts. */
+		msix_disable_migration = 0;
+	}
+
 	msi_enabled = 1;
 	intr_register_pic(&msi_pic);
 	mtx_init(&msi_lock, "msi", NULL, MTX_DEF);

Modified: stable/10/sys/x86/xen/hvm.c
==============================================================================
--- stable/10/sys/x86/xen/hvm.c	Fri Aug  5 16:53:24 2016	(r303775)
+++ stable/10/sys/x86/xen/hvm.c	Fri Aug  5 17:13:25 2016	(r303776)
@@ -385,9 +385,29 @@ xen_hvm_init_hypercall_stubs(void)
 		return (ENXIO);
 
 	if (hypercall_stubs == NULL) {
+		int major, minor;
+
 		do_cpuid(base + 1, regs);
-		printf("XEN: Hypervisor version %d.%d detected.\n",
-		    regs[0] >> 16, regs[0] & 0xffff);
+
+		major = regs[0] >> 16;
+		minor = regs[0] & 0xffff;
+		printf("XEN: Hypervisor version %d.%d detected.\n", major,
+			minor);
+
+		if (((major < 4) || (major == 4 && minor <= 5)) &&
+		    msix_disable_migration == -1) {
+			/*
+			 * Xen hypervisors prior to 4.6.0 do not properly
+			 * handle updates to enabled MSI-X table entries,
+			 * so disable MSI-X interrupt migration in that
+			 * case.
+			 */
+			if (bootverbose)
+				printf(
+"Disabling MSI-X interrupt migration due to Xen hypervisor bug.\n"
+"Set machdep.msix_disable_migration=0 to forcefully enable it.\n");
+			msix_disable_migration = 1;
+		}
 	}
 
 	/*



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201608051713.u75HDQe8060409>