Skip site navigation (1)Skip section navigation (2)
Date:      Tue, 9 Jul 2013 23:12:26 +0000 (UTC)
From:      Marius Strobl <marius@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org
Subject:   svn commit: r253120 - head/sys/dev/pci
Message-ID:  <201307092312.r69NCQY1082902@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: marius
Date: Tue Jul  9 23:12:26 2013
New Revision: 253120
URL: http://svnweb.freebsd.org/changeset/base/253120

Log:
  - As it turns out, not only MSI-X is broken for devices passed through by
    VMware up to at least ESXi 5.1. Actually, using INTx in that case instead
    may still result in interrupt storms, with MSI being the only working
    option in some configurations. So introduce a PCI_QUIRK_DISABLE_MSIX quirk
    which only blacklists MSI-X but not also MSI and use it for the VMware
    PCI-PCI-bridges. Note that, currently, we still assume that if MSI doesn't
    work, MSI-X won't work either - but that's part of the internal logic and
    not guaranteed as part of the API contract. While at it, add and employ
    a pci_has_quirk() helper.
    Reported and tested by: Paul Bucher
  - Use NULL instead of 0 for pointers.
  
  Submitted by:	jhb (mostly)
  Approved by:	jhb
  MFC after:	3 days

Modified:
  head/sys/dev/pci/pci.c
  head/sys/dev/pci/pci_pci.c
  head/sys/dev/pci/pcib_private.h
  head/sys/dev/pci/pcivar.h

Modified: head/sys/dev/pci/pci.c
==============================================================================
--- head/sys/dev/pci/pci.c	Tue Jul  9 22:04:35 2013	(r253119)
+++ head/sys/dev/pci/pci.c	Tue Jul  9 23:12:26 2013	(r253120)
@@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$");
 	(((cfg)->hdrtype == PCIM_HDRTYPE_NORMAL && reg == PCIR_BIOS) ||	\
 	 ((cfg)->hdrtype == PCIM_HDRTYPE_BRIDGE && reg == PCIR_BIOS_1))
 
+static int		pci_has_quirk(uint32_t devid, int quirk);
 static pci_addr_t	pci_mapbase(uint64_t mapreg);
 static const char	*pci_maptype(uint64_t mapreg);
 static int		pci_mapsize(uint64_t testval);
@@ -119,6 +120,7 @@ static void		pci_enable_msix(device_t de
 static void		pci_mask_msix(device_t dev, u_int index);
 static void		pci_unmask_msix(device_t dev, u_int index);
 static int		pci_msi_blacklisted(void);
+static int		pci_msix_blacklisted(void);
 static void		pci_resume_msi(device_t dev);
 static void		pci_resume_msix(device_t dev);
 static int		pci_remap_intr_method(device_t bus, device_t dev,
@@ -185,7 +187,7 @@ static device_method_t pci_methods[] = {
 DEFINE_CLASS_0(pci, pci_driver, pci_methods, sizeof(struct pci_softc));
 
 static devclass_t pci_devclass;
-DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, 0);
+DRIVER_MODULE(pci, pcib, pci_driver, pci_devclass, pci_modevent, NULL);
 MODULE_VERSION(pci, 1);
 
 static char	*pci_vendordata;
@@ -195,15 +197,16 @@ struct pci_quirk {
 	uint32_t devid;	/* Vendor/device of the card */
 	int	type;
 #define	PCI_QUIRK_MAP_REG	1 /* PCI map register in weird place */
-#define	PCI_QUIRK_DISABLE_MSI	2 /* MSI/MSI-X doesn't work */
+#define	PCI_QUIRK_DISABLE_MSI	2 /* Neither MSI nor MSI-X work */
 #define	PCI_QUIRK_ENABLE_MSI_VM	3 /* Older chipset in VM where MSI works */
 #define	PCI_QUIRK_UNMAP_REG	4 /* Ignore PCI map register */
+#define	PCI_QUIRK_DISABLE_MSIX	5 /* MSI-X doesn't work */
 	int	arg1;
 	int	arg2;
 };
 
 static const struct pci_quirk pci_quirks[] = {
-	/* The Intel 82371AB and 82443MX has a map register at offset 0x90. */
+	/* The Intel 82371AB and 82443MX have a map register at offset 0x90. */
 	{ 0x71138086, PCI_QUIRK_MAP_REG,	0x90,	 0 },
 	{ 0x719b8086, PCI_QUIRK_MAP_REG,	0x90,	 0 },
 	/* As does the Serverworks OSB4 (the SMBus mapping register) */
@@ -238,8 +241,8 @@ static const struct pci_quirk pci_quirks
 	 * MSI-X allocation doesn't work properly for devices passed through
 	 * by VMware up to at least ESXi 5.1.
 	 */
-	{ 0x079015ad, PCI_QUIRK_DISABLE_MSI,	0,	0 }, /* PCI/PCI-X */
-	{ 0x07a015ad, PCI_QUIRK_DISABLE_MSI,	0,	0 }, /* PCIe */
+	{ 0x079015ad, PCI_QUIRK_DISABLE_MSIX,	0,	0 }, /* PCI/PCI-X */
+	{ 0x07a015ad, PCI_QUIRK_DISABLE_MSIX,	0,	0 }, /* PCIe */
 
 	/*
 	 * Some virtualization environments emulate an older chipset
@@ -321,7 +324,7 @@ SYSCTL_INT(_hw_pci, OID_AUTO, enable_msi
 static int pci_honor_msi_blacklist = 1;
 TUNABLE_INT("hw.pci.honor_msi_blacklist", &pci_honor_msi_blacklist);
 SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RD,
-    &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI");
+    &pci_honor_msi_blacklist, 1, "Honor chipset blacklist for MSI/MSI-X");
 
 #if defined(__i386__) || defined(__amd64__)
 static int pci_usb_takeover = 1;
@@ -334,6 +337,18 @@ SYSCTL_INT(_hw_pci, OID_AUTO, usb_early_
 Disable this if you depend on BIOS emulation of USB devices, that is\n\
 you use USB devices (like keyboard or mouse) but do not load USB drivers");
 
+static int
+pci_has_quirk(uint32_t devid, int quirk)
+{
+	const struct pci_quirk *q;
+
+	for (q = &pci_quirks[0]; q->devid; q++) {
+		if (q->devid == devid && q->type == quirk)
+			return (1);
+	}
+	return (0);
+}
+
 /* Find a device_t by bus/slot/function in domain 0 */
 
 device_t
@@ -1426,8 +1441,8 @@ pci_alloc_msix_method(device_t dev, devi
 	if (cfg->msi.msi_alloc != 0 || cfg->msix.msix_alloc != 0)
 		return (ENXIO);
 
-	/* If MSI is blacklisted for this system, fail. */
-	if (pci_msi_blacklisted())
+	/* If MSI-X is blacklisted for this system, fail. */
+	if (pci_msix_blacklisted())
 		return (ENXIO);
 
 	/* MSI-X capability present? */
@@ -1983,38 +1998,15 @@ pci_remap_intr_method(device_t bus, devi
 int
 pci_msi_device_blacklisted(device_t dev)
 {
-	const struct pci_quirk *q;
 
 	if (!pci_honor_msi_blacklist)
 		return (0);
 
-	for (q = &pci_quirks[0]; q->devid; q++) {
-		if (q->devid == pci_get_devid(dev) &&
-		    q->type == PCI_QUIRK_DISABLE_MSI)
-			return (1);
-	}
-	return (0);
-}
-
-/*
- * Returns true if a specified chipset supports MSI when it is
- * emulated hardware in a virtual machine.
- */
-static int
-pci_msi_vm_chipset(device_t dev)
-{
-	const struct pci_quirk *q;
-
-	for (q = &pci_quirks[0]; q->devid; q++) {
-		if (q->devid == pci_get_devid(dev) &&
-		    q->type == PCI_QUIRK_ENABLE_MSI_VM)
-			return (1);
-	}
-	return (0);
+	return (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSI));
 }
 
 /*
- * Determine if MSI is blacklisted globally on this sytem.  Currently,
+ * Determine if MSI is blacklisted globally on this system.  Currently,
  * we just check for blacklisted chipsets as represented by the
  * host-PCI bridge at device 0:0:0.  In the future, it may become
  * necessary to check other system attributes, such as the kenv values
@@ -2031,9 +2023,14 @@ pci_msi_blacklisted(void)
 	/* Blacklist all non-PCI-express and non-PCI-X chipsets. */
 	if (!(pcie_chipset || pcix_chipset)) {
 		if (vm_guest != VM_GUEST_NO) {
+			/*
+			 * Whitelist older chipsets in virtual
+			 * machines known to support MSI.
+			 */
 			dev = pci_find_bsf(0, 0, 0);
 			if (dev != NULL)
-				return (pci_msi_vm_chipset(dev) == 0);
+				return (!pci_has_quirk(pci_get_devid(dev),
+					PCI_QUIRK_ENABLE_MSI_VM));
 		}
 		return (1);
 	}
@@ -2045,6 +2042,45 @@ pci_msi_blacklisted(void)
 }
 
 /*
+ * Returns true if the specified device is blacklisted because MSI-X
+ * doesn't work.  Note that this assumes that if MSI doesn't work,
+ * MSI-X doesn't either.
+ */
+int
+pci_msix_device_blacklisted(device_t dev)
+{
+
+	if (!pci_honor_msi_blacklist)
+		return (0);
+
+	if (pci_has_quirk(pci_get_devid(dev), PCI_QUIRK_DISABLE_MSIX))
+		return (1);
+
+	return (pci_msi_device_blacklisted(dev));
+}
+
+/*
+ * Determine if MSI-X is blacklisted globally on this system.  If MSI
+ * is blacklisted, assume that MSI-X is as well.  Check for additional
+ * chipsets where MSI works but MSI-X does not.
+ */
+static int
+pci_msix_blacklisted(void)
+{
+	device_t dev;
+
+	if (!pci_honor_msi_blacklist)
+		return (0);
+
+	dev = pci_find_bsf(0, 0, 0);
+	if (dev != NULL && pci_has_quirk(pci_get_devid(dev),
+	    PCI_QUIRK_DISABLE_MSIX))
+		return (1);
+
+	return (pci_msi_blacklisted());
+}
+
+/*
  * Attempt to allocate *count MSI messages.  The actual number allocated is
  * returned in *count.  After this function returns, each message will be
  * available to the driver as SYS_RES_IRQ resources starting at a rid 1.

Modified: head/sys/dev/pci/pci_pci.c
==============================================================================
--- head/sys/dev/pci/pci_pci.c	Tue Jul  9 22:04:35 2013	(r253119)
+++ head/sys/dev/pci/pci_pci.c	Tue Jul  9 23:12:26 2013	(r253120)
@@ -100,7 +100,7 @@ static device_method_t pcib_methods[] = 
 static devclass_t pcib_devclass;
 
 DEFINE_CLASS_0(pcib, pcib_driver, pcib_methods, sizeof(struct pcib_softc));
-DRIVER_MODULE(pcib, pci, pcib_driver, pcib_devclass, 0, 0);
+DRIVER_MODULE(pcib, pci, pcib_driver, pcib_devclass, NULL, NULL);
 
 #ifdef NEW_PCIB
 /*
@@ -624,6 +624,9 @@ pcib_attach_common(device_t dev)
     if (pci_msi_device_blacklisted(dev))
 	sc->flags |= PCIB_DISABLE_MSI;
 
+    if (pci_msix_device_blacklisted(dev))
+	sc->flags |= PCIB_DISABLE_MSIX;
+
     /*
      * Intel 815, 845 and other chipsets say they are PCI-PCI bridges,
      * but have a ProgIF of 0x80.  The 82801 family (AA, AB, BAM/CAM,
@@ -1379,7 +1382,7 @@ pcib_alloc_msix(device_t pcib, device_t 
 	struct pcib_softc *sc = device_get_softc(pcib);
 	device_t bus;
 
-	if (sc->flags & PCIB_DISABLE_MSI)
+	if (sc->flags & PCIB_DISABLE_MSIX)
 		return (ENXIO);
 	bus = device_get_parent(pcib);
 	return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));

Modified: head/sys/dev/pci/pcib_private.h
==============================================================================
--- head/sys/dev/pci/pcib_private.h	Tue Jul  9 22:04:35 2013	(r253119)
+++ head/sys/dev/pci/pcib_private.h	Tue Jul  9 23:12:26 2013	(r253120)
@@ -91,6 +91,7 @@ struct pcib_softc 
     uint32_t	flags;		/* flags */
 #define	PCIB_SUBTRACTIVE	0x1
 #define	PCIB_DISABLE_MSI	0x2
+#define	PCIB_DISABLE_MSIX	0x4
     uint16_t	command;	/* command register */
     u_int	domain;		/* domain number */
     u_int	pribus;		/* primary bus number */

Modified: head/sys/dev/pci/pcivar.h
==============================================================================
--- head/sys/dev/pci/pcivar.h	Tue Jul  9 22:04:35 2013	(r253119)
+++ head/sys/dev/pci/pcivar.h	Tue Jul  9 23:12:26 2013	(r253120)
@@ -490,6 +490,7 @@ device_t pci_find_class(uint8_t class, u
 int	pci_pending_msix(device_t dev, u_int index);
 
 int	pci_msi_device_blacklisted(device_t dev);
+int	pci_msix_device_blacklisted(device_t dev);
 
 void	pci_ht_map_msi(device_t dev, uint64_t addr);
 



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201307092312.r69NCQY1082902>