Date: Tue, 14 Mar 2017 15:15:38 +0000 (UTC) From: Hans Petter Selasky <hselasky@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-10@freebsd.org Subject: svn commit: r315247 - stable/10/sys/dev/pci Message-ID: <201703141515.v2EFFcVl099288@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: hselasky Date: Tue Mar 14 15:15:37 2017 New Revision: 315247 URL: https://svnweb.freebsd.org/changeset/base/315247 Log: MFC r310806: Add MSIX rewrite table quirk for use with VMs. This patch solves IRQ generation problems using the mlx5en(4) driver with xenserver v6.5.0 in SRIOV and PCI-passthrough modes. Until further the hw.pci.msix_rewrite_table quirk must be set manually in /boot/loader.conf . Reviewed by: jhb @ Sponsored by: Mellanox Technologies Modified: stable/10/sys/dev/pci/pci.c Directory Properties: stable/10/ (props changed) Modified: stable/10/sys/dev/pci/pci.c ============================================================================== --- stable/10/sys/dev/pci/pci.c Tue Mar 14 15:12:58 2017 (r315246) +++ stable/10/sys/dev/pci/pci.c Tue Mar 14 15:15:37 2017 (r315247) @@ -351,6 +351,11 @@ TUNABLE_INT("hw.pci.enable_msix", &pci_d SYSCTL_INT(_hw_pci, OID_AUTO, enable_msix, CTLFLAG_RW, &pci_do_msix, 1, "Enable support for MSI-X interrupts"); +static int pci_msix_rewrite_table = 0; +SYSCTL_INT(_hw_pci, OID_AUTO, msix_rewrite_table, CTLFLAG_RWTUN, + &pci_msix_rewrite_table, 0, + "Rewrite entire MSI-X table when updating MSI-X entries"); + static int pci_honor_msi_blacklist = 1; TUNABLE_INT("hw.pci.honor_msi_blacklist", &pci_honor_msi_blacklist); SYSCTL_INT(_hw_pci, OID_AUTO, honor_msi_blacklist, CTLFLAG_RD, @@ -1375,11 +1380,10 @@ pci_find_extcap_method(device_t dev, dev /* * Support for MSI-X message interrupts. */ -void -pci_enable_msix_method(device_t dev, device_t child, u_int index, - uint64_t address, uint32_t data) +static void +pci_write_msix_entry(device_t dev, u_int index, uint64_t address, uint32_t data) { - struct pci_devinfo *dinfo = device_get_ivars(child); + struct pci_devinfo *dinfo = device_get_ivars(dev); struct pcicfg_msix *msix = &dinfo->cfg.msix; uint32_t offset; @@ -1388,6 +1392,31 @@ pci_enable_msix_method(device_t dev, dev bus_write_4(msix->msix_table_res, offset, address & 0xffffffff); bus_write_4(msix->msix_table_res, offset + 4, address >> 32); bus_write_4(msix->msix_table_res, offset + 8, data); +} + +void +pci_enable_msix_method(device_t dev, device_t child, u_int index, + uint64_t address, uint32_t data) +{ + + if (pci_msix_rewrite_table) { + struct pci_devinfo *dinfo = device_get_ivars(child); + struct pcicfg_msix *msix = &dinfo->cfg.msix; + + /* + * Some VM hosts require MSIX to be disabled in the + * control register before updating the MSIX table + * entries are allowed. It is not enough to only + * disable MSIX while updating a single entry. MSIX + * must be disabled while updating all entries in the + * table. + */ + pci_write_config(child, + msix->msix_location + PCIR_MSIX_CTRL, + msix->msix_ctrl & ~PCIM_MSIXCTRL_MSIX_ENABLE, 2); + pci_resume_msix(child); + } else + pci_write_msix_entry(child, index, address, data); /* Enable MSI -> HT mapping. */ pci_ht_map_msi(child, address); @@ -1463,7 +1492,8 @@ pci_resume_msix(device_t dev) if (mte->mte_vector == 0 || mte->mte_handlers == 0) continue; mv = &msix->msix_vectors[mte->mte_vector - 1]; - pci_enable_msix(dev, i, mv->mv_address, mv->mv_data); + pci_write_msix_entry(dev, i, mv->mv_address, + mv->mv_data); pci_unmask_msix(dev, i); } } @@ -3980,12 +4010,20 @@ pci_setup_intr(device_t dev, device_t ch mv->mv_address = addr; mv->mv_data = data; } - if (mte->mte_handlers == 0) { + + /* + * The MSIX table entry must be made valid by + * incrementing the mte_handlers before + * calling pci_enable_msix() and + * pci_resume_msix(). Else the MSIX rewrite + * table quirk will not work as expected. + */ + mte->mte_handlers++; + if (mte->mte_handlers == 1) { pci_enable_msix(child, rid - 1, mv->mv_address, mv->mv_data); pci_unmask_msix(child, rid - 1); } - mte->mte_handlers++; } /*
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201703141515.v2EFFcVl099288>