Date: Thu, 7 Feb 2019 10:30:11 +0000 (UTC) From: Marius Strobl <marius@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-12@freebsd.org Subject: svn commit: r343864 - in stable/12/sys: dev/e1000 dev/ixgbe dev/ixl dev/vmware/vmxnet3 net Message-ID: <201902071030.x17AUBGC029932@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: marius Date: Thu Feb 7 10:30:11 2019 New Revision: 343864 URL: https://svnweb.freebsd.org/changeset/base/343864 Log: MFC: r343578 (partial) - Stop iflib(4) from leaking MSI messages on detachment by calling bus_teardown_intr(9) before pci_release_msi(9). - Ensure that iflib(4) and associated drivers pass correct RIDs to bus_release_resource(9) by obtaining the RIDs via rman_get_rid(9) on the corresponding resources instead of using the RIDs initially passed to bus_alloc_resource_any(9) as the latter function may change those RIDs. Solely em(4) for the ioport resource (but not others) and bnxt(4) were using the correct RIDs by caching the ones returned by bus_alloc_resource_any(9). - Change the logic of iflib_msix_init() around to only map the MSI-X BAR if MSI-X is actually supported, i. e. pci_msix_count(9) returns > 0. Otherwise the "Unable to map MSIX table " message triggers for devices that simply don't support MSI-X and the user may think that something is wrong while in fact everything works as expected. - Put some (mostly redundant) debug messages emitted by iflib(4) and em(4) during attachment under bootverbose. The non-verbose output of em(4) seen during attachment now is close to the one prior to the conversion to iflib(4). - Replace various variants of spelling "MSI-X" (several in messages) with "MSI-X" as used in the PCI specifications. - Remove some trailing whitespace from messages emitted by iflib(4) and change them to consistently start with uppercase. - Remove some obsolete comments about releasing interrupts from drivers and correct a few others. Reviewed by: erj, Jacob Keller, shurd Differential Revision: https://reviews.freebsd.org/D18980 Modified: stable/12/sys/dev/e1000/if_em.c stable/12/sys/dev/e1000/if_em.h stable/12/sys/dev/ixgbe/if_ix.c stable/12/sys/dev/ixgbe/if_ixv.c stable/12/sys/dev/ixl/if_iavf.c stable/12/sys/dev/ixl/if_ixl.c stable/12/sys/dev/ixl/ixl_iw.c stable/12/sys/dev/ixl/ixl_pf_main.c stable/12/sys/dev/vmware/vmxnet3/if_vmx.c stable/12/sys/net/iflib.c Directory Properties: stable/12/ (props changed) Modified: stable/12/sys/dev/e1000/if_em.c ============================================================================== --- stable/12/sys/dev/e1000/if_em.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/e1000/if_em.c Thu Feb 7 10:30:11 2019 (r343864) @@ -293,7 +293,7 @@ static void em_disable_aspm(struct adapter *); int em_intr(void *arg); static void em_disable_promisc(if_ctx_t ctx); -/* MSIX handlers */ +/* MSI-X handlers */ static int em_if_msix_intr_assign(if_ctx_t, int); static int em_msix_link(void *); static void em_handle_link(void *context); @@ -780,7 +780,9 @@ em_if_attach_pre(if_ctx_t ctx) scctx->isc_msix_bar = PCIR_BAR(EM_MSIX_BAR); scctx->isc_tx_nsegments = EM_MAX_SCATTER; scctx->isc_nrxqsets_max = scctx->isc_ntxqsets_max = em_set_num_queues(ctx); - device_printf(dev, "attach_pre capping queues at %d\n", scctx->isc_ntxqsets_max); + if (bootverbose) + device_printf(dev, "attach_pre capping queues at %d\n", + scctx->isc_ntxqsets_max); if (adapter->hw.mac.type >= igb_mac_min) { int try_second_bar; @@ -1293,7 +1295,7 @@ em_if_init(if_ctx_t ctx) em_if_set_promisc(ctx, IFF_PROMISC); e1000_clear_hw_cntrs_base_generic(&adapter->hw); - /* MSI/X configuration for 82574 */ + /* MSI-X configuration for 82574 */ if (adapter->hw.mac.type == e1000_82574) { int tmp = E1000_READ_REG(&adapter->hw, E1000_CTRL_EXT); @@ -1419,7 +1421,7 @@ em_if_tx_queue_intr_enable(if_ctx_t ctx, uint16_t txqi /********************************************************************* * - * MSIX RX Interrupt Service routine + * MSI-X RX Interrupt Service routine * **********************************************************************/ static int @@ -1434,7 +1436,7 @@ em_msix_que(void *arg) /********************************************************************* * - * MSIX Link Fast Interrupt Service routine + * MSI-X Link Fast Interrupt Service routine * **********************************************************************/ static int @@ -1904,7 +1906,6 @@ em_allocate_pci_resources(if_ctx_t ctx) for (rid = PCIR_BAR(0); rid < PCIR_CIS;) { val = pci_read_config(dev, rid, 4); if (EM_BAR_TYPE(val) == EM_BAR_TYPE_IO) { - adapter->io_rid = rid; break; } rid += 4; @@ -1916,8 +1917,8 @@ em_allocate_pci_resources(if_ctx_t ctx) device_printf(dev, "Unable to locate IO BAR\n"); return (ENXIO); } - adapter->ioport = bus_alloc_resource_any(dev, - SYS_RES_IOPORT, &adapter->io_rid, RF_ACTIVE); + adapter->ioport = bus_alloc_resource_any(dev, SYS_RES_IOPORT, + &rid, RF_ACTIVE); if (adapter->ioport == NULL) { device_printf(dev, "Unable to allocate bus resource: " "ioport\n"); @@ -1937,7 +1938,7 @@ em_allocate_pci_resources(if_ctx_t ctx) /********************************************************************* * - * Setup the MSIX Interrupt handlers + * Set up the MSI-X Interrupt handlers * **********************************************************************/ static int @@ -1966,7 +1967,7 @@ em_if_msix_intr_assign(if_ctx_t ctx, int msix) * Set the bit to enable interrupt * in E1000_IMS -- bits 20 and 21 * are for RX0 and RX1, note this has - * NOTHING to do with the MSIX vector + * NOTHING to do with the MSI-X vector */ if (adapter->hw.mac.type == e1000_82574) { rx_que->eims = 1 << (20 + i); @@ -1993,7 +1994,7 @@ em_if_msix_intr_assign(if_ctx_t ctx, int msix) * Set the bit to enable interrupt * in E1000_IMS -- bits 22 and 23 * are for TX0 and TX1, note this has - * NOTHING to do with the MSIX vector + * NOTHING to do with the MSI-X vector */ if (adapter->hw.mac.type == e1000_82574) { tx_que->eims = 1 << (22 + i); @@ -2042,7 +2043,7 @@ igb_configure_queues(struct adapter *adapter) E1000_GPIE_MSIX_MODE | E1000_GPIE_EIAME | E1000_GPIE_PBA | E1000_GPIE_NSICR); - /* Turn on MSIX */ + /* Turn on MSI-X */ switch (adapter->hw.mac.type) { case e1000_82580: case e1000_i350: @@ -2176,7 +2177,7 @@ em_free_pci_resources(if_ctx_t ctx) struct em_rx_queue *que = adapter->rx_queues; device_t dev = iflib_get_dev(ctx); - /* Release all msix queue resources */ + /* Release all MSI-X queue resources */ if (adapter->intr_type == IFLIB_INTR_MSIX) iflib_irq_free(ctx, &adapter->irq); @@ -2184,24 +2185,26 @@ em_free_pci_resources(if_ctx_t ctx) iflib_irq_free(ctx, &que->que_irq); } - /* First release all the interrupt resources */ if (adapter->memory != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->memory); + rman_get_rid(adapter->memory), adapter->memory); adapter->memory = NULL; } if (adapter->flash != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, - EM_FLASH, adapter->flash); + rman_get_rid(adapter->flash), adapter->flash); adapter->flash = NULL; } - if (adapter->ioport != NULL) + + if (adapter->ioport != NULL) { bus_release_resource(dev, SYS_RES_IOPORT, - adapter->io_rid, adapter->ioport); + rman_get_rid(adapter->ioport), adapter->ioport); + adapter->ioport = NULL; + } } -/* Setup MSI or MSI/X */ +/* Set up MSI or MSI-X */ static int em_setup_msix(if_ctx_t ctx) { @@ -2843,7 +2846,9 @@ em_if_tx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, u txr->tx_paddr = paddrs[i*ntxqs]; } - device_printf(iflib_get_dev(ctx), "allocated for %d tx_queues\n", adapter->tx_num_queues); + if (bootverbose) + device_printf(iflib_get_dev(ctx), + "allocated for %d tx_queues\n", adapter->tx_num_queues); return (0); fail: em_if_queues_free(ctx); @@ -2881,9 +2886,11 @@ em_if_rx_queues_alloc(if_ctx_t ctx, caddr_t *vaddrs, u rxr->rx_base = (union e1000_rx_desc_extended *)vaddrs[i*nrxqs]; rxr->rx_paddr = paddrs[i*nrxqs]; } + + if (bootverbose) + device_printf(iflib_get_dev(ctx), + "allocated for %d rx_queues\n", adapter->rx_num_queues); - device_printf(iflib_get_dev(ctx), "allocated for %d rx_queues\n", adapter->rx_num_queues); - return (0); fail: em_if_queues_free(ctx); @@ -3125,7 +3132,7 @@ em_initialize_receive_unit(if_ctx_t ctx) rfctl = E1000_READ_REG(hw, E1000_RFCTL); rfctl |= E1000_RFCTL_EXTEN; /* - * When using MSIX interrupts we need to throttle + * When using MSI-X interrupts we need to throttle * using the EITR register (82574 only) */ if (hw->mac.type == e1000_82574) { @@ -3999,7 +4006,7 @@ em_add_hw_stats(struct adapter *adapter) "Driver dropped packets"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "link_irq", CTLFLAG_RD, &adapter->link_irq, - "Link MSIX IRQ Handled"); + "Link MSI-X IRQ Handled"); SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "mbuf_defrag_fail", CTLFLAG_RD, &adapter->mbuf_defrag_failed, "Defragmenting mbuf chain failed"); @@ -4516,7 +4523,7 @@ em_print_debug_info(struct adapter *adapter) /* * 82574 only: - * Write a new value to the EEPROM increasing the number of MSIX + * Write a new value to the EEPROM increasing the number of MSI-X * vectors from 3 to 5, for proper multiqueue support. */ static void @@ -4531,7 +4538,7 @@ em_enable_vectors_82574(if_ctx_t ctx) printf("Current cap: %#06x\n", edata); if (((edata & EM_NVM_MSIX_N_MASK) >> EM_NVM_MSIX_N_SHIFT) != 4) { device_printf(dev, "Writing to eeprom: increasing " - "reported MSIX vectors from 3 to 5...\n"); + "reported MSI-X vectors from 3 to 5...\n"); edata &= ~(EM_NVM_MSIX_N_MASK); edata |= 4 << EM_NVM_MSIX_N_SHIFT; e1000_write_nvm(hw, EM_NVM_PCIE_CTRL, 1, &edata); Modified: stable/12/sys/dev/e1000/if_em.h ============================================================================== --- stable/12/sys/dev/e1000/if_em.h Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/e1000/if_em.h Thu Feb 7 10:30:11 2019 (r343864) @@ -352,8 +352,8 @@ /* * 82574 has a nonstandard address for EIAC - * and since its only used in MSIX, and in - * the em driver only 82574 uses MSIX we can + * and since its only used in MSI-X, and in + * the em driver only 82574 uses MSI-X we can * solve it just using this define. */ #define EM_EIAC 0x000DC @@ -468,7 +468,6 @@ struct adapter { struct resource *memory; struct resource *flash; struct resource *ioport; - int io_rid; struct resource *res; void *tag; Modified: stable/12/sys/dev/ixgbe/if_ix.c ============================================================================== --- stable/12/sys/dev/ixgbe/if_ix.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/ixgbe/if_ix.c Thu Feb 7 10:30:11 2019 (r343864) @@ -2020,7 +2020,7 @@ ixgbe_if_msix_intr_assign(if_ctx_t ctx, int msix) cpu_id = rss_getcpu(i % rss_getnumbuckets()); } else { /* - * Bind the msix vector, and thus the + * Bind the MSI-X vector, and thus the * rings to the corresponding cpu. * * This just happens to match the default RSS @@ -3813,7 +3813,7 @@ ixgbe_free_pci_resources(if_ctx_t ctx) struct ix_rx_queue *que = adapter->rx_queues; device_t dev = iflib_get_dev(ctx); - /* Release all msix queue resources */ + /* Release all MSI-X queue resources */ if (adapter->intr_type == IFLIB_INTR_MSIX) iflib_irq_free(ctx, &adapter->irq); @@ -3823,13 +3823,9 @@ ixgbe_free_pci_resources(if_ctx_t ctx) } } - /* - * Free link/admin interrupt - */ if (adapter->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->pci_mem); - + rman_get_rid(adapter->pci_mem), adapter->pci_mem); } /* ixgbe_free_pci_resources */ /************************************************************************ Modified: stable/12/sys/dev/ixgbe/if_ixv.c ============================================================================== --- stable/12/sys/dev/ixgbe/if_ixv.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/ixgbe/if_ixv.c Thu Feb 7 10:30:11 2019 (r343864) @@ -1132,7 +1132,7 @@ ixv_free_pci_resources(if_ctx_t ctx) struct ix_rx_queue *que = adapter->rx_queues; device_t dev = iflib_get_dev(ctx); - /* Release all msix queue resources */ + /* Release all MSI-X queue resources */ if (adapter->intr_type == IFLIB_INTR_MSIX) iflib_irq_free(ctx, &adapter->irq); @@ -1142,10 +1142,9 @@ ixv_free_pci_resources(if_ctx_t ctx) } } - /* Clean the Legacy or Link interrupt last */ if (adapter->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), adapter->pci_mem); + rman_get_rid(adapter->pci_mem), adapter->pci_mem); } /* ixv_free_pci_resources */ /************************************************************************ Modified: stable/12/sys/dev/ixl/if_iavf.c ============================================================================== --- stable/12/sys/dev/ixl/if_iavf.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/ixl/if_iavf.c Thu Feb 7 10:30:11 2019 (r343864) @@ -359,7 +359,7 @@ iavf_if_attach_pre(if_ctx_t ctx) goto err_early; } - iavf_dbg_init(sc, "Allocated PCI resources and MSIX vectors\n"); + iavf_dbg_init(sc, "Allocated PCI resources and MSI-X vectors\n"); /* * XXX: This is called by init_shared_code in the PF driver, @@ -408,7 +408,8 @@ iavf_if_attach_pre(if_ctx_t ctx) goto err_aq; } - device_printf(dev, "VSIs %d, QPs %d, MSIX %d, RSS sizes: key %d lut %d\n", + device_printf(dev, + "VSIs %d, QPs %d, MSI-X %d, RSS sizes: key %d lut %d\n", sc->vf_res->num_vsis, sc->vf_res->num_queue_pairs, sc->vf_res->max_vectors, @@ -1486,7 +1487,7 @@ iavf_free_pci_resources(struct iavf_sc *sc) struct ixl_rx_queue *rx_que = vsi->rx_queues; device_t dev = sc->dev; - /* We may get here before stations are setup */ + /* We may get here before stations are set up */ if (rx_que == NULL) goto early; @@ -1499,7 +1500,7 @@ iavf_free_pci_resources(struct iavf_sc *sc) early: if (sc->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), sc->pci_mem); + rman_get_rid(sc->pci_mem), sc->pci_mem); } Modified: stable/12/sys/dev/ixl/if_ixl.c ============================================================================== --- stable/12/sys/dev/ixl/if_ixl.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/ixl/if_ixl.c Thu Feb 7 10:30:11 2019 (r343864) @@ -299,7 +299,7 @@ int ixl_limit_iwarp_msix = IXL_IW_MAX_MSIX; #endif TUNABLE_INT("hw.ixl.limit_iwarp_msix", &ixl_limit_iwarp_msix); SYSCTL_INT(_hw_ixl, OID_AUTO, limit_iwarp_msix, CTLFLAG_RDTUN, - &ixl_limit_iwarp_msix, 0, "Limit MSIX vectors assigned to iWARP"); + &ixl_limit_iwarp_msix, 0, "Limit MSI-X vectors assigned to iWARP"); #endif extern struct if_txrx ixl_txrx_hwb; @@ -684,14 +684,14 @@ ixl_if_attach_post(if_ctx_t ctx) error = ixl_iw_pf_attach(pf); if (error) { device_printf(dev, - "interfacing to iwarp driver failed: %d\n", + "interfacing to iWARP driver failed: %d\n", error); goto err; } else device_printf(dev, "iWARP ready\n"); } else - device_printf(dev, - "iwarp disabled on this device (no msix vectors)\n"); + device_printf(dev, "iWARP disabled on this device " + "(no MSI-X vectors)\n"); } else { pf->iw_enabled = false; device_printf(dev, "The device is not iWARP enabled\n"); @@ -857,7 +857,7 @@ ixl_if_init(if_ctx_t ctx) /* Set up RSS */ ixl_config_rss(pf); - /* Set up MSI/X routing and the ITR settings */ + /* Set up MSI-X routing and the ITR settings */ if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { ixl_configure_queue_intr_msix(pf); ixl_configure_itr(pf); Modified: stable/12/sys/dev/ixl/ixl_iw.c ============================================================================== --- stable/12/sys/dev/ixl/ixl_iw.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/ixl/ixl_iw.c Thu Feb 7 10:30:11 2019 (r343864) @@ -333,7 +333,7 @@ ixl_iw_pf_msix_init(void *pf_handle, if ((msix_info->aeq_vector < IXL_IW_VEC_BASE(pf)) || (msix_info->aeq_vector >= IXL_IW_VEC_LIMIT(pf))) { - printf("%s: invalid MSIX vector (%i) for AEQ\n", + printf("%s: invalid MSI-X vector (%i) for AEQ\n", __func__, msix_info->aeq_vector); return (EINVAL); } Modified: stable/12/sys/dev/ixl/ixl_pf_main.c ============================================================================== --- stable/12/sys/dev/ixl/ixl_pf_main.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/ixl/ixl_pf_main.c Thu Feb 7 10:30:11 2019 (r343864) @@ -278,7 +278,8 @@ retry: } /* Print a subset of the capability information. */ - device_printf(dev, "PF-ID[%d]: VFs %d, MSIX %d, VF MSIX %d, QPs %d, %s\n", + device_printf(dev, + "PF-ID[%d]: VFs %d, MSI-X %d, VF MSI-X %d, QPs %d, %s\n", hw->pf_id, hw->func_caps.num_vfs, hw->func_caps.num_msix_vectors, hw->func_caps.num_msix_vectors_vf, hw->func_caps.num_tx_qp, (hw->func_caps.mdio_port_mode == 2) ? "I2C" : @@ -505,7 +506,7 @@ ixl_intr(void *arg) /********************************************************************* * - * MSIX VSI Interrupt Service routine + * MSI-X VSI Interrupt Service routine * **********************************************************************/ int @@ -524,7 +525,7 @@ ixl_msix_que(void *arg) /********************************************************************* * - * MSIX Admin Queue Interrupt Service routine + * MSI-X Admin Queue Interrupt Service routine * **********************************************************************/ int @@ -791,7 +792,7 @@ ixl_configure_intr0_msix(struct ixl_pf *pf) /* * 0x7FF is the end of the queue list. * This means we won't use MSI-X vector 0 for a queue interrupt - * in MSIX mode. + * in MSI-X mode. */ wr32(hw, I40E_PFINT_LNKLST0, 0x7FF); /* Value is in 2 usec units, so 0x3E is 62*2 = 124 usecs. */ @@ -909,12 +910,12 @@ ixl_free_pci_resources(struct ixl_pf *pf) device_t dev = iflib_get_dev(vsi->ctx); struct ixl_rx_queue *rx_que = vsi->rx_queues; - /* We may get here before stations are setup */ + /* We may get here before stations are set up */ if (rx_que == NULL) goto early; /* - ** Release all msix VSI resources: + ** Release all MSI-X VSI resources: */ iflib_irq_free(vsi->ctx, &vsi->irq); @@ -923,7 +924,7 @@ ixl_free_pci_resources(struct ixl_pf *pf) early: if (pf->pci_mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, - PCIR_BAR(0), pf->pci_mem); + rman_get_rid(pf->pci_mem), pf->pci_mem); } void Modified: stable/12/sys/dev/vmware/vmxnet3/if_vmx.c ============================================================================== --- stable/12/sys/dev/vmware/vmxnet3/if_vmx.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/dev/vmware/vmxnet3/if_vmx.c Thu Feb 7 10:30:11 2019 (r343864) @@ -461,26 +461,24 @@ static void vmxnet3_free_resources(struct vmxnet3_softc *sc) { device_t dev; - int rid; dev = sc->vmx_dev; if (sc->vmx_res0 != NULL) { - rid = PCIR_BAR(0); - bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res0); + bus_release_resource(dev, SYS_RES_MEMORY, + rman_get_rid(sc->vmx_res0), sc->vmx_res0); sc->vmx_res0 = NULL; } if (sc->vmx_res1 != NULL) { - rid = PCIR_BAR(1); - bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->vmx_res1); + bus_release_resource(dev, SYS_RES_MEMORY, + rman_get_rid(sc->vmx_res1), sc->vmx_res1); sc->vmx_res1 = NULL; } if (sc->vmx_msix_res != NULL) { - rid = PCIR_BAR(2); - bus_release_resource(dev, SYS_RES_MEMORY, rid, - sc->vmx_msix_res); + bus_release_resource(dev, SYS_RES_MEMORY, + rman_get_rid(sc->vmx_msix_res), sc->vmx_msix_res); sc->vmx_msix_res = NULL; } } Modified: stable/12/sys/net/iflib.c ============================================================================== --- stable/12/sys/net/iflib.c Thu Feb 7 06:55:26 2019 (r343863) +++ stable/12/sys/net/iflib.c Thu Feb 7 10:30:11 2019 (r343864) @@ -4408,8 +4408,8 @@ iflib_device_register(device_t dev, void *sc, if_share main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; /* XXX change for per-queue sizes */ - device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", - scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); + device_printf(dev, "Using %d tx descriptors and %d rx descriptors\n", + scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); for (i = 0; i < sctx->isc_nrxqs; i++) { if (!powerof2(scctx->isc_nrxd[i])) { /* round down instead? */ @@ -4471,9 +4471,8 @@ iflib_device_register(device_t dev, void *sc, if_share MPASS(CPU_COUNT(&ctx->ifc_cpus) > 0); /* - ** Now setup MSI or MSI/X, should - ** return us the number of supported - ** vectors. (Will be 1 for MSI) + ** Now set up MSI or MSI-X, should return us the number of supported + ** vectors (will be 1 for a legacy interrupt and MSI). */ if (sctx->isc_flags & IFLIB_SKIP_MSIX) { msix = scctx->isc_vectors; @@ -4668,8 +4667,8 @@ iflib_pseudo_register(device_t dev, if_shared_ctx_t sc main_rxq = (sctx->isc_flags & IFLIB_HAS_RXCQ) ? 1 : 0; /* XXX change for per-queue sizes */ - device_printf(dev, "using %d tx descriptors and %d rx descriptors\n", - scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); + device_printf(dev, "Using %d tx descriptors and %d rx descriptors\n", + scctx->isc_ntxd[main_txq], scctx->isc_nrxd[main_rxq]); for (i = 0; i < sctx->isc_nrxqs; i++) { if (!powerof2(scctx->isc_nrxd[i])) { /* round down instead? */ @@ -4922,15 +4921,15 @@ static void iflib_free_intr_mem(if_ctx_t ctx) { - if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { - pci_release_msi(ctx->ifc_dev); - } if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_MSIX) { iflib_irq_free(ctx, &ctx->ifc_legacy_irq); } + if (ctx->ifc_softc_ctx.isc_intr != IFLIB_INTR_LEGACY) { + pci_release_msi(ctx->ifc_dev); + } if (ctx->ifc_msix_mem != NULL) { bus_release_resource(ctx->ifc_dev, SYS_RES_MEMORY, - ctx->ifc_softc_ctx.isc_msix_bar, ctx->ifc_msix_mem); + rman_get_rid(ctx->ifc_msix_mem), ctx->ifc_msix_mem); ctx->ifc_msix_mem = NULL; } } @@ -5757,11 +5756,13 @@ iflib_softirq_alloc_generic(if_ctx_t ctx, if_irq_t irq void iflib_irq_free(if_ctx_t ctx, if_irq_t irq) { + if (irq->ii_tag) bus_teardown_intr(ctx->ifc_dev, irq->ii_res, irq->ii_tag); if (irq->ii_res) - bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, irq->ii_rid, irq->ii_res); + bus_release_resource(ctx->ifc_dev, SYS_RES_IRQ, + rman_get_rid(irq->ii_res), irq->ii_res); } static int @@ -5955,7 +5956,9 @@ iflib_msix_init(if_ctx_t ctx) iflib_num_tx_queues = ctx->ifc_sysctl_ntxqs; iflib_num_rx_queues = ctx->ifc_sysctl_nrxqs; - device_printf(dev, "msix_init qsets capped at %d\n", imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); + if (bootverbose) + device_printf(dev, "msix_init qsets capped at %d\n", + imax(scctx->isc_ntxqsets, scctx->isc_nrxqsets)); bar = ctx->ifc_softc_ctx.isc_msix_bar; admincnt = sctx->isc_admin_intrcnt; @@ -5963,30 +5966,27 @@ iflib_msix_init(if_ctx_t ctx) if (scctx->isc_disable_msix) goto msi; + /* First try MSI-X */ + if ((msgs = pci_msix_count(dev)) == 0) { + if (bootverbose) + device_printf(dev, "MSI-X not supported or disabled\n"); + goto msi; + } /* * bar == -1 => "trust me I know what I'm doing" * Some drivers are for hardware that is so shoddily * documented that no one knows which bars are which * so the developer has to map all bars. This hack - * allows shoddy garbage to use msix in this framework. + * allows shoddy garbage to use MSI-X in this framework. */ if (bar != -1) { ctx->ifc_msix_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar, RF_ACTIVE); if (ctx->ifc_msix_mem == NULL) { - /* May not be enabled */ - device_printf(dev, "Unable to map MSIX table \n"); + device_printf(dev, "Unable to map MSI-X table\n"); goto msi; } } - /* First try MSI/X */ - if ((msgs = pci_msix_count(dev)) == 0) { /* system has msix disabled */ - device_printf(dev, "System has MSIX disabled \n"); - bus_release_resource(dev, SYS_RES_MEMORY, - bar, ctx->ifc_msix_mem); - ctx->ifc_msix_mem = NULL; - goto msi; - } #if IFLIB_DEBUG /* use only 1 qset in debug mode */ queuemsgs = min(msgs - admincnt, 1); @@ -5999,8 +5999,10 @@ iflib_msix_init(if_ctx_t ctx) queues = queuemsgs; #endif queues = imin(CPU_COUNT(&ctx->ifc_cpus), queues); - device_printf(dev, "pxm cpus: %d queue msgs: %d admincnt: %d\n", - CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt); + if (bootverbose) + device_printf(dev, + "intr CPUs: %d queue msgs: %d admincnt: %d\n", + CPU_COUNT(&ctx->ifc_cpus), queuemsgs, admincnt); #ifdef RSS /* If we're doing RSS, clamp at the number of RSS buckets */ if (queues > rss_getnumbuckets()) @@ -6036,11 +6038,13 @@ iflib_msix_init(if_ctx_t ctx) rx_queues = min(rx_queues, tx_queues); } - device_printf(dev, "using %d rx queues %d tx queues \n", rx_queues, tx_queues); + device_printf(dev, "Using %d rx queues %d tx queues\n", + rx_queues, tx_queues); vectors = rx_queues + admincnt; if ((err = pci_alloc_msix(dev, &vectors)) == 0) { - device_printf(dev, "Using MSIX interrupts with %d vectors\n", vectors); + device_printf(dev, "Using MSI-X interrupts with %d vectors\n", + vectors); scctx->isc_vectors = vectors; scctx->isc_nrxqsets = rx_queues; scctx->isc_ntxqsets = tx_queues; @@ -6049,7 +6053,8 @@ iflib_msix_init(if_ctx_t ctx) return (vectors); } else { device_printf(dev, - "failed to allocate %d msix vectors, err: %d - using MSI\n", vectors, err); + "failed to allocate %d MSI-X vectors, err: %d - using MSI\n", + vectors, err); bus_release_resource(dev, SYS_RES_MEMORY, bar, ctx->ifc_msix_mem); ctx->ifc_msix_mem = NULL; @@ -6185,7 +6190,7 @@ iflib_add_device_sysctl_pre(if_ctx_t ctx) "permit #txq != #rxq"); SYSCTL_ADD_INT(ctx_list, oid_list, OID_AUTO, "disable_msix", CTLFLAG_RWTUN, &ctx->ifc_softc_ctx.isc_disable_msix, 0, - "disable MSIX (default 0)"); + "disable MSI-X (default 0)"); SYSCTL_ADD_U16(ctx_list, oid_list, OID_AUTO, "rx_budget", CTLFLAG_RWTUN, &ctx->ifc_sysctl_rx_budget, 0, "set the rx budget");
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201902071030.x17AUBGC029932>