Date: Wed, 14 Oct 2015 23:47:23 +0000 (UTC) From: "Conrad E. Meyer" <cem@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-head@freebsd.org Subject: svn commit: r289344 - head/sys/dev/ntb/ntb_hw Message-ID: <201510142347.t9ENlN9L050637@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: cem Date: Wed Oct 14 23:47:23 2015 New Revision: 289344 URL: https://svnweb.freebsd.org/changeset/base/289344 Log: NTB: Remap MSI-X messages over available slots Remap MSI-X messages over available slots rather than falling back to legacy INTx when fewer MSI-X slots are available than were requested. N.B. the Linux driver does *not* do this. To aid in testing, a tunable 'hw.ntb.force_remap_mode' has been added. It defaults to off (0). When the tunable is enabled and sufficient slots were available, the driver restricts the number of slots by one and remaps the MSI-X messages over the remaining slots. In case this is actually not okay (as I don't yet have access to this hardware to test), a tunable 'hw.ntb.prefer_intx_to_remap' has been added. It defaults to off (0). When the tunable is enabled and fewer slots are available than requested, fall back to legacy INTx mode rather than attempting to remap MSI-X messages. Suggested by: jhb Reviewed by: jhb (earlier version) Sponsored by: EMC / Isilon Storage Division Modified: head/sys/dev/ntb/ntb_hw/ntb_hw.c Modified: head/sys/dev/ntb/ntb_hw/ntb_hw.c ============================================================================== --- head/sys/dev/ntb/ntb_hw/ntb_hw.c Wed Oct 14 23:47:08 2015 (r289343) +++ head/sys/dev/ntb/ntb_hw/ntb_hw.c Wed Oct 14 23:47:23 2015 (r289344) @@ -200,6 +200,7 @@ static int map_mmr_bar(struct ntb_softc static int map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); static void ntb_unmap_pci_bar(struct ntb_softc *ntb); +static int ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); static int ntb_setup_interrupts(struct ntb_softc *ntb); static int ntb_setup_legacy_interrupt(struct ntb_softc *ntb); static int ntb_setup_xeon_msix(struct ntb_softc *ntb, uint32_t num_vectors); @@ -551,10 +552,55 @@ ntb_setup_soc_msix(struct ntb_softc *ntb return (0); } +/* + * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector + * cannot be allocated for each MSI-X message. JHB seems to think remapping + * should be okay. This tunable should enable us to test that hypothesis + * when someone gets their hands on some Xeon hardware. + */ +static int ntb_force_remap_mode; +SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, + &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" + " to a smaller number of ithreads, even if the desired number are " + "available"); + +/* + * In case it is NOT ok, give consumers an abort button. + */ +static int ntb_prefer_intx; +SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, + &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " + "than remapping MSI-X messages over available slots (match Linux driver " + "behavior)"); + +/* + * Remap the desired number of MSI-X messages to available ithreads in a simple + * round-robin fashion. + */ +static int +ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) +{ + u_int *vectors; + uint32_t i; + int rc; + + if (ntb_prefer_intx != 0) + return (ENXIO); + + vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); + + for (i = 0; i < desired; i++) + vectors[i] = (i % avail) + 1; + + rc = pci_remap_msix(dev, desired, vectors); + free(vectors, M_NTB); + return (rc); +} + static int ntb_setup_interrupts(struct ntb_softc *ntb) { - uint32_t num_vectors; + uint32_t desired_vectors, num_vectors; int rc; ntb->allocated_interrupts = 0; @@ -568,10 +614,27 @@ ntb_setup_interrupts(struct ntb_softc *n ntb_reg_write(2, ntb->reg_ofs.ldb_mask, (uint16_t) ~(1 << XEON_LINK_DB)); - num_vectors = MIN(pci_msix_count(ntb->device), + num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), ntb->limits.max_db_bits); - if (num_vectors >= 1) - pci_alloc_msix(ntb->device, &num_vectors); + if (desired_vectors >= 1) { + rc = pci_alloc_msix(ntb->device, &num_vectors); + + if (ntb_force_remap_mode != 0 && rc == 0 && + num_vectors == desired_vectors) + num_vectors--; + + if (rc == 0 && num_vectors < desired_vectors) { + rc = ntb_remap_msix(ntb->device, desired_vectors, + num_vectors); + if (rc == 0) + num_vectors = desired_vectors; + else + pci_release_msi(ntb->device); + } + if (rc != 0) + num_vectors = 1; + } else + num_vectors = 1; ntb_create_callbacks(ntb, num_vectors);
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201510142347.t9ENlN9L050637>