Date: Sat, 8 Feb 2003 12:30:56 -0800 (PST) From: Sam Leffler <sam@FreeBSD.org> To: Perforce Change Reviews <perforce@freebsd.org> Subject: PERFORCE change 24844 for review Message-ID: <200302082030.h18KUuR9063864@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=24844 Change 24844 by sam@sam_ebb on 2003/02/08 12:30:08 bus_dma'd em driver (works on x86, not on sparc) replace rx logic with jumbo buffers (needs more work) Affected files ... .. //depot/projects/em/if_em.c#4 edit .. //depot/projects/em/if_em.h#4 edit Differences ... ==== //depot/projects/em/if_em.c#4 (text+ko) ==== @@ -1,6 +1,6 @@ /************************************************************************** -Copyright (c) 2001-2002, Intel Corporation +Copyright (c) 2001-2003, Intel Corporation All rights reserved. Redistribution and use in source and binary forms, with or without @@ -31,10 +31,14 @@ ***************************************************************************/ -/*$FreeBSD: src/sys/dev/em/if_em.c,v 1.18 2003/01/21 08:55:30 alfred Exp $*/ +/*$FreeBSD: src/sys/dev/em/if_em.c,v 1.2.2.11 2003/01/03 18:18:55 pdeuskar Exp $*/ #include <dev/em/if_em.h> +#ifndef M_DONTWAIT +#define M_DONTWAIT M_NOWAIT /* XXX temporary workaround */ +#endif + /********************************************************************* * Set this to one to display debug statistics *********************************************************************/ @@ -112,6 +116,9 @@ static void em_identify_hardware(struct adapter *); static int em_allocate_pci_resources(struct adapter *); static void em_free_pci_resources(struct adapter *); +static int em_dma_malloc(struct adapter *, bus_size_t, + struct em_dma_alloc *, int); +static void em_dma_free(struct adapter *, struct em_dma_alloc *); static void em_local_timer(void *); static int em_hardware_init(struct adapter *); static void em_setup_interface(device_t, struct adapter *); @@ -140,8 +147,8 @@ static void em_set_multi(struct adapter *); static void em_print_hw_stats(struct adapter *); static void em_print_link_status(struct adapter *); -static int em_get_buf(int i, struct adapter *, - struct mbuf *); +static int em_get_buf(int i, struct adapter *); +static int em_set_buf(int i, struct adapter *, struct mbuf *); static void em_enable_vlans(struct adapter *adapter); static int em_encap(struct adapter *adapter, struct mbuf *m_head); @@ -303,42 +310,38 @@ splx(s); return(ENXIO); } - - tsize = EM_ROUNDUP(adapter->num_tx_desc * - sizeof(struct em_tx_desc), 4096); - /* Allocate Transmit Descriptor ring */ - if (!(adapter->tx_desc_base = (struct em_tx_desc *) - contigmalloc(tsize, M_DEVBUF, M_NOWAIT, 0, ~0, PAGE_SIZE, 0))) { - printf("em%d: Unable to allocate TxDescriptor memory\n", - adapter->unit); + tsize = adapter->num_tx_desc * sizeof(struct em_tx_desc); + if (em_dma_malloc(adapter, tsize, &adapter->txdma, BUS_DMA_NOWAIT)) { + printf("em%d: Unable to allocate tx_desc memory\n", + adapter->unit); em_free_pci_resources(adapter); splx(s); return(ENOMEM); } - - rsize = EM_ROUNDUP(adapter->num_rx_desc * - sizeof(struct em_rx_desc), 4096); + adapter->tx_desc_base = (struct em_tx_desc *) adapter->txdma.dma_vaddr; + adapter->tx_desc_end = adapter->tx_desc_base + adapter->num_tx_desc; /* Allocate Receive Descriptor ring */ - if (!(adapter->rx_desc_base = (struct em_rx_desc *) - contigmalloc(rsize, M_DEVBUF, M_NOWAIT, 0, ~0, PAGE_SIZE, 0))) { - printf("em%d: Unable to allocate rx_desc memory\n", - adapter->unit); + rsize = adapter->num_rx_desc * sizeof(struct em_rx_desc); + if (em_dma_malloc(adapter, rsize, &adapter->rxdma, BUS_DMA_NOWAIT)) { + printf("em%d: Unable to allocate rx_desc memory\n", + adapter->unit); em_free_pci_resources(adapter); - contigfree(adapter->tx_desc_base, tsize, M_DEVBUF); + em_dma_free(adapter, &adapter->txdma); splx(s); return(ENOMEM); } + adapter->rx_desc_base = (struct em_rx_desc *) adapter->rxdma.dma_vaddr; /* Initialize the hardware */ if (em_hardware_init(adapter)) { printf("em%d: Unable to initialize the hardware\n", adapter->unit); em_free_pci_resources(adapter); - contigfree(adapter->tx_desc_base, tsize, M_DEVBUF); - contigfree(adapter->rx_desc_base, rsize, M_DEVBUF); + em_dma_free(adapter, &adapter->txdma); + em_dma_free(adapter, &adapter->rxdma); splx(s); return(EIO); } @@ -395,7 +398,6 @@ struct adapter * adapter = device_get_softc(dev); struct ifnet *ifp = &adapter->interface_data.ac_if; int s; - int size; INIT_DEBUGOUT("em_detach: begin"); s = splimp(); @@ -409,21 +411,15 @@ #endif em_free_pci_resources(adapter); - size = EM_ROUNDUP(adapter->num_tx_desc * - sizeof(struct em_tx_desc), 4096); - /* Free Transmit Descriptor ring */ if (adapter->tx_desc_base) { - contigfree(adapter->tx_desc_base, size, M_DEVBUF); + em_dma_free(adapter, &adapter->txdma); adapter->tx_desc_base = NULL; } - size = EM_ROUNDUP(adapter->num_rx_desc * - sizeof(struct em_rx_desc), 4096); - /* Free Receive Descriptor ring */ if (adapter->rx_desc_base) { - contigfree(adapter->rx_desc_base, size, M_DEVBUF); + em_dma_free(adapter, &adapter->rxdma); adapter->rx_desc_base = NULL; } @@ -519,7 +515,7 @@ static int em_ioctl(struct ifnet *ifp, IOCTL_CMD_TYPE command, caddr_t data) { - int s, mask, error = 0; + int s, mask, error = 0, buf_size; struct ifreq *ifr = (struct ifreq *) data; struct adapter * adapter = ifp->if_softc; @@ -537,23 +533,56 @@ } else { ifp->if_mtu = ifr->ifr_mtu; adapter->hw.max_frame_size = - ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; + /* + * Adjust the receive buffer size according to the + * mtu. The hardware only handles fixed-size values + * for bounding receive frame sizes so we have to + * round the mtu up. Note that for jumbo frames + * we have a per-frame overhead that we must account + * for in selecting the buffer size. + */ + buf_size = adapter->hw.max_frame_size; + if (buf_size > MCLBYTES) + buf_size += JUMBO_OVERHEAD; + if (buf_size <= 2048) { + adapter->rx_buffer_len = EM_RXBUFFER_2048; + } else if (buf_size <= 4096) { + adapter->rx_buffer_len = EM_RXBUFFER_4096; + } else if (buf_size <= 8192) { + adapter->rx_buffer_len = EM_RXBUFFER_8192; + } else { + adapter->rx_buffer_len = EM_RXBUFFER_16384; + } em_init(adapter); } break; - case SIOCSIFFLAGS: - IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)"); - if (ifp->if_flags & IFF_UP) { + case SIOCSIFFLAGS: + IOCTL_DEBUGOUT("ioctl rcv'd: SIOCSIFFLAGS (Set Interface Flags)"); + if (ifp->if_flags & IFF_UP) { +#if __FreeBSD_version < 500000 + switch (ifp->if_flags & (IFF_RUNNING|IFF_PROMISC)) { + case IFF_RUNNING|IFF_PROMISC: + em_set_promisc(adapter); + break; + case IFF_RUNNING: + em_disable_promisc(adapter); + break; + default: + em_init(adapter); + break; + } +#else if (!(ifp->if_flags & IFF_RUNNING)) em_init(adapter); - em_disable_promisc(adapter); em_set_promisc(adapter); - } else { - if (ifp->if_flags & IFF_RUNNING) { - em_stop(adapter); - } - } +#endif + } else { + if (ifp->if_flags & IFF_RUNNING) { + em_stop(adapter); + } + } break; case SIOCADDMULTI: case SIOCDELMULTI: @@ -617,7 +646,6 @@ ifp->if_flags &= ~IFF_RUNNING; - em_stop(adapter); em_init(adapter); ifp->if_oerrors++; @@ -703,7 +731,7 @@ em_disable_intr(adapter); else #endif /* DEVICE_POLLING */ - em_enable_intr(adapter); + em_enable_intr(adapter); splx(s); return; @@ -769,7 +797,6 @@ } #endif /* DEVICE_POLLING */ - em_disable_intr(adapter); while (loop_cnt > 0 && (reg_icr = E1000_READ_REG(&adapter->hw, ICR)) != 0) { @@ -924,6 +951,19 @@ return(0); } +static void +em_tx_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error) +{ + struct em_q *q = arg; + + if (error) + return; + KASSERT(nsegs <= EM_MAX_SCATTER, + ("Too many DMA segments returned when mapping tx packet")); + q->nsegs = nsegs; + bcopy(seg, q->segs, nsegs * sizeof(seg[0])); +} + /********************************************************************* * * This routine maps the mbufs to tx descriptors. @@ -934,104 +974,142 @@ static int em_encap(struct adapter *adapter, struct mbuf *m_head) { - vm_offset_t virtual_addr; u_int32_t txd_upper; u_int32_t txd_lower; - int txd_used, i, txd_saved; - struct mbuf *mp; + int i; #if __FreeBSD_version < 500000 struct ifvlan *ifv = NULL; #else struct m_tag *mtag; #endif - struct em_buffer *tx_buffer = NULL; - struct em_tx_desc *current_tx_desc = NULL; + struct em_q *q; + struct em_tx_desc *tx_desc; struct ifnet *ifp = &adapter->interface_data.ac_if; + int error; - /* Force a cleanup if number of TX descriptors available hits the threshold */ - if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) + /* + * Force a cleanup if the number of available + * TX descriptors hits the threshold. + */ + if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { em_clean_transmit_interrupts(adapter); + if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { + adapter->no_tx_desc_avail1++; + return (ENOBUFS); + } + } + + /* + * Grab a queue entry. + */ + q = STAILQ_FIRST(&adapter->tx_freeq); + if (!q) { + adapter->no_tx_buffer++; + return (ENOBUFS); + } + STAILQ_REMOVE_HEAD(&adapter->tx_freeq, next); - if (adapter->num_tx_desc_avail <= EM_TX_CLEANUP_THRESHOLD) { - adapter->no_tx_desc_avail1++; + /* + * Map the packet for DMA. + */ + if (bus_dmamap_create(adapter->txtag, BUS_DMA_NOWAIT, &q->map)) { + adapter->no_tx_map_avail++; + STAILQ_INSERT_TAIL(&adapter->tx_freeq, q, next); + return (ENOMEM); + } + error = bus_dmamap_load_mbuf(adapter->txtag, q->map, + m_head, em_tx_cb, q, BUS_DMA_NOWAIT); + if (error != 0) { + adapter->no_tx_dma_setup++; + bus_dmamap_destroy(adapter->txtag, q->map); + STAILQ_INSERT_TAIL(&adapter->tx_freeq, q, next); + return (error); + } + KASSERT(q->nsegs != 0, ("em_encap: empty packet")); + if (q->nsegs > adapter->num_tx_desc_avail) { + adapter->no_tx_desc_avail2++; + bus_dmamap_destroy(adapter->txtag, q->map); + STAILQ_INSERT_TAIL(&adapter->tx_freeq, q, next); return (ENOBUFS); } - if (ifp->if_hwassist > 0) { + /* + * Check if an offload context is needed for this packet. + * One is generated when checksum processing is offloaded + * to the NIC. This is done by inserting a descriptor + * before the packet. Offload descriptors must be inserted + * each time the ``current protocol'' (UDP or TCP) changes. + * + * NB: if we ever find out how to do IPsec offloading + * this would be done here. + */ + if (ifp->if_hwassist > 0) em_transmit_checksum_setup(adapter, m_head, &txd_upper, &txd_lower); - } else txd_upper = txd_lower = 0; - /* Find out if we are in vlan mode */ + /* + * Setup the tx descriptors for the packet. + */ + tx_desc = adapter->tx_desc_next; + for (i = 0; i < q->nsegs; i++) { + tx_desc->buffer_addr = htole64(q->segs[i].ds_addr); + tx_desc->lower.data = htole32( + adapter->txd_cmd | txd_lower | q->segs[i].ds_len); + tx_desc->upper.data = htole32(txd_upper); + + if (++tx_desc == adapter->tx_desc_end) + tx_desc = adapter->tx_desc_base; + } + adapter->tx_desc_next = tx_desc; + adapter->num_tx_desc_avail -= q->nsegs; + + /* + * Backup to the last descriptor in packet to tack on any + * VLAN bits and an EOP (End of Packet) marker. + */ + if (--tx_desc < adapter->tx_desc_base) + tx_desc = adapter->tx_desc_end-1; #if __FreeBSD_version < 500000 if ((m_head->m_flags & (M_PROTO1|M_PKTHDR)) == (M_PROTO1|M_PKTHDR) && m_head->m_pkthdr.rcvif != NULL && - m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN) + m_head->m_pkthdr.rcvif->if_type == IFT_L2VLAN) { ifv = m_head->m_pkthdr.rcvif->if_softc; -#else - mtag = VLAN_OUTPUT_TAG(ifp, m_head); -#endif - - i = adapter->next_avail_tx_desc; - txd_saved = i; - txd_used = 0; - for (mp = m_head; mp != NULL; mp = mp->m_next) { - if (mp->m_len == 0) - continue; - - if (txd_used == adapter->num_tx_desc_avail) { - adapter->next_avail_tx_desc = txd_saved; - adapter->no_tx_desc_avail2++; - return (ENOBUFS); + if (ifv != NULL) { + /* Set the vlan id */ + tx_desc->upper.fields.special = htole16(ifv->ifv_tag); + /* Tell hardware to add tag */ + tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE); } - - tx_buffer = &adapter->tx_buffer_area[i]; - current_tx_desc = &adapter->tx_desc_base[i]; - virtual_addr = mtod(mp, vm_offset_t); - current_tx_desc->buffer_addr = vtophys(virtual_addr); - - current_tx_desc->lower.data = (adapter->txd_cmd | txd_lower | mp->m_len); - current_tx_desc->upper.data = (txd_upper); - - if (++i == adapter->num_tx_desc) - i = 0; - - tx_buffer->m_head = NULL; - - txd_used++; } - - adapter->num_tx_desc_avail -= txd_used; - adapter->next_avail_tx_desc = i; - -#if __FreeBSD_version < 500000 - if (ifv != NULL) { - /* Set the vlan id */ - current_tx_desc->upper.fields.special = ifv->ifv_tag; #else + mtag = VLAN_OUTPUT_TAG(ifp, m_head); if (mtag != NULL) { /* Set the vlan id */ - current_tx_desc->upper.fields.special = VLAN_TAG_VALUE(mtag); -#endif + tx_desc->upper.fields.special = htole16(VLAN_TAG_VALUE(mtag)); /* Tell hardware to add tag */ - current_tx_desc->lower.data |= E1000_TXD_CMD_VLE; + tx_desc->lower.data |= htole32(E1000_TXD_CMD_VLE); } +#endif + tx_desc->lower.data |= htole32(E1000_TXD_CMD_EOP); - tx_buffer->m_head = m_head; - - /* - * Last Descriptor of Packet needs End Of Packet (EOP) + /* + * Mark the queue entry, sync the data to be sent, and + * stick the request at the end of the transmit queue. */ - current_tx_desc->lower.data |= (E1000_TXD_CMD_EOP); + q->m_head = m_head; + bus_dmamap_sync(adapter->txtag, q->map, BUS_DMASYNC_PREWRITE); + q->desc = tx_desc; + STAILQ_INSERT_TAIL(&adapter->tx_q, q, next); /* * Advance the Transmit Descriptor Tail (Tdt), this tells the E1000 * that this frame is available to transmit. */ - E1000_WRITE_REG(&adapter->hw, TDT, i); + E1000_WRITE_REG(&adapter->hw, TDT, + adapter->tx_desc_next - adapter->tx_desc_base); return (0); } @@ -1348,6 +1426,94 @@ return; } +/* + * Manage DMA'able memory. + */ +static void +em_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + if (error) + return; + *(bus_addr_t*) arg = segs->ds_addr; +} + +static int +em_dma_malloc( + struct adapter *adapter, + bus_size_t size, + struct em_dma_alloc *dma, + int mapflags +) +{ + int r; + + r = bus_dma_tag_create(NULL, /* parent */ + PAGE_SIZE, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + size, /* maxsize */ + 1, /* nsegments */ + size, /* maxsegsize */ + BUS_DMA_ALLOCNOW, /* flags */ + &dma->dma_tag); + if (r != 0) { + printf("em%d: em_dma_malloc: bus_dma_tag_create failed; " + "error %u\n", adapter->unit, r); + goto fail_0; + } + + r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map); + if (r != 0) { + printf("em%d: em_dma_malloc: bus_dmamap_create failed; " + "error %u\n", adapter->unit, r); + goto fail_1; + } + + r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr, + BUS_DMA_NOWAIT, &dma->dma_map); + if (r != 0) { + printf("em%d: em_dma_malloc: bus_dmammem_alloc failed; " + "size %u, error %u\n", adapter->unit, size, r); + goto fail_2; + } + + r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr, + size, + em_dmamap_cb, + &dma->dma_paddr, + mapflags | BUS_DMA_NOWAIT); + if (r != 0) { + printf("em%d: em_dma_malloc: bus_dmamap_load failed; " + "error %u\n", adapter->unit, r); + goto fail_3; + } + + dma->dma_size = size; + return (0); + +fail_3: + bus_dmamap_unload(dma->dma_tag, dma->dma_map); +fail_2: + bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); +fail_1: + bus_dmamap_destroy(dma->dma_tag, dma->dma_map); + bus_dma_tag_destroy(dma->dma_tag); +fail_0: + dma->dma_map = NULL; + dma->dma_tag = NULL; + return (r); +} + +static void +em_dma_free(struct adapter *adapter, struct em_dma_alloc *dma) +{ + bus_dmamap_unload(dma->dma_tag, dma->dma_map); + bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map); + bus_dmamap_destroy(dma->dma_tag, dma->dma_map); + bus_dma_tag_destroy(dma->dma_tag); +} + /********************************************************************* * * Initialize the hardware to a configuration as specified by the @@ -1477,25 +1643,27 @@ /********************************************************************* * - * Allocate memory for tx_buffer structures. The tx_buffer stores all - * the information needed to transmit a packet on the wire. + * Allocate memory for transmission. The free q is populated with + * the transmit buffers needed to transmit a packet on the wire. * **********************************************************************/ static int em_allocate_transmit_structures(struct adapter * adapter) { - if (!(adapter->tx_buffer_area = - (struct em_buffer *) malloc(sizeof(struct em_buffer) * - adapter->num_tx_desc, M_DEVBUF, - M_NOWAIT))) { - printf("em%d: Unable to allocate tx_buffer memory\n", - adapter->unit); - return ENOMEM; + int i; + + for (i = 0; i < adapter->num_tx_desc; i++) { + struct em_q *q; + + q = malloc(sizeof(struct em_q), M_DEVBUF, M_NOWAIT | M_ZERO); + if (!q) { + printf("em%d: Unable to allocate tx_buffer memory\n", + adapter->unit); + return ENOMEM; + } + STAILQ_INSERT_TAIL(&adapter->tx_freeq, q, next); } - bzero(adapter->tx_buffer_area, - sizeof(struct em_buffer) * adapter->num_tx_desc); - return 0; } @@ -1507,14 +1675,42 @@ static int em_setup_transmit_structures(struct adapter * adapter) { - if (em_allocate_transmit_structures(adapter)) + /* + * Insure these are initialized so cleanup always works. + */ + STAILQ_INIT(&adapter->tx_freeq); + STAILQ_INIT(&adapter->tx_q); + + /* + * Setup DMA descriptor areas. + */ + /* XXX have 64-bit addressability */ + if (bus_dma_tag_create(NULL, /* parent */ + PAGE_SIZE, 0, /* alignment, bounds */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + EM_RXBUFFER_16384, /* maxsize */ + EM_MAX_SCATTER, /* nsegments */ + EM_RXBUFFER_16384, /* maxsegsize */ + BUS_DMA_ALLOCNOW, /* flags */ + &adapter->txtag)) { + printf("em%d: Unable to allocate TX DMA tag\n", adapter->unit); + return ENOMEM; + } + + if (em_allocate_transmit_structures(adapter)) { + /* XXX who cleans up freeq? */ + bus_dma_tag_destroy(adapter->txtag); + adapter->txtag = NULL; return ENOMEM; + } bzero((void *) adapter->tx_desc_base, (sizeof(struct em_tx_desc)) * adapter->num_tx_desc); - adapter->next_avail_tx_desc = 0; - adapter->oldest_used_tx_desc = 0; + adapter->tx_desc_next = adapter->tx_desc_base; + adapter->tx_desc_next_to_check = adapter->tx_desc_next; /* Set number of descriptors available */ adapter->num_tx_desc_avail = adapter->num_tx_desc; @@ -1537,8 +1733,7 @@ u_int32_t reg_tipg = 0; /* Setup the Base and Length of the Tx Descriptor Ring */ - E1000_WRITE_REG(&adapter->hw, TDBAL, - vtophys((vm_offset_t) adapter->tx_desc_base)); + E1000_WRITE_REG(&adapter->hw, TDBAL, adapter->txdma.dma_paddr); E1000_WRITE_REG(&adapter->hw, TDBAH, 0); E1000_WRITE_REG(&adapter->hw, TDLEN, adapter->num_tx_desc * @@ -1614,24 +1809,29 @@ static void em_free_transmit_structures(struct adapter * adapter) { - struct em_buffer *tx_buffer; - int i; + INIT_DEBUGOUT("free_transmit_structures: begin"); + + while (!STAILQ_EMPTY(&adapter->tx_q)) { + struct em_q *q; - INIT_DEBUGOUT("free_transmit_structures: begin"); + q = STAILQ_FIRST(&adapter->tx_q); + STAILQ_REMOVE_HEAD(&adapter->tx_q, next); + m_freem(q->m_head); + bus_dmamap_unload(adapter->txtag, q->map); + bus_dmamap_destroy(adapter->txtag, q->map); + free(q, M_DEVBUF); + } + while (!STAILQ_EMPTY(&adapter->tx_freeq)) { + struct em_q *q; - if (adapter->tx_buffer_area != NULL) { - tx_buffer = adapter->tx_buffer_area; - for (i = 0; i < adapter->num_tx_desc; i++, tx_buffer++) { - if (tx_buffer->m_head != NULL) - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; - } + q = STAILQ_FIRST(&adapter->tx_freeq); + STAILQ_REMOVE_HEAD(&adapter->tx_freeq, next); + free(q, M_DEVBUF); } - if (adapter->tx_buffer_area != NULL) { - free(adapter->tx_buffer_area, M_DEVBUF); - adapter->tx_buffer_area = NULL; + if (adapter->txtag != NULL) { + bus_dma_tag_destroy(adapter->txtag); + adapter->txtag = NULL; } - return; } /********************************************************************* @@ -1648,8 +1848,6 @@ u_int32_t *txd_lower) { struct em_context_desc *TXD; - struct em_buffer *tx_buffer; - int curr_txd; if (mp->m_pkthdr.csum_flags) { @@ -1679,22 +1877,23 @@ return; } - /* If we reach this point, the checksum offload context + /* + * If we reach this point, the checksum offload context * needs to be reset. + * + * XXX doesn't handle being out of descriptors */ - curr_txd = adapter->next_avail_tx_desc; - tx_buffer = &adapter->tx_buffer_area[curr_txd]; - TXD = (struct em_context_desc *) &adapter->tx_desc_base[curr_txd]; + TXD = (struct em_context_desc *) adapter->tx_desc_next; TXD->lower_setup.ip_fields.ipcss = ETHER_HDR_LEN; TXD->lower_setup.ip_fields.ipcso = ETHER_HDR_LEN + offsetof(struct ip, ip_sum); TXD->lower_setup.ip_fields.ipcse = - ETHER_HDR_LEN + sizeof(struct ip) - 1; + htole16(ETHER_HDR_LEN + sizeof(struct ip) - 1); TXD->upper_setup.tcp_fields.tucss = ETHER_HDR_LEN + sizeof(struct ip); - TXD->upper_setup.tcp_fields.tucse = 0; + TXD->upper_setup.tcp_fields.tucse = htole16(0); if (adapter->active_checksum_context == OFFLOAD_TCP_IP) { TXD->upper_setup.tcp_fields.tucso = @@ -1706,16 +1905,12 @@ offsetof(struct udphdr, uh_sum); } - TXD->tcp_seg_setup.data = 0; - TXD->cmd_and_length = (adapter->txd_cmd | E1000_TXD_CMD_DEXT); + TXD->tcp_seg_setup.data = htole32(0); + TXD->cmd_and_length = htole32(adapter->txd_cmd | E1000_TXD_CMD_DEXT); - tx_buffer->m_head = NULL; - - if (++curr_txd == adapter->num_tx_desc) - curr_txd = 0; - + if (++(adapter->tx_desc_next) == adapter->tx_desc_end) + adapter->tx_desc_next = adapter->tx_desc_base; adapter->num_tx_desc_avail--; - adapter->next_avail_tx_desc = curr_txd; return; } @@ -1730,42 +1925,44 @@ static void em_clean_transmit_interrupts(struct adapter * adapter) { - int s; - int i, num_avail; - struct em_buffer *tx_buffer; + int num_avail; struct em_tx_desc *tx_desc; + struct em_q *q; - if (adapter->num_tx_desc_avail == adapter->num_tx_desc) - return; + if (adapter->num_tx_desc_avail == adapter->num_tx_desc) + return; - s = splimp(); #ifdef DBG_STATS adapter->clean_tx_interrupts++; #endif num_avail = adapter->num_tx_desc_avail; - i = adapter->oldest_used_tx_desc; + tx_desc = adapter->tx_desc_next_to_check; + q = STAILQ_FIRST(&adapter->tx_q); - tx_buffer = &adapter->tx_buffer_area[i]; - tx_desc = &adapter->tx_desc_base[i]; - - while(tx_desc->upper.fields.status & E1000_TXD_STAT_DD) { - + while (tx_desc->upper.fields.status & E1000_TXD_STAT_DD) { tx_desc->upper.data = 0; num_avail++; - - if (tx_buffer->m_head) { - m_freem(tx_buffer->m_head); - tx_buffer->m_head = NULL; + KASSERT(q != NULL, + ("em_clean_transmit_interrupts: descriptors w/o packet!")); + if (q && q->desc == tx_desc) { + /* + * Packet is complete, reclaim resources. + */ + STAILQ_REMOVE_HEAD(&adapter->tx_q, next); + bus_dmamap_sync(adapter->txtag, q->map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(adapter->txtag, q->map); + bus_dmamap_destroy(adapter->txtag, q->map); + m_freem(q->m_head); + STAILQ_INSERT_TAIL(&adapter->tx_freeq, q, next); + + q = STAILQ_FIRST(&adapter->tx_q); } - - if (++i == adapter->num_tx_desc) - i = 0; + if (++tx_desc == adapter->tx_desc_end) + tx_desc = adapter->tx_desc_base; + } - tx_buffer = &adapter->tx_buffer_area[i]; - tx_desc = &adapter->tx_desc_base[i]; - } - - adapter->oldest_used_tx_desc = i; + adapter->tx_desc_next_to_check = tx_desc; /* * If we have enough room, clear IFF_OACTIVE to tell the stack @@ -1783,88 +1980,392 @@ ifp->if_timer = EM_TX_TIMEOUT; } adapter->num_tx_desc_avail = num_avail; - splx(s); - return; +} + +/* + * Memory management for jumbo frames. + */ + +/* + * Accounting info goes at the end so the data area is page-aligned. + */ +#define JENTRY(_base) (&((struct em_jumbo *)(_base))[-1]) +#define JDATA(_entry) ((void *)((caddr_t)&(_entry)[1])) + +static void +em_jcleanup(struct adapter *adapter) +{ + struct em_jumbo *entry; + + while ((entry = SLIST_FIRST(&adapter->jfree)) != NULL) { + SLIST_REMOVE_HEAD(&adapter->jfree, u.next); + +#if __FreeBSD_version < 500000 + KASSERT(entry->refcnt == 0, + ("em_jcleanup: freeing referenced jumbogram (ref %u)", + entry->refcnt)); +#endif + KASSERT(entry->ix < adapter->num_rx_buffers, + ("em_jcleanup: invalid rx descriptor index (%u > %u)", + entry->ix, adapter->num_rx_desc)); + bus_dmamem_free(adapter->rxtag, entry, + adapter->rx_buffer_area[entry->ix].map); + } +} + +static int +em_jsetup(struct adapter *adapter) +{ + int i, error = 0; + + for (i = 0; i < adapter->num_rx_buffers; i++) { + struct em_buffer *rx_buffer = &adapter->rx_buffer_area[i]; + struct em_jumbo *entry; + void *v; + + error = bus_dmamem_alloc(adapter->rxtag, &v, + BUS_DMA_NOWAIT, &rx_buffer->map); + if (error != 0) + break; + /* + * Load the buffer now and record the physical address + * so we can avoid doing this for each receive operation. + */ + entry = (struct em_jumbo *) v; + error = bus_dmamap_load(adapter->rxtag, + rx_buffer->map, + JDATA(entry), + adapter->rxtag_maxsize - JUMBO_OVERHEAD, + em_dmamap_cb, + &entry->paddr, + BUS_DMA_NOWAIT); + if (error != 0) { + printf("em%d: em_jsetup: bus_dmamap_load failed; " + "error %u\n", adapter->unit, error); + bus_dmamem_free(adapter->rxtag, v, rx_buffer->map); + break; + } + entry->ix = i; + entry->refcnt = 0; + SLIST_INSERT_HEAD(&adapter->jfree, entry, u.next); + } + if (i != adapter->num_rx_buffers) { + printf("em%d: Unable to allocate memory for jumbo buffers\n", + adapter->unit); + em_jcleanup(adapter); + } + return (error); +} + +#if __FreeBSD_version < 500000 +/* + * Adjust usage count on a jumbo buffer. + */ +static void +em_jref(caddr_t buf, u_int size) +{ + struct em_jumbo *entry = JENTRY(buf); + + KASSERT(entry->u.adapter != NULL, ("em_jref: no adapter pointer!")); + entry->refcnt++; +} + +/* + * Release a jumbo buffer. + */ +static void +em_jfree(caddr_t buf, u_int size) +{ + struct em_jumbo *entry = JENTRY(buf); + + KASSERT(entry->refcnt != 0, ("em_jfree: reference count is zero!")); + if (--(entry->refcnt) == 0) { + struct adapter *adapter = entry->u.adapter; + + KASSERT(adapter != NULL, ("em_jfree: no adapter pointer!")); + KASSERT(size + JUMBO_OVERHEAD == adapter->rxtag_maxsize, + ("em_jfree: buffer has wrong size! (%u != %u)", + size + JUMBO_OVERHEAD, adapter->rxtag_maxsize)); + KASSERT(entry->ix < adapter->num_rx_buffers, + ("em_jfree: bogus index %u > %u", entry->ix, + adapter->num_rx_buffers)); + /* NB: could also verify physical address */ + + /* return jumbo buffer to adapter's free list */ + SLIST_INSERT_HEAD(&adapter->jfree, entry, u.next); + } +} +#else +/* + * Release a jumbo buffer. + */ +static void +em_jfree(void *buf, void *arg) +{ + struct em_jumbo *entry = arg; + struct adapter *adapter = entry->u.adapter; + + KASSERT(adapter != NULL, ("em_jfree: no adapter pointer!")); + KASSERT(entry->ix < adapter->num_rx_buffers, + ("em_jfree: bogus index %u > %u", entry->ix, + adapter->num_rx_buffers)); + /* NB: could also verify physical address */ + + /* return jumbo buffer to adapter's free list */ + SLIST_INSERT_HEAD(&adapter->jfree, entry, u.next); +} +#endif + +/* + * Allocate a jumbo buffer from the free list and set it up for use. + */ +static int +em_jalloc(struct adapter *adapter, int i, struct mbuf *mp) +{ + struct em_jumbo *entry; + + entry = SLIST_FIRST(&adapter->jfree); + if (entry == NULL) { + INIT_DEBUGOUT1("em%d: em_jalloc: no jumbo buffers", + adapter->unit); + return (ENOBUFS); + } + + /* + * Take it off the free list, rewrite the list pointer + * with the adapter, and mark the reference count. + */ + SLIST_REMOVE_HEAD(&adapter->jfree, u.next); + entry->u.adapter = adapter; + >>> TRUNCATED FOR MAIL (1000 lines) <<< To Unsubscribe: send mail to majordomo@FreeBSD.org with "unsubscribe p4-projects" in the body of the message
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200302082030.h18KUuR9063864>