Date: Mon, 18 May 2009 22:54:47 +0000 (UTC) From: Kip Macy <kmacy@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r192354 - in user/kmacy/releng_7_2_fcs/sys: dev/cxgb dev/cxgb/sys dev/ti dev/vx kern net netgraph netinet nfsclient nfsserver sys vm Message-ID: <200905182254.n4IMslGT077662@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: kmacy Date: Mon May 18 22:54:47 2009 New Revision: 192354 URL: http://svn.freebsd.org/changeset/base/192354 Log: merge 191958 - Merge in changes from jeff_mbuf with one addition. - In m_refm() make all referenced data available in the ext area for compatibility with existing code that makes assumptions about referenced sizes. also fix cxgb compile Modified: user/kmacy/releng_7_2_fcs/sys/dev/cxgb/cxgb_sge.c user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/mvec.h user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/uipc_mvec.c user/kmacy/releng_7_2_fcs/sys/dev/ti/if_ti.c user/kmacy/releng_7_2_fcs/sys/dev/vx/if_vx.c user/kmacy/releng_7_2_fcs/sys/kern/kern_mbuf.c user/kmacy/releng_7_2_fcs/sys/kern/uipc_mbuf.c user/kmacy/releng_7_2_fcs/sys/kern/uipc_mbuf2.c user/kmacy/releng_7_2_fcs/sys/kern/uipc_sockbuf.c user/kmacy/releng_7_2_fcs/sys/net/if_gre.c user/kmacy/releng_7_2_fcs/sys/netgraph/ng_tty.c user/kmacy/releng_7_2_fcs/sys/netinet/ip_options.c user/kmacy/releng_7_2_fcs/sys/netinet/sctp_os_bsd.h user/kmacy/releng_7_2_fcs/sys/nfsclient/nfsm_subs.h user/kmacy/releng_7_2_fcs/sys/nfsserver/nfsm_subs.h user/kmacy/releng_7_2_fcs/sys/sys/mbuf.h user/kmacy/releng_7_2_fcs/sys/sys/sockbuf.h user/kmacy/releng_7_2_fcs/sys/vm/uma.h user/kmacy/releng_7_2_fcs/sys/vm/uma_core.c user/kmacy/releng_7_2_fcs/sys/vm/uma_dbg.c user/kmacy/releng_7_2_fcs/sys/vm/uma_int.h Modified: user/kmacy/releng_7_2_fcs/sys/dev/cxgb/cxgb_sge.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/dev/cxgb/cxgb_sge.c Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/dev/cxgb/cxgb_sge.c Mon May 18 22:54:47 2009 (r192354) @@ -517,7 +517,7 @@ refill_fl(adapter_t *sc, struct sge_fl * struct refill_fl_cb_arg cb_arg; caddr_t cl; int err, count = 0; - int header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t); + int header_size = sizeof(struct mbuf) + sizeof(uint32_t); cb_arg.error = 0; while (n--) { @@ -2362,7 +2362,7 @@ t3_sge_alloc_qset(adapter_t *sc, u_int i q->rspq.size = p->rspq_size; - header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + sizeof(struct m_ext_) + sizeof(uint32_t); + header_size = sizeof(struct mbuf) + sizeof(uint32_t); q->txq[TXQ_ETH].stop_thres = nports * flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); @@ -2537,26 +2537,15 @@ init_cluster_mbuf(caddr_t cl, int flags, struct mbuf *m; int header_size; - header_size = sizeof(struct m_hdr) + sizeof(struct pkthdr) + - sizeof(struct m_ext_) + sizeof(uint32_t); - + header_size = sizeof(struct mbuf); bzero(cl, header_size); m = (struct mbuf *)cl; cxgb_ext_inited++; SLIST_INIT(&m->m_pkthdr.tags); - m->m_type = MT_DATA; - m->m_flags = flags | M_NOFREE | M_EXT; - m->m_data = cl + header_size; - m->m_ext.ext_buf = cl; - m->m_ext.ref_cnt = (uint32_t *)(cl + header_size - sizeof(uint32_t)); - m->m_ext.ext_size = m_getsizefromtype(type); - m->m_ext.ext_free = ext_free_handler; - m->m_ext.ext_arg1 = cl; - m->m_ext.ext_arg2 = (void *)(uintptr_t)type; - m->m_ext.ext_type = EXT_EXTREF; - *(m->m_ext.ref_cnt) = 1; - DPRINTF("data=%p ref_cnt=%p\n", m->m_data, m->m_ext.ref_cnt); + m_extadd(m, cl + header_size, + m_getsizefromtype(type), ext_free_handler, cl, NULL, + flags | M_NOFREE | M_EXT, EXT_NET_DRV); } Modified: user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/mvec.h ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/mvec.h Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/mvec.h Mon May 18 22:54:47 2009 (r192354) @@ -113,9 +113,9 @@ struct mbuf_iovec { #define MIOVBYTES 512 #define MAX_MBUF_IOV ((MHLEN-8)/sizeof(struct mbuf_iovec)) -#define MAX_MIOVEC_IOV ((MIOVBYTES-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) -#define MAX_CL_IOV ((MCLBYTES-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) -#define MAX_PAGE_IOV ((MJUMPAGESIZE-sizeof(struct m_hdr)-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) +#define MAX_MIOVEC_IOV ((MIOVBYTES-MBUF_HEADER_SIZE-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) +#define MAX_CL_IOV ((MCLBYTES-MBUF_HEADER_SIZE-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) +#define MAX_PAGE_IOV ((MJUMPAGESIZE-MBUF_HEADER_SIZE-sizeof(struct pkthdr)-8)/sizeof(struct mbuf_iovec)) struct mbuf_vec { uint16_t mv_first; /* first valid cluster */ @@ -250,7 +250,7 @@ m_freem_iovec(struct mbuf_iovec *mi) KASSERT((mi->mi_flags & M_NOFREE) == 0, ("no free set on mbuf")); KASSERT(m->m_next == NULL, ("freeing chain")); cxgb_mbufs_outstanding--; - m_free_fast(m); + m_free_fast(zone_mbuf, m); break; case EXT_PACKET: cxgb_pack_outstanding--; @@ -270,7 +270,6 @@ m_freem_iovec(struct mbuf_iovec *mi) case EXT_NET_DRV: case EXT_MOD_TYPE: case EXT_DISPOSABLE: - case EXT_EXTREF: mb_free_ext_fast(mi, mi->mi_type, -1); break; default: Modified: user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/uipc_mvec.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/uipc_mvec.c Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/dev/cxgb/sys/uipc_mvec.c Mon May 18 22:54:47 2009 (r192354) @@ -147,7 +147,6 @@ _mcl_collapse_mbuf(struct mbuf_iovec *mi mi->mi_base = m->m_ext.ext_buf; mi->mi_type = m->m_ext.ext_type; mi->mi_size = m->m_ext.ext_size; - mi->mi_refcnt = m->m_ext.ref_cnt; if (m->m_ext.ext_type == EXT_PACKET) { mi->mi_mbuf = m; #ifdef INVARIANTS @@ -256,7 +255,7 @@ retry: goto err_out; } - memcpy(m0, *m, sizeof(struct m_hdr) + sizeof(struct pkthdr)); + memcpy(m0, *m, MBUF_HEADER_SIZE + sizeof(struct pkthdr)); m0->m_type = type; KASSERT(m0->m_pkthdr.len, ("empty packet being marshalled")); mv = mtomv(m0); @@ -313,8 +312,7 @@ busdma_map_sg_vec(struct mbuf **m, struc if ((m0 = mcl_alloc(pkt_count, &type)) == NULL) return (ENOMEM); - memcpy(m0, *m, sizeof(struct m_hdr) + - sizeof(struct pkthdr)); + memcpy(m0, *m, MBUF_HEADER_SIZE + sizeof(struct pkthdr)); m0->m_type = type; mv = mtomv(m0); mv->mv_count = pkt_count; @@ -357,7 +355,7 @@ mb_free_ext_fast(struct mbuf_iovec *mi, #ifdef INVARIANTS cxgb_mbufs_outstanding--; #endif - m_free_fast((struct mbuf *)cl); + m_free_fast(zone_mbuf, (struct mbuf *)cl); return; default: break; @@ -391,10 +389,6 @@ mb_free_ext_fast(struct mbuf_iovec *mi, case EXT_MOD_TYPE: case EXT_DISPOSABLE: *(mi->mi_refcnt) = 0; - uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, - mi->mi_ext.ref_cnt)); - /* FALLTHROUGH */ - case EXT_EXTREF: KASSERT(mi->mi_ext.ext_free != NULL, ("%s: ext_free not set", __func__)); #if __FreeBSD_version >= 800016 Modified: user/kmacy/releng_7_2_fcs/sys/dev/ti/if_ti.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/dev/ti/if_ti.c Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/dev/ti/if_ti.c Mon May 18 22:54:47 2009 (r192354) @@ -1308,12 +1308,11 @@ ti_newbuf_mini(sc, i, m) if (m_new == NULL) { return (ENOBUFS); } - m_new->m_len = m_new->m_pkthdr.len = MHLEN; } else { m_new = m; - m_new->m_data = m_new->m_pktdat; - m_new->m_len = m_new->m_pkthdr.len = MHLEN; + m_new->m_data = M_START(m); } + m_new->m_len = m_new->m_pkthdr.len = m_new->m_size; m_adj(m_new, ETHER_ALIGN); r = &sc->ti_rdata->ti_rx_mini_ring[i]; Modified: user/kmacy/releng_7_2_fcs/sys/dev/vx/if_vx.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/dev/vx/if_vx.c Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/dev/vx/if_vx.c Mon May 18 22:54:47 2009 (r192354) @@ -825,8 +825,11 @@ vx_get(struct vx_softc *sc, u_int totlen /* Convert one of our saved mbuf's. */ sc->vx_next_mb = (sc->vx_next_mb + 1) % MAX_MBS; m->m_data = m->m_pktdat; - m->m_flags = M_PKTHDR; - bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); + m->m_flags |= M_PKTHDR; + if (m_pkthdr_init(m, M_NOWAIT)) { + m_free(m); + return NULL; + } } m->m_pkthdr.rcvif = ifp; m->m_pkthdr.len = totlen; Modified: user/kmacy/releng_7_2_fcs/sys/kern/kern_mbuf.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/kern/kern_mbuf.c Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/kern/kern_mbuf.c Mon May 18 22:54:47 2009 (r192354) @@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$"); #include <sys/domain.h> #include <sys/eventhandler.h> #include <sys/kernel.h> +#include <sys/ktr.h> #include <sys/protosw.h> #include <sys/smp.h> #include <sys/sysctl.h> @@ -210,27 +211,24 @@ SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbsta /* * Zones from which we allocate. */ -uma_zone_t zone_mbuf; uma_zone_t zone_clust; -uma_zone_t zone_pack; uma_zone_t zone_jumbop; uma_zone_t zone_jumbo9; uma_zone_t zone_jumbo16; -uma_zone_t zone_ext_refcnt; +uma_zone_t zone_mbuf; +uma_zone_t zone_pack; /* * Local prototypes. */ -static int mb_ctor_mbuf(void *, int, void *, int); -static int mb_ctor_clust(void *, int, void *, int); -static int mb_ctor_pack(void *, int, void *, int); -static void mb_dtor_mbuf(void *, int, void *); -static void mb_dtor_clust(void *, int, void *); -static void mb_dtor_pack(void *, int, void *); -static int mb_zinit_pack(void *, int, int); -static void mb_zfini_pack(void *, int); - +#ifdef INVARIANTS +static int mb_ctor_pack(void *mem, int size, void *arg, int how); +#endif +static void mb_dtor_pack(void *mem, int size, void *arg); static void mb_reclaim(void *); +static int mb_zinit_pack(void *mem, int size, int how); +static void mb_zfini_pack(void *mem, int size); + static void mbuf_init(void *); static void *mbuf_jumbo_alloc(uma_zone_t, int, u_int8_t *, int); static void mbuf_jumbo_free(void *, int, u_int8_t); @@ -252,73 +250,66 @@ mbuf_init(void *dummy) * Configure UMA zones for Mbufs, Clusters, and Packets. */ zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, - mb_ctor_mbuf, mb_dtor_mbuf, #ifdef INVARIANTS - trash_init, trash_fini, + trash_ctor, trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, NULL, #endif MSIZE - 1, UMA_ZONE_MAXBUCKET); zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, - mb_ctor_clust, mb_dtor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_ctor, trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, UMA_ZONE_MAXBUCKET); if (nmbclusters > 0) uma_zone_set_max(zone_clust, nmbclusters); - zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, + zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, +#ifdef INVARIANTS + mb_ctor_pack, +#else + NULL, +#endif mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); /* Make jumbo frame zone too. Page size, 9k and 16k. */ zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, - mb_ctor_clust, mb_dtor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_ctor, trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); if (nmbjumbop > 0) uma_zone_set_max(zone_jumbop, nmbjumbop); zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, - mb_ctor_clust, mb_dtor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_ctor, trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); if (nmbjumbo9 > 0) uma_zone_set_max(zone_jumbo9, nmbjumbo9); uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc); uma_zone_set_freef(zone_jumbo9, mbuf_jumbo_free); zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, - mb_ctor_clust, mb_dtor_clust, #ifdef INVARIANTS - trash_init, trash_fini, + trash_ctor, trash_dtor, trash_init, trash_fini, #else - NULL, NULL, + NULL, NULL, NULL, NULL, #endif - UMA_ALIGN_PTR, UMA_ZONE_REFCNT); + UMA_ALIGN_PTR, 0); if (nmbjumbo16 > 0) uma_zone_set_max(zone_jumbo16, nmbjumbo16); uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc); uma_zone_set_freef(zone_jumbo16, mbuf_jumbo_free); - zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), - NULL, NULL, - NULL, NULL, - UMA_ALIGN_PTR, UMA_ZONE_ZINIT); - - /* uma_prealloc() goes here... */ - /* * Hook event handler for low-memory situation, used to * drain protocols and push data back to the caches (UMA @@ -373,85 +364,18 @@ mbuf_jumbo_free(void *mem, int size, u_i contigfree(mem, size, M_JUMBOFRAME); } -/* - * Constructor for Mbuf master zone. - * - * The 'arg' pointer points to a mb_args structure which - * contains call-specific information required to support the - * mbuf allocation API. See mbuf.h. - */ +#ifdef INVARIANTS static int -mb_ctor_mbuf(void *mem, int size, void *arg, int how) +mb_ctor_pack(void *mem, int size, void *arg, int how) { struct mbuf *m; - struct mb_args *args; -#ifdef MAC - int error; -#endif - int flags; - short type; -#ifdef INVARIANTS - trash_ctor(mem, size, arg, how); -#endif m = (struct mbuf *)mem; - args = (struct mb_args *)arg; - flags = args->flags; - type = args->type; - - /* - * The mbuf is initialized later. The caller has the - * responsibility to set up any MAC labels too. - */ - if (type == MT_NOINIT) - return (0); + trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_len = 0; - m->m_flags = flags; - m->m_type = type; - if (flags & M_PKTHDR) { - m->m_data = m->m_pktdat; - m->m_pkthdr.rcvif = NULL; - m->m_pkthdr.len = 0; - m->m_pkthdr.header = NULL; - m->m_pkthdr.csum_flags = 0; - m->m_pkthdr.csum_data = 0; - m->m_pkthdr.tso_segsz = 0; - m->m_pkthdr.ether_vtag = 0; - SLIST_INIT(&m->m_pkthdr.tags); -#ifdef MAC - /* If the label init fails, fail the alloc */ - error = mac_init_mbuf(m, how); - if (error) - return (error); -#endif - } else - m->m_data = m->m_dat; return (0); } - -/* - * The Mbuf master zone destructor. - */ -static void -mb_dtor_mbuf(void *mem, int size, void *arg) -{ - struct mbuf *m; - unsigned long flags; - - m = (struct mbuf *)mem; - flags = (unsigned long)arg; - - if ((flags & MB_NOTAGS) == 0 && (m->m_flags & M_PKTHDR) != 0) - m_tag_delete_chain(m, NULL); - KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); - KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__)); -#ifdef INVARIANTS - trash_dtor(mem, size, arg); #endif -} /* * The Mbuf Packet zone destructor. @@ -462,21 +386,21 @@ mb_dtor_pack(void *mem, int size, void * struct mbuf *m; m = (struct mbuf *)mem; - if ((m->m_flags & M_PKTHDR) != 0) - m_tag_delete_chain(m, NULL); - /* Make sure we've got a clean cluster back. */ KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); - KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); + KASSERT(m->m_ext.ext_free == m_ext_free_nop, + ("%s: ext_free != m_ext_free_nop", __func__)); KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__)); KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__)); - KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); - KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); - KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__)); -#ifdef INVARIANTS + KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", + __func__)); + KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", + __func__)); +#ifdef INVARIANTS trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); #endif + /* * If there are processes blocked on zone_clust, waiting for pages to be freed up, * cause them to be woken up by draining the packet zone. We are exposed to a race here @@ -488,85 +412,6 @@ mb_dtor_pack(void *mem, int size, void * } /* - * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor. - * - * Here the 'arg' pointer points to the Mbuf which we - * are configuring cluster storage for. If 'arg' is - * empty we allocate just the cluster without setting - * the mbuf to it. See mbuf.h. - */ -static int -mb_ctor_clust(void *mem, int size, void *arg, int how) -{ - struct mbuf *m; - u_int *refcnt; - int type; - uma_zone_t zone; - -#ifdef INVARIANTS - trash_ctor(mem, size, arg, how); -#endif - switch (size) { - case MCLBYTES: - type = EXT_CLUSTER; - zone = zone_clust; - break; -#if MJUMPAGESIZE != MCLBYTES - case MJUMPAGESIZE: - type = EXT_JUMBOP; - zone = zone_jumbop; - break; -#endif - case MJUM9BYTES: - type = EXT_JUMBO9; - zone = zone_jumbo9; - break; - case MJUM16BYTES: - type = EXT_JUMBO16; - zone = zone_jumbo16; - break; - default: - panic("unknown cluster size"); - break; - } - - m = (struct mbuf *)arg; - refcnt = uma_find_refcnt(zone, mem); - *refcnt = 1; - if (m != NULL) { - m->m_ext.ext_buf = (caddr_t)mem; - m->m_data = m->m_ext.ext_buf; - m->m_flags |= M_EXT; - m->m_ext.ext_free = NULL; - m->m_ext.ext_arg1 = NULL; - m->m_ext.ext_arg2 = NULL; - m->m_ext.ext_size = size; - m->m_ext.ext_type = type; - m->m_ext.ref_cnt = refcnt; - } - - return (0); -} - -/* - * The Mbuf Cluster zone destructor. - */ -static void -mb_dtor_clust(void *mem, int size, void *arg) -{ -#ifdef INVARIANTS - uma_zone_t zone; - - zone = m_getzone(size); - KASSERT(*(uma_find_refcnt(zone, mem)) <= 1, - ("%s: refcnt incorrect %u", __func__, - *(uma_find_refcnt(zone, mem))) ); - - trash_dtor(mem, size, arg); -#endif -} - -/* * The Packet secondary zone's init routine, executed on the * object's transition from mbuf keg slab to zone cache. */ @@ -576,13 +421,16 @@ mb_zinit_pack(void *mem, int size, int h struct mbuf *m; m = (struct mbuf *)mem; /* m is virgin. */ - if (uma_zalloc_arg(zone_clust, m, how) == NULL || - m->m_ext.ext_buf == NULL) + /* + * Allocate and attach the cluster to the ext. + */ + if ((mem = uma_zalloc(zone_clust, how)) == NULL) return (ENOMEM); - m->m_ext.ext_type = EXT_PACKET; /* Override. */ + m_extadd(m, mem, MCLBYTES, m_ext_free_nop, NULL, NULL, 0, EXT_PACKET); #ifdef INVARIANTS - trash_init(m->m_ext.ext_buf, MCLBYTES, how); + return trash_init(m->m_ext.ext_buf, MCLBYTES, how); #endif + return (0); } @@ -605,56 +453,33 @@ mb_zfini_pack(void *mem, int size) #endif } -/* - * The "packet" keg constructor. - */ -static int -mb_ctor_pack(void *mem, int size, void *arg, int how) +int +m_pkthdr_init(struct mbuf *m, int how) { - struct mbuf *m; - struct mb_args *args; #ifdef MAC int error; #endif - int flags; - short type; - - m = (struct mbuf *)mem; - args = (struct mb_args *)arg; - flags = args->flags; - type = args->type; - -#ifdef INVARIANTS - trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); -#endif - m->m_next = NULL; - m->m_nextpkt = NULL; - m->m_data = m->m_ext.ext_buf; - m->m_len = 0; - m->m_flags = (flags | M_EXT); - m->m_type = type; - - if (flags & M_PKTHDR) { - m->m_pkthdr.rcvif = NULL; - m->m_pkthdr.len = 0; - m->m_pkthdr.header = NULL; - m->m_pkthdr.csum_flags = 0; - m->m_pkthdr.csum_data = 0; - m->m_pkthdr.tso_segsz = 0; - m->m_pkthdr.ether_vtag = 0; - SLIST_INIT(&m->m_pkthdr.tags); + m->m_data = m->m_pktdat; + SLIST_INIT(&m->m_pkthdr.tags); + m->m_pkthdr.rcvif = NULL; + m->m_pkthdr.header = NULL; + m->m_pkthdr.len = 0; + m->m_pkthdr.flowid = 0; + m->m_pkthdr.csum_flags = 0; + m->m_pkthdr.csum_data = 0; + m->m_pkthdr.tso_segsz = 0; + m->m_pkthdr.ether_vtag = 0; #ifdef MAC - /* If the label init fails, fail the alloc */ - error = mac_init_mbuf(m, how); - if (error) - return (error); + /* If the label init fails, fail the alloc */ + error = mac_mbuf_init(m, how); + if (error) + return (error); #endif - } - /* m_ext is already initialized. */ return (0); } + /* * This is the protocol drain routine. * @@ -676,3 +501,45 @@ mb_reclaim(void *junk) if (pr->pr_drain != NULL) (*pr->pr_drain)(); } + +struct mbuf * +_m_getjcl(int how, short type, int flags, int size, uma_zone_t zone, + int exttype) +{ + struct mbuf *m; + void *mem; + + if (size == MCLBYTES) + return m_getcl(how, type, flags); + /* + * Allocate the memory and header seperate for these sizes. + */ + mem = uma_zalloc(zone, how); + if (mem == NULL) + return (NULL); + m = m_alloc(zone_mbuf, 0, how, type, flags); + if (m == NULL) { + uma_zfree(zone, mem); + return (NULL); + } + m_extadd(m, mem, size, m_ext_free_zone, zone, mem, flags, exttype); + + return (m); +} + +void * +_m_cljget(struct mbuf *m, int how, int size, uma_zone_t zone, int exttype) +{ + void *mem; + + if (m && m->m_flags & M_EXT) + printf("%s: %p mbuf already has cluster\n", __func__, m); + if (m != NULL) + m->m_ext.ext_buf = NULL; + mem = uma_zalloc(zone, how); + if (mem == NULL) + return (NULL); + if (m) + m_extadd(m, mem, size, m_ext_free_zone, zone, mem, 0, exttype); + return (mem); +} Modified: user/kmacy/releng_7_2_fcs/sys/kern/uipc_mbuf.c ============================================================================== --- user/kmacy/releng_7_2_fcs/sys/kern/uipc_mbuf.c Mon May 18 22:46:59 2009 (r192353) +++ user/kmacy/releng_7_2_fcs/sys/kern/uipc_mbuf.c Mon May 18 22:54:47 2009 (r192354) @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> +#include <sys/ktr.h> #include <sys/limits.h> #include <sys/lock.h> #include <sys/malloc.h> @@ -86,6 +87,8 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, m_defrag &m_defragrandomfailures, 0, ""); #endif +static void m_refm(struct mbuf *mb, struct mbuf *m); + /* * Allocate a given length worth of mbufs and/or clusters (whatever fits * best) and return a pointer to the top of the allocated chain. If an @@ -127,8 +130,7 @@ m_getm2(struct mbuf *m, int len, int how } /* Book keeping. */ - len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size : - ((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN); + len -= mb->m_size; if (mtail != NULL) mtail->m_next = mb; else @@ -163,147 +165,63 @@ m_freem(struct mbuf *mb) mb = m_free(mb); } -/*- - * Configure a provided mbuf to refer to the provided external storage - * buffer and setup a reference count for said buffer. If the setting - * up of the reference count fails, the M_EXT bit will not be set. If - * successfull, the M_EXT bit is set in the mbuf's flags. - * - * Arguments: - * mb The existing mbuf to which to attach the provided buffer. - * buf The address of the provided external storage buffer. - * size The size of the provided buffer. - * freef A pointer to a routine that is responsible for freeing the - * provided external storage buffer. - * args A pointer to an argument structure (of any type) to be passed - * to the provided freef routine (may be NULL). - * flags Any other flags to be passed to the provided mbuf. - * type The type that the external storage buffer should be - * labeled with. - * - * Returns: - * Nothing. +/* + * Reference the existing storage area of an mbuf. The reference is readonly + * and the referenced data can not be freed until the referencing mbuf is + * freed. */ -void -m_extadd(struct mbuf *mb, caddr_t buf, u_int size, - void (*freef)(void *, void *), void *arg1, void *arg2, int flags, int type) +static void +m_refm(struct mbuf *mb, struct mbuf *m) { - KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); - if (type != EXT_EXTREF) - mb->m_ext.ref_cnt = (u_int *)uma_zalloc(zone_ext_refcnt, M_NOWAIT); - if (mb->m_ext.ref_cnt != NULL) { - *(mb->m_ext.ref_cnt) = 1; - mb->m_flags |= (M_EXT | flags); - mb->m_ext.ext_buf = buf; - mb->m_data = mb->m_ext.ext_buf; - mb->m_ext.ext_size = size; - mb->m_ext.ext_free = freef; - mb->m_ext.ext_arg1 = arg1; - mb->m_ext.ext_arg2 = arg2; - mb->m_ext.ext_type = type; - } + if (m->m_ref > 1) + atomic_add_int(&m->m_ref, 1); + else + m->m_ref++; + mb->m_flags |= M_EXT | M_RDONLY; + mb->m_data = m->m_data; + mb->m_size = m->m_size; + mb->m_ext.ext_buf = M_START(m); + mb->m_ext.ext_size = m->m_size; + mb->m_ext.ext_free = m_ext_free_mbuf; + mb->m_ext.ext_arg1 = m; + mb->m_ext.ext_arg2 = NULL; + mb->m_ext.ext_type = EXT_MBUF; + + CTR3(KTR_NET, "m_refm: %p ref %d buf %p", + mb, mb->m_ref, mb->m_ext.ext_buf); } /* - * Non-directly-exported function to clean up after mbufs with M_EXT - * storage attached to them if the reference count hits 1. + * Free the ext area of a mbuf assuming a uma zone and argument are + * presented. */ void -mb_free_ext(struct mbuf *m) +m_ext_free_zone(void *arg1, void *arg2) { - int skipmbuf; - - KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); - KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); + uma_zfree(arg1, arg2); +} + +/* + * Free the ext area of a mbuf assuming it has been acquired with m_refm(). + */ +void +m_ext_free_mbuf(void *arg1, void *arg2) +{ /* - * check if the header is embedded in the cluster - */ - skipmbuf = (m->m_flags & M_NOFREE); - - /* Free attached storage if this mbuf is the only reference to it. */ - if (*(m->m_ext.ref_cnt) == 1 || - atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) { - switch (m->m_ext.ext_type) { - case EXT_PACKET: /* The packet zone is special. */ - if (*(m->m_ext.ref_cnt) == 0) - *(m->m_ext.ref_cnt) = 1; - uma_zfree(zone_pack, m); - return; /* Job done. */ - case EXT_CLUSTER: - uma_zfree(zone_clust, m->m_ext.ext_buf); - break; - case EXT_JUMBOP: - uma_zfree(zone_jumbop, m->m_ext.ext_buf); - break; - case EXT_JUMBO9: - uma_zfree(zone_jumbo9, m->m_ext.ext_buf); - break; - case EXT_JUMBO16: - uma_zfree(zone_jumbo16, m->m_ext.ext_buf); - break; - case EXT_SFBUF: - case EXT_NET_DRV: - case EXT_MOD_TYPE: - case EXT_DISPOSABLE: - *(m->m_ext.ref_cnt) = 0; - uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, - m->m_ext.ref_cnt)); - /* FALLTHROUGH */ - case EXT_EXTREF: - KASSERT(m->m_ext.ext_free != NULL, - ("%s: ext_free not set", __func__)); - (*(m->m_ext.ext_free))(m->m_ext.ext_arg1, - m->m_ext.ext_arg2); - break; - default: - KASSERT(m->m_ext.ext_type == 0, - ("%s: unknown ext_type", __func__)); - } - } - if (skipmbuf) - return; - - /* - * Free this mbuf back to the mbuf zone with all m_ext - * information purged. + * Release one more reference to this mbuf. If it is the last it + * will be freed. */ - m->m_ext.ext_buf = NULL; - m->m_ext.ext_free = NULL; - m->m_ext.ext_arg1 = NULL; - m->m_ext.ext_arg2 = NULL; - m->m_ext.ref_cnt = NULL; - m->m_ext.ext_size = 0; - m->m_ext.ext_type = 0; - m->m_flags &= ~M_EXT; - uma_zfree(zone_mbuf, m); + m_free(arg1); } -/* - * Attach the the cluster from *m to *n, set up m_ext in *n - * and bump the refcount of the cluster. - */ -static void -mb_dupcl(struct mbuf *n, struct mbuf *m) +void +m_ext_free_nop(void *arg1, void *arg2) { - KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); - KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__)); - KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); - if (*(m->m_ext.ref_cnt) == 1) - *(m->m_ext.ref_cnt) += 1; - else - atomic_add_int(m->m_ext.ref_cnt, 1); - n->m_ext.ext_buf = m->m_ext.ext_buf; - n->m_ext.ext_free = m->m_ext.ext_free; - n->m_ext.ext_arg1 = m->m_ext.ext_arg1; - n->m_ext.ext_arg2 = m->m_ext.ext_arg2; - n->m_ext.ext_size = m->m_ext.ext_size; - n->m_ext.ref_cnt = m->m_ext.ref_cnt; - n->m_ext.ext_type = m->m_ext.ext_type; - n->m_flags |= M_EXT; + /* Nothing to do. */ } /* @@ -356,11 +274,8 @@ m_sanity(struct mbuf *m0, int sanitize) * unrelated kernel memory before or after us is trashed. * No way to recover from that. */ - a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : - ((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) : - (caddr_t)(&m->m_dat)) ); - b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size : - ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN))); + a = M_START(m); + b = (caddr_t)(a + m->m_size); if ((caddr_t)m->m_data < a) M_SANITY_ACTION("m_data outside mbuf data range left"); if ((caddr_t)m->m_data > b) @@ -531,6 +446,7 @@ m_copym(struct mbuf *m, int off0, int le struct mbuf *top; int copyhdr = 0; + CTR3(KTR_NET, "m_copym(%p, %d, %d)", m, off0, len); KASSERT(off >= 0, ("m_copym, negative off %d", off)); KASSERT(len >= 0, ("m_copym, negative len %d", len)); MBUF_CHECKSLEEP(wait); @@ -567,13 +483,16 @@ m_copym(struct mbuf *m, int off0, int le n->m_pkthdr.len = len; copyhdr = 0; } + /* + * If the copied data will fit in the space of standard + * mbuf prefer to copy rather than reference. + */ n->m_len = min(len, m->m_len - off); - if (m->m_flags & M_EXT) { - n->m_data = m->m_data + off; - mb_dupcl(n, m); + if (n->m_len > n->m_size) { + m_refm(n, m); + n->m_data += off; } else - bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), - (u_int)n->m_len); + bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), n->m_len); if (len != M_COPYALL) len -= n->m_len; off = 0; @@ -751,7 +670,9 @@ struct mbuf * m_copypacket(struct mbuf *m, int how) { struct mbuf *top, *n, *o; + int leading; + CTR1(KTR_NET, "m_copypacket(%p)", m); MBUF_CHECKSLEEP(how); MGET(n, how, m->m_type); top = n; @@ -761,13 +682,10 @@ m_copypacket(struct mbuf *m, int how) if (!m_dup_pkthdr(n, m, how)) goto nospace; n->m_len = m->m_len; - if (m->m_flags & M_EXT) { - n->m_data = m->m_data; - mb_dupcl(n, m); - } else { - n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); - bcopy(mtod(m, char *), mtod(n, char *), n->m_len); - } + if (n->m_len > n->m_size) + m_refm(n, m); + else + bcopy(mtod(m, caddr_t), mtod(n, caddr_t), m->m_len); m = m->m_next; while (m) { @@ -779,13 +697,13 @@ m_copypacket(struct mbuf *m, int how) n = n->m_next; n->m_len = m->m_len; - if (m->m_flags & M_EXT) { - n->m_data = m->m_data; - mb_dupcl(n, m); + leading = M_LEADINGSPACE(m); + if (n->m_len + leading > n->m_size) { + m_refm(n, m); } else { - bcopy(mtod(m, char *), mtod(n, char *), n->m_len); + n->m_data = M_START(n) + leading; + bcopy(mtod(m, caddr_t), mtod(n, caddr_t), n->m_len); } - m = m->m_next; } return top; @@ -804,6 +722,7 @@ m_copydata(const struct mbuf *m, int off { u_int count; + CTR3(KTR_NET, "m_copydata(%p, %d, %d)", m, off, len); KASSERT(off >= 0, ("m_copydata, negative off %d", off)); KASSERT(len >= 0, ("m_copydata, negative len %d", len)); while (off > 0) { @@ -833,8 +752,9 @@ struct mbuf * m_dup(struct mbuf *m, int how) { struct mbuf **p, *top = NULL; - int remain, moff, nsize; + int remain, moff; + CTR1(KTR_NET, "m_dup(%p)", m); MBUF_CHECKSLEEP(how); /* Sanity check */ if (m == NULL) @@ -849,13 +769,10 @@ m_dup(struct mbuf *m, int how) struct mbuf *n; /* Get the next new mbuf */ - if (remain >= MINCLSIZE) { + if (remain >= MINCLSIZE) n = m_getcl(how, m->m_type, 0); - nsize = MCLBYTES; - } else { + else n = m_get(how, m->m_type); - nsize = MLEN; - } if (n == NULL) goto nospace; @@ -864,8 +781,6 @@ m_dup(struct mbuf *m, int how) m_free(n); goto nospace; } - if ((n->m_flags & M_EXT) == 0) *** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200905182254.n4IMslGT077662>