Date: Fri, 14 Oct 2016 08:14:51 +0000 (UTC) From: Sepherosa Ziehau <sephe@FreeBSD.org> To: src-committers@freebsd.org, svn-src-all@freebsd.org, svn-src-stable@freebsd.org, svn-src-stable-11@freebsd.org Subject: svn commit: r307303 - in stable/11/sys/dev/hyperv: include netvsc vmbus Message-ID: <201610140814.u9E8EphR086005@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: sephe Date: Fri Oct 14 08:14:51 2016 New Revision: 307303 URL: https://svnweb.freebsd.org/changeset/base/307303 Log: MFC 302632-302634 302632 hyperv/vmbus: More verbose for GPADL_connect/chan_{rescind,offer} Reviewed by: Dexuan Cui <decui microsoft com>, Hongjiang Zhang <honzhan microsoft com> Sponsored by: Microsoft OSTC Differential Revision: https://reviews.freebsd.org/D6976 302633 hyperv/vmbus: Free sysctl properly upon channel close. Prepare for sub-channel re-open. Sponsored by: Microsoft OSTC Differential Revision: https://reviews.freebsd.org/D6977 302634 hyperv/vmbus: Fix sub-channel re-open support. For multi-channel devices, once the primary channel is closed, a set of 'rescind' messages for sub-channels will be delivered by Hypervisor. Sub-channel MUST be freed according to these 'rescind' messages; directly re-openning sub-channels in the same fashion as the primary channel's re-opening does NOT work at all. After the primary channel is re-opened, requested # of sub- channels will be delivered though 'channel offer' messages, and this set of newly offered channels can be opened along side with the primary channel. This unbreaks the MTU setting for hn(4), which requires re- openning all existsing channels upon MTU change. Sponsored by: Microsoft OSTC Differential Revision: https://reviews.freebsd.org/D6978 Modified: stable/11/sys/dev/hyperv/include/hyperv.h stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c stable/11/sys/dev/hyperv/vmbus/hv_channel.c stable/11/sys/dev/hyperv/vmbus/hv_channel_mgmt.c stable/11/sys/dev/hyperv/vmbus/vmbus_reg.h Directory Properties: stable/11/ (props changed) Modified: stable/11/sys/dev/hyperv/include/hyperv.h ============================================================================== --- stable/11/sys/dev/hyperv/include/hyperv.h Fri Oct 14 08:02:37 2016 (r307302) +++ stable/11/sys/dev/hyperv/include/hyperv.h Fri Oct 14 08:14:51 2016 (r307303) @@ -49,6 +49,7 @@ #include <sys/smp.h> #include <sys/mutex.h> #include <sys/bus.h> +#include <sys/sysctl.h> #include <vm/vm.h> #include <vm/vm_param.h> #include <vm/pmap.h> @@ -627,6 +628,8 @@ typedef struct hv_vmbus_channel { struct task ch_detach_task; TAILQ_ENTRY(hv_vmbus_channel) ch_link; + + struct sysctl_ctx_list ch_sysctl_ctx; } hv_vmbus_channel; #define HV_VMBUS_CHAN_ISPRIMARY(chan) ((chan)->primary_channel == NULL) @@ -714,6 +717,7 @@ void vmbus_channel_cpu_rr(struct hv_vmb struct hv_vmbus_channel ** vmbus_get_subchan(struct hv_vmbus_channel *pri_chan, int subchan_cnt); void vmbus_rel_subchan(struct hv_vmbus_channel **subchan, int subchan_cnt); +void vmbus_drain_subchan(struct hv_vmbus_channel *pri_chan); /** * @brief Get physical address from virtual Modified: stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c ============================================================================== --- stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c Fri Oct 14 08:02:37 2016 (r307302) +++ stable/11/sys/dev/hyperv/netvsc/hv_netvsc_drv_freebsd.c Fri Oct 14 08:14:51 2016 (r307303) @@ -345,6 +345,7 @@ static void hn_destroy_rx_data(struct hn static void hn_set_tx_chimney_size(struct hn_softc *, int); static void hn_channel_attach(struct hn_softc *, struct hv_vmbus_channel *); static void hn_subchan_attach(struct hn_softc *, struct hv_vmbus_channel *); +static void hn_subchan_setup(struct hn_softc *); static int hn_transmit(struct ifnet *, struct mbuf *); static void hn_xmit_qflush(struct ifnet *); @@ -575,25 +576,8 @@ netvsc_attach(device_t dev) device_printf(dev, "%d TX ring, %d RX ring\n", sc->hn_tx_ring_inuse, sc->hn_rx_ring_inuse); - if (sc->net_dev->num_channel > 1) { - struct hv_vmbus_channel **subchan; - int subchan_cnt = sc->net_dev->num_channel - 1; - int i; - - /* Wait for sub-channels setup to complete. */ - subchan = vmbus_get_subchan(pri_chan, subchan_cnt); - - /* Attach the sub-channels. */ - for (i = 0; i < subchan_cnt; ++i) { - /* NOTE: Calling order is critical. */ - hn_subchan_attach(sc, subchan[i]); - hv_nv_subchan_attach(subchan[i]); - } - - /* Release the sub-channels */ - vmbus_rel_subchan(subchan, subchan_cnt); - device_printf(dev, "%d sub-channels setup done\n", subchan_cnt); - } + if (sc->net_dev->num_channel > 1) + hn_subchan_setup(sc); #if __FreeBSD_version >= 1100099 if (sc->hn_rx_ring_inuse > 1) { @@ -1620,6 +1604,10 @@ hn_ioctl(struct ifnet *ifp, u_long cmd, NV_UNLOCK(sc); break; } + + /* Wait for subchannels to be destroyed */ + vmbus_drain_subchan(hn_dev->channel); + error = hv_rf_on_device_add(hn_dev, &device_info, sc->hn_rx_ring_inuse); if (error) { @@ -1628,6 +1616,26 @@ hn_ioctl(struct ifnet *ifp, u_long cmd, NV_UNLOCK(sc); break; } + KASSERT(sc->hn_rx_ring_cnt == sc->net_dev->num_channel, + ("RX ring count %d and channel count %u mismatch", + sc->hn_rx_ring_cnt, sc->net_dev->num_channel)); + if (sc->net_dev->num_channel > 1) { + int r; + + /* + * Skip the rings on primary channel; they are + * handled by the hv_rf_on_device_add() above. + */ + for (r = 1; r < sc->hn_rx_ring_cnt; ++r) { + sc->hn_rx_ring[r].hn_rx_flags &= + ~HN_RX_FLAG_ATTACHED; + } + for (r = 1; r < sc->hn_tx_ring_cnt; ++r) { + sc->hn_tx_ring[r].hn_tx_flags &= + ~HN_TX_FLAG_ATTACHED; + } + hn_subchan_setup(sc); + } sc->hn_tx_chimney_max = sc->net_dev->send_section_size; if (sc->hn_tx_ring[0].hn_tx_chimney_size > @@ -2980,6 +2988,29 @@ hn_subchan_attach(struct hn_softc *sc, s } static void +hn_subchan_setup(struct hn_softc *sc) +{ + struct hv_device *device_ctx = vmbus_get_devctx(sc->hn_dev); + struct hv_vmbus_channel **subchan; + int subchan_cnt = sc->net_dev->num_channel - 1; + int i; + + /* Wait for sub-channels setup to complete. */ + subchan = vmbus_get_subchan(device_ctx->channel, subchan_cnt); + + /* Attach the sub-channels. */ + for (i = 0; i < subchan_cnt; ++i) { + /* NOTE: Calling order is critical. */ + hn_subchan_attach(sc, subchan[i]); + hv_nv_subchan_attach(subchan[i]); + } + + /* Release the sub-channels */ + vmbus_rel_subchan(subchan, subchan_cnt); + if_printf(sc->hn_ifp, "%d sub-channels setup done\n", subchan_cnt); +} + +static void hn_tx_taskq_create(void *arg __unused) { if (!hn_share_tx_taskq) Modified: stable/11/sys/dev/hyperv/vmbus/hv_channel.c ============================================================================== --- stable/11/sys/dev/hyperv/vmbus/hv_channel.c Fri Oct 14 08:02:37 2016 (r307302) +++ stable/11/sys/dev/hyperv/vmbus/hv_channel.c Fri Oct 14 08:14:51 2016 (r307303) @@ -111,7 +111,8 @@ vmbus_channel_sysctl_create(hv_vmbus_cha ch_id = primary_ch->offer_msg.child_rel_id; sub_ch_id = channel->offer_msg.offer.sub_channel_index; } - ctx = device_get_sysctl_ctx(dev); + ctx = &channel->ch_sysctl_ctx; + sysctl_ctx_init(ctx); /* This creates dev.DEVNAME.DEVUNIT.channel tree */ devch_sysctl = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), @@ -423,6 +424,11 @@ hv_vmbus_channel_establish_gpadl(struct device_printf(sc->vmbus_dev, "gpadl->chan%u failed: " "status %u\n", channel->offer_msg.child_rel_id, status); return EIO; + } else { + if (bootverbose) { + device_printf(sc->vmbus_dev, "gpadl->chan%u " + "succeeded\n", channel->offer_msg.child_rel_id); + } } return 0; } @@ -477,6 +483,7 @@ hv_vmbus_channel_close_internal(hv_vmbus int error; channel->state = HV_CHANNEL_OPEN_STATE; + sysctl_ctx_free(&channel->ch_sysctl_ctx); /* * set rxq to NULL to avoid more requests be scheduled Modified: stable/11/sys/dev/hyperv/vmbus/hv_channel_mgmt.c ============================================================================== --- stable/11/sys/dev/hyperv/vmbus/hv_channel_mgmt.c Fri Oct 14 08:02:37 2016 (r307302) +++ stable/11/sys/dev/hyperv/vmbus/hv_channel_mgmt.c Fri Oct 14 08:14:51 2016 (r307303) @@ -142,6 +142,19 @@ vmbus_channel_process_offer(hv_vmbus_cha } mtx_unlock(&sc->vmbus_chlist_lock); + if (bootverbose) { + char logstr[64]; + + logstr[0] = '\0'; + if (channel != NULL) { + snprintf(logstr, sizeof(logstr), ", primary chan%u", + channel->offer_msg.child_rel_id); + } + device_printf(sc->vmbus_dev, "chan%u subchanid%u offer%s\n", + new_channel->offer_msg.child_rel_id, + new_channel->offer_msg.offer.sub_channel_index, logstr); + } + if (channel != NULL) { /* * Check if this is a sub channel. @@ -157,13 +170,6 @@ vmbus_channel_process_offer(hv_vmbus_cha new_channel, sc_list_entry); mtx_unlock(&channel->sc_lock); - if (bootverbose) { - printf("VMBUS get multi-channel offer, " - "rel=%u, sub=%u\n", - new_channel->offer_msg.child_rel_id, - new_channel->offer_msg.offer.sub_channel_index); - } - /* * Insert the new channel to the end of the global * channel list. @@ -178,11 +184,6 @@ vmbus_channel_process_offer(hv_vmbus_cha ch_link); mtx_unlock(&sc->vmbus_chlist_lock); - if(bootverbose) - printf("VMBUS: new multi-channel offer <%p>, " - "its primary channel is <%p>.\n", - new_channel, new_channel->primary_channel); - new_channel->state = HV_CHANNEL_OPEN_STATE; /* @@ -345,6 +346,10 @@ vmbus_channel_on_offer_rescind(struct vm hv_vmbus_channel* channel; rescind = (const hv_vmbus_channel_rescind_offer *)msg->msg_data; + if (bootverbose) { + device_printf(sc->vmbus_dev, "chan%u rescind\n", + rescind->child_rel_id); + } channel = hv_vmbus_g_connection.channels[rescind->child_rel_id]; if (channel == NULL) @@ -362,6 +367,54 @@ vmbus_chan_detach_task(void *xchan, int if (HV_VMBUS_CHAN_ISPRIMARY(chan)) { /* Only primary channel owns the hv_device */ hv_vmbus_child_device_unregister(chan->device); + /* NOTE: DO NOT free primary channel for now */ + } else { + struct vmbus_softc *sc = chan->vmbus_sc; + struct hv_vmbus_channel *pri_chan = chan->primary_channel; + struct vmbus_chanmsg_chfree *req; + struct vmbus_msghc *mh; + int error; + + mh = vmbus_msghc_get(sc, sizeof(*req)); + if (mh == NULL) { + device_printf(sc->vmbus_dev, + "can not get msg hypercall for chfree(chan%u)\n", + chan->offer_msg.child_rel_id); + goto remove; + } + + req = vmbus_msghc_dataptr(mh); + req->chm_hdr.chm_type = VMBUS_CHANMSG_TYPE_CHFREE; + req->chm_chanid = chan->offer_msg.child_rel_id; + + error = vmbus_msghc_exec_noresult(mh); + vmbus_msghc_put(sc, mh); + + if (error) { + device_printf(sc->vmbus_dev, + "chfree(chan%u) failed: %d", + chan->offer_msg.child_rel_id, error); + /* NOTE: Move on! */ + } else { + if (bootverbose) { + device_printf(sc->vmbus_dev, "chan%u freed\n", + chan->offer_msg.child_rel_id); + } + } +remove: + mtx_lock(&sc->vmbus_chlist_lock); + TAILQ_REMOVE(&sc->vmbus_chlist, chan, ch_link); + mtx_unlock(&sc->vmbus_chlist_lock); + + mtx_lock(&pri_chan->sc_lock); + TAILQ_REMOVE(&pri_chan->sc_list_anchor, chan, sc_list_entry); + KASSERT(pri_chan->subchan_cnt > 0, + ("invalid subchan_cnt %d", pri_chan->subchan_cnt)); + pri_chan->subchan_cnt--; + mtx_unlock(&pri_chan->sc_lock); + wakeup(pri_chan); + + hv_vmbus_free_vmbus_channel(chan); } } @@ -499,6 +552,15 @@ vmbus_rel_subchan(struct hv_vmbus_channe } void +vmbus_drain_subchan(struct hv_vmbus_channel *pri_chan) +{ + mtx_lock(&pri_chan->sc_lock); + while (pri_chan->subchan_cnt > 0) + mtx_sleep(pri_chan, &pri_chan->sc_lock, 0, "dsubch", 0); + mtx_unlock(&pri_chan->sc_lock); +} + +void vmbus_chan_msgproc(struct vmbus_softc *sc, const struct vmbus_message *msg) { vmbus_chanmsg_proc_t msg_proc; Modified: stable/11/sys/dev/hyperv/vmbus/vmbus_reg.h ============================================================================== --- stable/11/sys/dev/hyperv/vmbus/vmbus_reg.h Fri Oct 14 08:02:37 2016 (r307302) +++ stable/11/sys/dev/hyperv/vmbus/vmbus_reg.h Fri Oct 14 08:14:51 2016 (r307303) @@ -102,6 +102,7 @@ struct vmbus_gpa_range { #define VMBUS_CHANMSG_TYPE_GPADL_CONNRESP 10 /* RESP */ #define VMBUS_CHANMSG_TYPE_GPADL_DISCONN 11 /* REQ */ #define VMBUS_CHANMSG_TYPE_GPADL_DISCONNRESP 12 /* RESP */ +#define VMBUS_CHANMSG_TYPE_CHFREE 13 /* REQ */ #define VMBUS_CHANMSG_TYPE_CONNECT 14 /* REQ */ #define VMBUS_CHANMSG_TYPE_CONNECT_RESP 15 /* RESP */ #define VMBUS_CHANMSG_TYPE_DISCONNECT 16 /* REQ */ @@ -206,4 +207,10 @@ struct vmbus_chanmsg_gpadl_disconn { uint32_t chm_gpadl; } __packed; +/* VMBUS_CHANMSG_TYPE_CHFREE */ +struct vmbus_chanmsg_chfree { + struct vmbus_chanmsg_hdr chm_hdr; + uint32_t chm_chanid; +} __packed; + #endif /* !_VMBUS_REG_H_ */
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201610140814.u9E8EphR086005>