Skip site navigation (1)Skip section navigation (2)
Date:      Sun, 02 Aug 2020 13:48:11 -0400
From:      Ernie Luzar <luzar722@gmail.com>
To:        "freebsd-questions@freebsd.org" <freebsd-questions@freebsd.org>,  freebsd-jail@FreeBSD.org
Subject:   jail(8) bug with vnet & non-vnet jails running at same time?
Message-ID:  <5F26FC5B.6030706@gmail.com>

next in thread | raw e-mail | index | archive | help
Hello list;
Please review configuration looking for something I may have missed. 
Hopping someone can suggest something that will change the behavior 
eliminating the problem.


Equipment. Real hardware, 12.1 release, amd64 dual cpu.

Description;
non-vnet jails and vnet jails using the bridge/epair method can ping the 
public internet when only non-vnet jails are started at a time or when 
only vnet jails are started at a time. But when both non-vnet jails and 
vnet jails are started together then neither one can ping the public 
internet. The order of the jails definitions in the jail.conf file has 
no effect on changing what is happening.

Bug description:
When non-vnet jails are started their ip addresses are added to the NIC 
facing the public AFTER the public ip address and the non-vnet jail has 
access to the public internet. But when both non-vnet jails and vnet 
jails are started at the same time then the non-vnet jails ip addresses 
gets added before the public ip address of the NIC facing the public 
internet causing the host to lose all access to the public internet. 
This seems to be a jail(8) bug.

It makes no difference which command method is used to start and stop 
the jails.
Service jail onestart jailname   or  jail –cv jailname

The following is a capture of the command sequence showing this bug. 
Follow the re0 NIC public ip address xx.25.51.0 in the ifconfig -a listing.


Before any jails are started.
/root >ifconfig -a
snip ...
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	media: Ethernet autoselect (1000baseT <full-duplex,master>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>


/root >cat /etc/jail.conf

#  non-vnet jail
zdir20 {
host.hostname       =  "zdir20";
path                =  "/usr/jails/zdir20";
mount.fstab         =  "/usr/local/etc/fstab/zdir20";
exec.consolelog     =  "/var/log/zdir20.console.log";
mount.devfs;
ip4.addr            =  10.0.22.5;
interface           =  "re0";
allow.raw_sockets;
devfs_ruleset       =  "4";
exec.start          =  "/bin/sh /etc/rc";
exec.stop           =  "/bin/sh /etc/rc.shutdown";
}

#  vnet jail using the bridge/epair method
v0jail1 {
host.hostname   = "v0jail1";
path            = "/usr/jails/v0jail1";
mount.fstab     = "/usr/local/etc/fstab/v0jail1";
exec.consolelog = "/var/log/v0jail1.console.log";
mount.devfs;
devfs_ruleset   = "4";
vnet            = "new";
vnet.interface  = "epair55b";
exec.prestart   = "ifconfig epair55  create up";
exec.prestart  += "ifconfig bridge0 addm epair55a";
exec.prestart  += "ifconfig epair55a descr vnet-v0jail1";
exec.prestart  += "ifconfig bridge0 inet 10.0.48.2 netmask 255.255.255.0 
alias";
exec.start      = "/bin/sh /etc/rc";
exec.start     += "ifconfig epair55b inet 10.0.48.1 netmask 255.255.255.0";
exec.start     += "route add default 10.0.48.2";
exec.prestop    = "ifconfig epair55b -vnet v0jail1";
exec.stop       = "/bin/sh /etc/rc.shutdown";
exec.poststop   = "ifconfig bridge0 deletem epair55a";
exec.poststop  += "sleep 2";
exec.poststop  += "ifconfig epair55a destroy";
exec.poststop  += "ifconfig bridge0 inet 10.0.48.2 -alias";
}


/root >jls
    JID  IP Address      Hostname                      Path

# start only the non-vnet jail
/root >service jail onestart zdir20
Starting jails: zdir20.

/root >jls
    JID  IP Address      Hostname                      Path
     18  10.0.22.5       zdir20                        /usr/jails/zdir20

# Take notice that the non-vnet jails ip address follows the nic’s
# public ip address.
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	inet 10.0.22.5 netmask 0xffffffff broadcast 10.0.22.5
	media: Ethernet autoselect (1000baseT <full-duplex,master>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>

# login to the non-vnet jail and ping the public
/root >jexec zdir20 login -f root
Last login: Sun Aug  2 11:30:40 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
zdir20 /root >
zdir20 /root >ping -c 2 freebsd.org
PING freebsd.org (96.47.72.84): 56 data bytes
64 bytes from 96.47.72.84: icmp_seq=0 ttl=48 time=44.426 ms
64 bytes from 96.47.72.84: icmp_seq=1 ttl=48 time=44.481 ms

--- freebsd.org ping statistics ---
2 packets transmitted, 2 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 44.426/44.453/44.481/0.027 ms
zdir20 /root >exit
logout

# stop the non-vnet jail and show that the network is back to
# starting condition.
/root >service jail onestop zdir20
Stopping jails: zdir20.

/root >jls
    JID  IP Address      Hostname                      Path

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	media: Ethernet autoselect (1000baseT <full-duplex,master>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>

# start only the vnet jail and see the bridge0
/root >service jail onestart v0jail1
Starting jails: v0jail1.
/root >jls
    JID  IP Address      Hostname                      Path
     19                  v0jail1                       /usr/jails/v0jail1

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=82099<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	media: Ethernet autoselect (1000baseT <full-duplex>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	inet 10.0.48.2 netmask 0xffffff00 broadcast 10.0.48.255
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: epair55a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 5 priority 128 path cost 2000
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>
epair55a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> 
metric 0 mtu 1500
	description: vnet-v0jail1
	options=8<VLAN_MTU>
	ether 02:eb:be:f5:15:0a
	inet6 fe80::eb:beff:fef5:150a%epair55a prefixlen 64 scopeid 0x5
	groups: epair
	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
	status: active
	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>

# login to the vnet jail and ping the public internet.
/root >jexec v0jail1 login -f root
Last login: Sun Aug  2 11:29:41 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
v0jail1 /root >ping -c 2 freebsd.org
PING freebsd.org (96.47.72.84): 56 data bytes
64 bytes from 96.47.72.84: icmp_seq=0 ttl=47 time=46.745 ms
64 bytes from 96.47.72.84: icmp_seq=1 ttl=47 time=43.930 ms

--- freebsd.org ping statistics ---
2 packets transmitted, 2 packets received, 0.0% packet loss
round-trip min/avg/max/stddev = 43.930/45.337/46.745/1.407 ms
v0jail1 /root >exit
logout


# close the vnet jail and return to starting condition.
/root >service jail onestop v0jail1
Stopping jails: v0jail1.


/root >jls
    JID  IP Address      Hostname                      Path

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	media: Ethernet autoselect (1000baseT <full-duplex>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>

# Start both the non-vnet jail and the vnet jail together.
/root >service jail onestart
Starting jails: zdir20 v0jail1.

# login to the non-vnet jail and it has no public access.
/root >jexec zdir20 login -f root [K
Last login: Sun Aug  2 11:36:34 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
zdir20 /root >ping -c 2 freebsd.org
ping: cannot resolve freebsd.org: Host name lookup failure
zdir20 /root >exit
logout


# login to the vnet jail and it has no public access.
/root >jexec v0jail1 login -f root
Last login: Sun Aug  2 11:38:56 on pts/0
FreeBSD 12.1-RELEASE-p6 GENERIC

Welcome to your FreeBSD jail.
v0jail1 /root >ping -c 2 freebsd.org
ping: cannot resolve freebsd.org: Host name lookup failure
v0jail1 /root >exit
logout
/root >jls
    JID  IP Address      Hostname                      Path
     20  10.0.22.5       zdir20                        /usr/jails/zdir20
     21                  v0jail1                       /usr/jails/v0jail1

# Here is the bug. See that the non-vnet jail ip address comes before the
# public address causing the host to lose access to the public internet.
/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=82099<RXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet 10.0.22.5 netmask 0xffffffff broadcast 10.0.22.5
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	media: Ethernet autoselect (1000baseT <full-duplex>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	inet 10.0.48.2 netmask 0xffffff00 broadcast 10.0.48.255
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: epair55a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 5 priority 128 path cost 2000
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>
epair55a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> 
metric 0 mtu 1500
	description: vnet-v0jail1
	options=8<VLAN_MTU>
	ether 02:77:b8:5f:e4:0a
	inet6 fe80::77:b8ff:fe5f:e40a%epair55a prefixlen 64 scopeid 0x5
	groups: epair
	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
	status: active
	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>

# stop both jails and return to starting condition.
/root >service jail onestop
Stopping jails: zdir20 v0jail1.

/root >jls
    JID  IP Address      Hostname                      Path

/root >ifconfig -a
re0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0 
mtu 1500
	options=8209b<RXCSUM,TXCSUM,VLAN_MTU,VLAN_HWTAGGING,VLAN_HWCSUM,WOL_MAGIC,LINKSTATE>
	ether 50:3e:aa:06:11:22
	inet xx.25.51.0 netmask 0xfffff000 broadcast 255.255.255.255
	media: Ethernet autoselect (1000baseT <full-duplex,master>)
	status: active
	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
bridge0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu 
1500
	ether 02:3e:ba:a7:58:00
	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
	member: re0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
	        ifmaxaddr 0 port 2 priority 128 path cost 20000
	groups: bridge
	nd6 options=1<PERFORMNUD>





Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?5F26FC5B.6030706>