Date: Wed, 24 Apr 2013 07:37:25 +0200 From: Anders Hagman <anders.hagman@netplex.se> To: "freebsd-jail@freebsd.org" <freebsd-jail@freebsd.org> Subject: Re: jail(8) vimage epair bridge Message-ID: <77E31AD0-ABE2-44FA-AB19-CF557038DEBE@netplex.se> In-Reply-To: <5176892F.8050802@a1poweruser.com> References: <5176892F.8050802@a1poweruser.com>
next in thread | previous in thread | raw e-mail | index | archive | help
Hi
23 apr 2013 kl. 15:14 skrev Joe <fbsd8@a1poweruser.com>:
> Hello list
>=20
> I am using jail(8) trying to get a functional vimage environment on my
> 9.1-RELEASE system. My PC only has a single real NIC facing the public
> internet.
>=20
> My goal is to be able to have multiple vimage jails, each with
> their own epairXa epairXb and bridgeX where the "X" is the jails JID
> number all having their traffic passing through the single rl0 real
> interface. The vnet.start script shown below handles this nicely.
>=20
> The problem is after the first vimage jail is started the rl0 interface
> gets marked as busy when the second vimage jail is started.
>=20
You don't need more the one bridge.
Connect all epairXa and the rl0 interface to the bridge. Put the epairXb in t=
he right jail.
If you whant separation. Create vlan interfaces. Connect them to rl0 and put=
them inside the jail.
> How do I get all vnet jails to pass through the real rl0 interface?
>=20
> Thanks for you help
>=20
>=20
>=20
> # /root >cat /etc/jail.conf
> vimage33 {
> host.hostname =3D "vimage33";
> path =3D "/usr/jails/vimage33";
> mount.fstab =3D "/usr/local/etc/fstab/vimage33";
> exec.start =3D "/bin/sh /etc/rc";
> exec.stop =3D "/bin/sh /etc/rc.shutdown";
> exec.consolelog =3D "/var/log/vimage33.console.log";
> devfs_ruleset =3D "4";
> allow.mount.devfs;
> vnet;
> exec.poststart=3D"vnet.start vimage33 rl0";
> exec.prestop=3D"vnet.stop vimage33";
> }
>=20
> # /root >cat /usr/local/bin/vnet.start
> #!/bin/sh
> jailname=3D$1
> nicname=3D$2
>=20
> jid=3D`jls -j ${jailname} jid`
>=20
> if [ "${jid}" -gt "100" ]; then
> echo " "
> echo "The JID value is greater then 100."
> echo "You must shutdown the host and reboot"
> echo "to zero out the JID counter and recover"
> echo "the lost memory from stopping vimage jails."
> echo " "
> exit 2
> fi
>=20
> ifconfig bridge${jid} create > /dev/null 2> /dev/null
> ifconfig bridge${jid} 10.${jid}.0.1
> ifconfig bridge${jid} up
> ifconfig epair${jid} create > /dev/null 2> /dev/null
> ifconfig bridge${jid} addm ${nicname} addm epair${jid}a
> ifconfig epair${jid}a up
> ifconfig epair${jid}b vnet ${jid}
>=20
> jexec ${jailname} ifconfig epair${jid}b 10.${jid}.0.2
> jexec ${jailname} route add default 10.${jid}.0.1 > /dev/null 2> /dev/null=
> jexec ${jailname} ifconfig lo0 127.0.0.1
>=20
>=20
> # Display the hosts network view before starting any vnet jails
> # /root >ifconfig
> rl0: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
> options=3D2008<VLAN_MTU,WOL_MAGIC>
> ether 00:0c:6e:09:8b:74
> inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
> nd6 options=3D29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
> media: Ethernet autoselect (100baseTX <full-duplex>)
> status: active
> lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
> options=3D600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
> inet6 ::1 prefixlen 128
> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
> inet 127.0.0.1 netmask 0xff000000
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
>=20
> # Start the first vnet jail
> # /root >jail -f /etc/jail.conf -c vimage33
> vimage33: created
> bridge1: Ethernet address: 02:8f:94:84:0c:02
> epair1a: Ethernet address: 02:c0:a4:00:0b:0a
> epair1b: Ethernet address: 02:c0:a4:00:0c:0b
>=20
>=20
> # /root >jls
> JID IP Address Hostname Path
> 1 - vimage33 /usr/jails/vimage33
>=20
>=20
> # Lets display the hosts network after the first vnet jail has started
> # /root >ifconfig
> rl0: flags=3D8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric
> options=3D2008<VLAN_MTU,WOL_MAGIC>
> ether 00:0c:6e:09:8b:74
> inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
> nd6 options=3D29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
> media: Ethernet autoselect (100baseTX <full-duplex>)
> status: active
>=20
> lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
> options=3D600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
> inet6 ::1 prefixlen 128
> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
> inet 127.0.0.1 netmask 0xff000000
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> bridge1: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric
> ether 02:8f:94:84:0c:01
> inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
> maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
> root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
> member: epair1a flags=3D143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
> ifmaxaddr 0 port 9 priority 128 path cost 14183
> member: rl0 flags=3D143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
> ifmaxaddr 0 port 5 priority 128 path cost 200000
> epair1a: flags=3D8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
> options=3D8<VLAN_MTU>
> ether 02:c0:a4:00:09:0a
> inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
> status: active
>=20
>=20
> # Login to the vnet jail and display the jails view of the network
> # /root >jexec vimage33 tcsh
> vimage33 / >ifconfig
> lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
> options=3D600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
> inet 127.0.0.1 netmask 0xff000000
> inet6 ::1 prefixlen 128
> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> epair1b: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
> options=3D8<VLAN_MTU>
> ether 02:c0:a4:00:0a:0b
> inet 10.1.0.2 netmask 0xff000000 broadcast 10.255.255.255
> inet6 fe80::c0:a4ff:fe00:a0b%epair1b prefixlen 64 scopeid 0x2
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
> status: active
>=20
>=20
> # Yes the vnet jail can reach the public network
> vimage33 / >ping -c 4 8.8.178.135
> PING 8.8.178.135 (8.8.178.135): 56 data bytes
> 64 bytes from 8.8.178.135: icmp_seq=3D0 ttl=3D51 time=3D84.645 ms
> 64 bytes from 8.8.178.135: icmp_seq=3D1 ttl=3D51 time=3D86.950 ms
> 64 bytes from 8.8.178.135: icmp_seq=3D2 ttl=3D51 time=3D83.274 ms
> 64 bytes from 8.8.178.135: icmp_seq=3D3 ttl=3D51 time=3D82.660 ms
>=20
> --- 8.8.178.135 ping statistics ---
> 4 packets transmitted, 4 packets received, 0.0% packet loss
> round-trip min/avg/max/stddev =3D 82.660/84.382/86.950/1.647 ms
>=20
> vimage33 / >exit
> exit
>=20
>=20
> # Lets start the second vnet jail
> # /root >cat /etc/jail.conf.22
> vimage22 {
> host.hostname =3D "vimage22";
> path =3D "/usr/jails/vimage22";
> mount.fstab =3D "/usr/local/etc/fstab/vimage22";
> exec.start =3D "/bin/sh /etc/rc";
> exec.stop =3D "/bin/sh /etc/rc.shutdown";
> exec.consolelog =3D "/var/log/vimage22.console.log";
> devfs_ruleset =3D "4";
> allow.mount.devfs;
> vnet;
> exec.poststart=3D"vnet.start vimage22 rl0";
> exec.prestop=3D"vnet.stop vimage22";
> }
>=20
>=20
> # /root >jail -f /etc/jail.conf.22 -c vimage22
> vimage22: created
>=20
> # Notice this message about rl0
> ifconfig: BRDGADD rl0: Device busy
>=20
> bridge2: Ethernet address: 02:8f:94:84:0c:02
> epair2a: Ethernet address: 02:c0:a4:00:0b:0a
> epair2b: Ethernet address: 02:c0:a4:00:0c:0b
>=20
>=20
>=20
> # Lets check the hosts view of the network - no rl0 on bridge2
> # /root >ifconfig
> rl0: flags=3D8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
> options=3D2008<VLAN_MTU,WOL_MAGIC>
> ether 00:0c:6e:09:8b:74
> inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
> nd6 options=3D29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
> media: Ethernet autoselect (100baseTX <full-duplex>)
> status: active
> lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
> options=3D600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
> inet6 ::1 prefixlen 128
> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
> inet 127.0.0.1 netmask 0xff000000
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> bridge1: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
> ether 02:8f:94:84:0c:01
> inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
> maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
> root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
> member: epair1a flags=3D143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
> ifmaxaddr 0 port 9 priority 128 path cost 14183
> member: rl0 flags=3D143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
> ifmaxaddr 0 port 5 priority 128 path cost 200000
> epair1a: flags=3D8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
> options=3D8<VLAN_MTU>
> ether 02:c0:a4:00:09:0a
> inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
> status: active
> bridge2: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
> ether 02:8f:94:84:0c:02
> inet 10.2.0.1 netmask 0xff000000 broadcast 10.255.255.255
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
> maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
> root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
> epair2a: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
> options=3D8<VLAN_MTU>
> ether 02:c0:a4:00:0b:0a
> inet6 fe80::c0:a4ff:fe00:b0a%epair2a prefixlen 64 scopeid 0xb
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
> status: active
>=20
>=20
> # /root >jls
> JID IP Address Hostname Path
> 1 - vimage33 /usr/jails/vimage33
> 2 - vimage22 /usr/jails/vimage22
>=20
> # login to second vnet jail and see if it has public internet connection
> # /root >jexec vimage22 tcsh
> vimage22 / >ifconfig
> lo0: flags=3D8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
> options=3D600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
> inet 127.0.0.1 netmask 0xff000000
> inet6 ::1 prefixlen 128
> inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> epair2b: flags=3D8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
> options=3D8<VLAN_MTU>
> ether 02:c0:a4:00:0c:0b
> inet 10.2.0.2 netmask 0xff000000 broadcast 10.255.255.255
> inet6 fe80::c0:a4ff:fe00:c0b%epair2b prefixlen 64 scopeid 0x2
> nd6 options=3D21<PERFORMNUD,AUTO_LINKLOCAL>
> media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
> status: active
>=20
> vimage22 / >ping -c 4 8.8.178.135
> PING 8.8.178.135 (8.8.178.135): 56 data bytes
>=20
> --- 8.8.178.135 ping statistics ---
> 4 packets transmitted, 0 packets received, 100.0% packet loss
> vimage22 / >exit
> exit
>=20
>=20
>=20
> # Stop the second vnet jail
> # /root >jail -f /etc/jail.conf.22 -r vimage22
> vimage22: removed
> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
> Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
> Freed UMA keg was not empty (10 items). Lost 2 pages of memory.
> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
> hhook_vnet_uninit: hhook_head type=3D1, id=3D1 cleanup required
> hhook_vnet_uninit: hhook_head type=3D1, id=3D0 cleanup required
>=20
>=20
> # Stop the first vnet jail
> # /root >jail -f /etc/jail.conf -r vimage33
> vimage33: removed
> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
> Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
> Freed UMA keg was not empty (10 items). Lost 2 pages of memory.
> Freed UMA keg was not empty (30 items). Lost 2 pages of memory.
> hhook_vnet_uninit: hhook_head type=3D1, id=3D1 cleanup required
> hhook_vnet_uninit: hhook_head type=3D1, id=3D0 cleanup required
>=20
>=20
>=20
> _______________________________________________
> freebsd-jail@freebsd.org mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-jail
> To unsubscribe, send any mail to "freebsd-jail-unsubscribe@freebsd.org"
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?77E31AD0-ABE2-44FA-AB19-CF557038DEBE>