Skip site navigation (1)Skip section navigation (2)
Date:      Mon, 22 Apr 2013 10:56:16 -0400
From:      Joe <fbsd8@a1poweruser.com>
To:        FreeBSD questions <questions@freebsd.org>
Subject:   jail(8) vimage epair bridge
Message-ID:  <51754F90.1090807@a1poweruser.com>

Next in thread | Raw E-Mail | Index | Archive | Help
Hello questions list

I am using jail(8) trying to get a functional vimage environment on my 
9.1-RELEASE system. My PC only has a single real NIC facing the public 
internet. My goal is to be able to have multiple vimage jails, each with 
their own epairXa epairXb and bridgeX where the "X" is the jails JID 
number all having their traffic passing through the single rl0 real 
interface. The vnet.start script shown below handles this nicely.

The problem is after the first vimage jail is started the rl0 interface 
gets marked as busy when the second vimage jail is started.

How do I get all vnet jails to pass through the real rl0 interface?

Thanks for you help



  # /root >cat /etc/jail.conf
  vimage33 {
  host.hostname       =  "vimage33";
  path                =  "/usr/jails/vimage33";
  mount.fstab         =  "/usr/local/etc/fstab/vimage33";
  exec.start          =  "/bin/sh /etc/rc";
  exec.stop           =  "/bin/sh /etc/rc.shutdown";
  exec.consolelog     =  "/var/log/vimage33.console.log";
  devfs_ruleset       =  "4";
  allow.mount.devfs;
  vnet;
  exec.poststart="vnet.start vimage33 rl0";
  exec.prestop="vnet.stop vimage33";
  }

  # /root >cat /usr/local/bin/vnet.start
  #!/bin/sh
  jailname=$1
  nicname=$2

  jid=`jls -j ${jailname} jid`

  if [ "${jid}" -gt "100" ]; then
    echo " "
    echo "The JID value is greater then 100."
    echo "You must shutdown the host and reboot"
    echo "to zero out the JID counter and recover"
    echo "the lost memory from stopping vimage jails."
    echo " "
    exit 2
  fi

  ifconfig bridge${jid} create > /dev/null 2> /dev/null
  ifconfig bridge${jid} 10.${jid}.0.1
  ifconfig bridge${jid} up
  ifconfig epair${jid} create > /dev/null 2> /dev/null
  ifconfig bridge${jid} addm ${nicname} addm epair${jid}a
  ifconfig epair${jid}a up
  ifconfig epair${jid}b vnet ${jid}

  jexec ${jailname} ifconfig epair${jid}b 10.${jid}.0.2
  jexec ${jailname} route add default 10.${jid}.0.1 > /dev/null 2> /dev/null
  jexec ${jailname} ifconfig lo0 127.0.0.1


# Display the hosts network view before starting any vnet jails
# /root >ifconfig
  rl0: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
  	options=2008<VLAN_MTU,WOL_MAGIC>
  	ether 00:0c:6e:09:8b:74
  	inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
  	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
  	media: Ethernet autoselect (100baseTX <full-duplex>)
  	status: active
  plip0: flags=8810<POINTOPOINT,SIMPLEX,MULTICAST> metric 0 mtu 1500
  	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
  lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
  	options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
  	inet6 ::1 prefixlen 128
  	inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
  	inet 127.0.0.1 netmask 0xff000000
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>

  #  Start the first vnet jail
  # /root >jail -f /etc/jail.conf -c vimage33
  vimage33: created
  bridge1: Ethernet address: 02:8f:94:84:0c:02
  epair1a: Ethernet address: 02:c0:a4:00:0b:0a
  epair1b: Ethernet address: 02:c0:a4:00:0c:0b


  # /root >jls
     JID  IP Address      Hostname          Path
       1  -               vimage33          /usr/jails/vimage33

  # Lets display the hosts network after the first vnet jail has started
  # /root >ifconfig
  rl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST> metric 0
  	options=2008<VLAN_MTU,WOL_MAGIC>
  	ether 00:0c:6e:09:8b:74
  	inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
  	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
  	media: Ethernet autoselect (100baseTX <full-duplex>)
  	status: active
  plip0: flags=8810<POINTOPOINT,SIMPLEX,MULTICAST> metric 0 mtu 1500
  	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
  lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
  	options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
  	inet6 ::1 prefixlen 128
  	inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
  	inet 127.0.0.1 netmask 0xff000000
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  bridge1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0 mtu
  	ether 02:8f:94:84:0c:01
  	inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
  	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
  	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
  	member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
  	        ifmaxaddr 0 port 9 priority 128 path cost 14183
  	member: rl0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
  	        ifmaxaddr 0 port 5 priority 128 path cost 200000
  epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
  	options=8<VLAN_MTU>
  	ether 02:c0:a4:00:09:0a
  	inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
  	status: active


  # Login to the vnet jail and display the jails view of the network
  # /root >jexec vimage33 tcsh
  vimage33 / >ifconfig
  lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
  	options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
  	inet 127.0.0.1 netmask 0xff000000
  	inet6 ::1 prefixlen 128
  	inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  epair1b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
  	options=8<VLAN_MTU>
  	ether 02:c0:a4:00:0a:0b
  	inet 10.1.0.2 netmask 0xff000000 broadcast 10.255.255.255
  	inet6 fe80::c0:a4ff:fe00:a0b%epair1b prefixlen 64 scopeid 0x2
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
  	status: active


  #  Yes the vnet jail can reach the public network
  vimage33 / >ping -c 4 8.8.178.135
  PING 8.8.178.135 (8.8.178.135): 56 data bytes
  64 bytes from 8.8.178.135: icmp_seq=0 ttl=51 time=84.645 ms
  64 bytes from 8.8.178.135: icmp_seq=1 ttl=51 time=86.950 ms
  64 bytes from 8.8.178.135: icmp_seq=2 ttl=51 time=83.274 ms
  64 bytes from 8.8.178.135: icmp_seq=3 ttl=51 time=82.660 ms

  --- 8.8.178.135 ping statistics ---
  4 packets transmitted, 4 packets received, 0.0% packet loss
  round-trip min/avg/max/stddev = 82.660/84.382/86.950/1.647 ms

  vimage33 / >exit
  exit


  # Lets start the second vnet jail
  # /root >cat /etc/jail.conf.22
  vimage22 {
  host.hostname       =  "vimage22";
  path                =  "/usr/jails/vimage22";
  mount.fstab         =  "/usr/local/etc/fstab/vimage22";
  exec.start          =  "/bin/sh /etc/rc";
  exec.stop           =  "/bin/sh /etc/rc.shutdown";
  exec.consolelog     =  "/var/log/vimage22.console.log";
  devfs_ruleset       =  "4";
  allow.mount.devfs;
  vnet;
  exec.poststart="vnet.start vimage22 rl0";
  exec.prestop="vnet.stop vimage22";
  }


  # /root >jail -f /etc/jail.conf.22 -c vimage22
  vimage22: created

# Notice this message about rl0
  ifconfig: BRDGADD rl0: Device busy

  bridge2: Ethernet address: 02:8f:94:84:0c:02
  epair2a: Ethernet address: 02:c0:a4:00:0b:0a
  epair2b: Ethernet address: 02:c0:a4:00:0c:0b



# Lets check the hosts view of the network - no rl0 on bridge2
  # /root >ifconfig
  rl0: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
  	options=2008<VLAN_MTU,WOL_MAGIC>
  	ether 00:0c:6e:09:8b:74
  	inet 10.0.10.5 netmask 0xfffffff8 broadcast 10.0.10.7
  	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
  	media: Ethernet autoselect (100baseTX <full-duplex>)
  	status: active
  plip0: flags=8810<POINTOPOINT,SIMPLEX,MULTICAST> metric 0 mtu 1500
  	nd6 options=29<PERFORMNUD,IFDISABLED,AUTO_LINKLOCAL>
  lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
  	options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
  	inet6 ::1 prefixlen 128
  	inet6 fe80::1%lo0 prefixlen 64 scopeid 0x7
  	inet 127.0.0.1 netmask 0xff000000
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  bridge1: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
  	ether 02:8f:94:84:0c:01
  	inet 10.1.0.1 netmask 0xff000000 broadcast 10.255.255.255
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
  	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
  	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
  	member: epair1a flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
  	        ifmaxaddr 0 port 9 priority 128 path cost 14183
  	member: rl0 flags=143<LEARNING,DISCOVER,AUTOEDGE,AUTOPTP>
  	        ifmaxaddr 0 port 5 priority 128 path cost 200000
  epair1a: flags=8943<UP,BROADCAST,RUNNING,PROMISC,SIMPLEX,MULTICAST>
  	options=8<VLAN_MTU>
  	ether 02:c0:a4:00:09:0a
  	inet6 fe80::c0:a4ff:fe00:90a%epair1a prefixlen 64 scopeid 0x9
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
  	status: active
  bridge2: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
  	ether 02:8f:94:84:0c:02
  	inet 10.2.0.1 netmask 0xff000000 broadcast 10.255.255.255
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	id 00:00:00:00:00:00 priority 32768 hellotime 2 fwddelay 15
  	maxage 20 holdcnt 6 proto rstp maxaddr 2000 timeout 1200
  	root id 00:00:00:00:00:00 priority 32768 ifcost 0 port 0
  epair2a: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
  	options=8<VLAN_MTU>
  	ether 02:c0:a4:00:0b:0a
  	inet6 fe80::c0:a4ff:fe00:b0a%epair2a prefixlen 64 scopeid 0xb
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
  	status: active


  # /root >jls
     JID  IP Address      Hostname         Path
       1  -               vimage33         /usr/jails/vimage33
       2  -               vimage22         /usr/jails/vimage22

  # login to second vnet jail and see if it has public internet connection
  # /root >jexec vimage22 tcsh
  vimage22 / >ifconfig
  lo0: flags=8049<UP,LOOPBACK,RUNNING,MULTICAST> metric 0 mtu 16384
  	options=600003<RXCSUM,TXCSUM,RXCSUM_IPV6,TXCSUM_IPV6>
  	inet 127.0.0.1 netmask 0xff000000
  	inet6 ::1 prefixlen 128
  	inet6 fe80::1%lo0 prefixlen 64 scopeid 0x1
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  epair2b: flags=8843<UP,BROADCAST,RUNNING,SIMPLEX,MULTICAST> metric 0
  	options=8<VLAN_MTU>
  	ether 02:c0:a4:00:0c:0b
  	inet 10.2.0.2 netmask 0xff000000 broadcast 10.255.255.255
  	inet6 fe80::c0:a4ff:fe00:c0b%epair2b prefixlen 64 scopeid 0x2
  	nd6 options=21<PERFORMNUD,AUTO_LINKLOCAL>
  	media: Ethernet 10Gbase-T (10Gbase-T <full-duplex>)
  	status: active

  vimage22 / >ping -c 4 8.8.178.135
  PING 8.8.178.135 (8.8.178.135): 56 data bytes

  --- 8.8.178.135 ping statistics ---
  4 packets transmitted, 0 packets received, 100.0% packet loss
  vimage22 / >exit
  exit



# Stop the second vnet jail
  # /root >jail -f /etc/jail.conf.22 -r vimage22
  vimage22: removed
  Freed UMA keg was not empty (30 items).  Lost 2 pages of memory.
  Freed UMA keg was not empty (203 items). Lost 1 pages of memory.
  Freed UMA keg was not empty (30 items).  Lost 2 pages of memory.
  Freed UMA keg was not empty (10 items).  Lost 2 pages of memory.
  Freed UMA keg was not empty (30 items).  Lost 2 pages of memory.
  hhook_vnet_uninit: hhook_head type=1, id=1 cleanup required
  hhook_vnet_uninit: hhook_head type=1, id=0 cleanup required


  # Stop the first vnet jail
  # /root >jail -f /etc/jail.conf -r vimage33
  vimage33: removed
  Freed UMA keg was not empty (30 items).  Lost 2 pages of memory.
  Freed UMA keg was not empty (203 items).  Lost 1 pages of memory.
  Freed UMA keg was not empty (30 items).  Lost 2 pages of memory.
  Freed UMA keg was not empty (10 items).  Lost 2 pages of memory.
  Freed UMA keg was not empty (30 items).  Lost 2 pages of memory.
  hhook_vnet_uninit: hhook_head type=1, id=1 cleanup required
  hhook_vnet_uninit: hhook_head type=1, id=0 cleanup required



  # /root >exit
  exit

  Script done on Mon Apr 22 09:35:41 2013





Want to link to this message? Use this URL: <http://docs.FreeBSD.org/cgi/mid.cgi?51754F90.1090807>