Skip site navigation (1)Skip section navigation (2)
Date:      Thu, 23 May 2013 18:12:07 -0400
From:      Joe <fbsd8@a1poweruser.com>
To:        freebsd-questions@Freebsd.org
Subject:   netgraph network for jail(8) vnet jail unable to reach internet
Message-ID:  <519E9437.6010100@a1poweruser.com>

next in thread | raw e-mail | index | archive | help
Hello list.
Trying to get my script to work that creates a netgraph network for a 
jail(8) vnet jail. Every thing seems to work, but from inside of the 
started vnet jail I can not ping the public internet. The host can ping 
the public internet so the problem has to be in the netgraph script. The 
problem must be staring me in the face but I can just not see it. A 
fresh pair of eyes may see things I am missing.

I'm running 9.1-RELEASE with vimage compiled into the kernel. Non-vnet 
jails work fine and bridge/epair networked vnet jails work fine. The 
host has a single ethernet interface (rl0) facing the public internet. 
Dhcp is used to get the hosts ip address and dns server info. The 
vnet.ng script is designed to create a single ng bridge to rl0 and 
connect vnet jails to it as the jails are started. The following is a 
walk through of a test cycle showing what I can see from the host. At 
the end is a listing of the vnet.ng script.

Thanks for your help


# From the host lets see if there is a netgraph network before we start?
# Nope, no netgraph network running
# /root >ngctl ls -l
There are 2 total nodes:
   Name: rl0             Type: ether     ID: 00000001   Num hooks: 0
   Name: ngctl2850       Type: socket    ID: 00000037   Num hooks: 0



# Here is the jail(8) jail.conf definition statements
# /root >cat /usr/local/etc/vnet/vdir9
vdir9 {
host.hostname       =  "vdir9";
path                =  "/usr/jails/vdir9";
mount.fstab         =  "/usr/local/etc/fstab/vdir9";
exec.start          =  "/bin/sh /etc/rc";
exec.stop           =  "/bin/sh /etc/rc.shutdown";
exec.consolelog     =  "/var/log/vdir9.console.log";
devfs_ruleset       =  "4";
allow.mount.devfs;
vnet;
}



# Lets start the vnet jail
# /root >jail -f /usr/local/etc/qjail.vnet/vdir9 -c
vdir9: created


# Yes it’s really running.
# /root >jls
    JID  IP Address      Hostname      Path
      4  -               vdir9         /usr/jails/vdir9


# Lets start the netgraph network for the running vnet jail
# /root >vnet.ng start vdir9 rl0
Netgraph vnet jail network established successfully!


# Lets check the host for the vnet jail netgraph network
# Yep it’s there and looks complete to me.
# /root >ngctl ls -l

There are 4 total nodes:

   Name: rl0       Type: ether           ID: 00000001   Num hooks: 2
   Local hook      Peer name       Peer type    Peer ID         Peer 
hook
   ----------      ---------       ---------    ------- 
---------
   upper           bridge0         bridge       0000003d        link1 

   lower           bridge0         bridge       0000003d        link0 


   Name: bridge0   Type: bridge          ID: 0000003d   Num hooks: 3
   Local hook      Peer name       Peer type    Peer ID         Peer 
hook
   ----------      ---------       ---------    ------- 
---------
   link2           vdir9           eiface       00000041        ether 

   link1           rl0             ether        00000001        upper 

   link0           rl0             ether        00000001        lower

   Name: vdir9     Type: eiface          ID: 00000041   Num hooks: 1
   Local hook      Peer name       Peer type    Peer ID         Peer 
hook
   ----------      ---------       ---------    ------- 
---------
   ether           bridge0         bridge       0000003d        link2 


  Name: ngctl3126  Type: socket          ID: 00000046   Num hooks: 0


# Lets log into the running jail
# /root >jexec vdir9 tcsh

# Lets ping freebsd.org ip address. No public internet connection
# even though host can do same ping and get good reply.
vdir9 / >ping -c4 8.8.178.135
PING 8.8.178.135 (8.8.178.135): 56 data bytes

--- 8.8.178.135 ping statistics ---
4 packets transmitted, 0 packets received, 100.0% packet loss
vdir9 / >exit
exit

# Now stop the vnet jails netgraph network
# /root >vnet.ng stop vdir9 rl0
Netgraph vnet jail network shutdown successfully!

# and yes the netgraph network is shutdown
# /root >ngctl ls -l
There are 2 total nodes:
   Name: rl0             Type: ether    ID: 00000001   Num hooks: 0
   Name: ngctl3167       Type: socket   ID: 0000004b   Num hooks: 0



# Here is the vnet.ng  script
# It starts and stops the vnet jail’s netgraph network

#!/bin/sh
function=$1
jailname=$2
nicname=$3

jid=`jls -j ${jailname} jid`

# Load netgraph kernel modules if not done already.
for module in ng_socket netgraph ng_bridge ng_eiface ng_ether; do
   if ! kldstat -v | grep -qw ${module}; then
          kldload ${module} || exit 1
   fi
done

# mac manufacturer prefix. Modify if need be.
#mac_prefix="00:1d:92"
mac_prefix="07:22:49"

start() {

sysctl net.inet.ip.forwarding=1 > /dev/null 2> /dev/null

jid=`jls -j ${jailname} jid`

if [ "${jid}" -gt "100" ]; then
   echo " "
   echo "WARNING: The JID value is greater then 100."
   echo "This may indicate many cycles of starting/stopping vnet jails"
   echo "which results in lost memory pages. To recover the lost memory,"
   echo "shutdown the host and reboot. This will zero out the JID"
   echo "counter and make all the memory available again."
   echo " "
fi

# The jid is used in the jails ip address which has max value of 255,
# so draw the line at 250 running vnet jails and then force reboot.
#
if [ "${jid}" -gt "250" ]; then
   echo " "
   echo "ERROR: No more vnet jail ip addresses can be created."
   echo "You MUST shutdown the host and reboot before vnet jails are"
   echo "startable again."
   echo " "
   exit 2
fi

# Check real ethernet interface exists.
#
if ! ngctl info ${nicname}: >/dev/null 2>&1; then
   echo "Error: interface ${nicname} does not exist."
   exit 1
fi
ifconfig ${nicname} up || exit 1


# Check if this ethernet interface has a bridge already created on it.
# If not then create bridge to the ethernet interface.
#
bridge_name=`ngctl show ${nicname}: | grep -m 1 bridge | awk '{print $2}'`

if [ -z "${bridge_name}" ]; then
   # This means no bridge created yet on this ethernet interface.

   # Set NIC interfaces to promiscuous mode and don't overwrite src addr.
   ngctl msg ${nicname}: setpromisc 1 || exit 1
   ngctl msg ${nicname}: setautosrc 0 || exit 1

   bridge_name="bridge0"


   # Create new ng_bridge node and attach it to the ethernet interface,
   # connecting ng_ether:lower hook to bridge:link0
   #
   ngctl mkpeer ${nicname}: bridge lower link0 || exit 1

   # Assigning ${bridge_name} as bridge name.
   ngctl name ${nicname}:lower ${bridge_name} || exit 1

   # Connect ng_ether:upper hook to bridge:link1
   # after bridge name is set.
   #
   ngctl connect ${nicname}: ${bridge_name}: upper link1 || exit 1
fi

# Create and link vnet jail to bridge.
#
ngctl mkpeer eiface ether ether || exit 1
org_eiface=`ngctl l | grep ngeth | tail -n 1 | awk '{print $2}'`

# Renaming vnet jail ng node interface name to jailname.
ngctl name ${org_eiface}: ${jailname} || exit 1
eiface="${jailname}"
ifconfig ${org_eiface} name ${eiface}

# Finding next link number for this bridge.
used_linknum=`ngctl show bridge0: | grep link | cut -c 7-7`
highest_linknum=0
for used_num in ${used_linknum}; do
    if [ "${used_num}" -gt "${highest_linknum}" ]; then
      highest_linknum="${used_num}"
    fi
done
linknum=`expr ${highest_linknum} + 1`


# Connect vnet jail virtual interface to bridge interface.
#
ngctl connect ${eiface}: ${bridge_name}: ether link${linknum} || exit 1

# Move host virtual interface to vnet virtual node.
# Moved interface can be seen in the output of ifconfig command issued
# from within the started vnet jail or by issuing
# 'jexec jailname ifconfig' from the host system.
#
ifconfig ${eiface} vnet ${jailname} || exit 1

# Make lo0 interface localhost.
jexec ${jailname} ifconfig lo0 localhost || exit 1


# Set the MAC address of the new interface using a sensible
# algorithm to prevent conflicts on the network. First three
# octets can be changed by user. Last three octets are
# generated randomly.
#
M4=`od -An -N2 -i /dev/random | sed -e 's/ //g' | \
      awk '{ print $1 % 256 }'`
M5=`od -An -N2 -i /dev/random | sed -e 's/ //g' | \
      awk '{ print $1 % 256 }'`
M6=`od -An -N2 -i /dev/random | sed -e 's/ //g' | \
      awk '{ print $1 % 256 }'`

mac=`printf ${mac_prefix}:%02x:%02x:%02x ${M4} ${M5} ${M6}`


# Setting mac address of ${eiface} to ${mac}
jexec ${jailname} ifconfig ${eiface} link $mac || exit 1

# Set IP address inside of vnet jail ng virtual interface.
jexec ${jailname} ifconfig ${eiface} inet 10.${jid}.0.2 || exit 1

# Assign default route ip address to ng virtual interface
# inside of the vnet jail.
#
jexec ${jailname} route add default 10.${jid}.0.1 || exit 1

echo "Netgraph vnet jail network established successfully!"
}

stop() {

ngctl shutdown ${jailname}: >/dev/null 2>&1

# Get the bridge name for this NIC.
#
bridge_name=`ngctl show ${nicname}: | grep -m 1 bridge | awk '{print $2}'`


# If there are only 2 links (link0 & link1) on the bridge name,
# this means all the vnet jails have been shutdown so shutdown the bridge.
#
linknum=`ngctl show ${bridge_name}: | grep link | wc -l | sed -e "s/ //g"`
[ ${linknum} -eq 2 ] && ngctl shutdown ${bridge_name}: >/dev/null 2>&1

echo "Netgraph vnet jail network shutdown successfully!"

}

# Main entry point.
#
[ "${function}" = "start" ] && start  && exit 0
[ "${function}" = "stop" ]  && stop   && exit 0





Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?519E9437.6010100>