From owner-svn-src-projects@FreeBSD.ORG Thu Apr 26 20:14:27 2012 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id 468B9106564A; Thu, 26 Apr 2012 20:14:27 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 271F88FC14; Thu, 26 Apr 2012 20:14:27 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q3QKERta066663; Thu, 26 Apr 2012 20:14:27 GMT (envelope-from bapt@svn.freebsd.org) Received: (from bapt@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q3QKERnu066659; Thu, 26 Apr 2012 20:14:27 GMT (envelope-from bapt@svn.freebsd.org) Message-Id: <201204262014.q3QKERnu066659@svn.freebsd.org> From: Baptiste Daroussin Date: Thu, 26 Apr 2012 20:14:27 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r234718 - projects/portbuild/scripts X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Thu, 26 Apr 2012 20:14:27 -0000 Author: bapt Date: Thu Apr 26 20:14:26 2012 New Revision: 234718 URL: http://svn.freebsd.org/changeset/base/234718 Log: 100% jailed operation, all the operation on remote hosts are done in jails, with only the fetch section having access to the network. The bonus, is that it simplifies a lot cleanup While here cleanup the detection of mount points to umount, and don't forget anymore any /dev/md* device if any is used Reviewed by: linimon Approved by: linimon Modified: projects/portbuild/scripts/clean-chroot projects/portbuild/scripts/cleanup-chroots projects/portbuild/scripts/portbuild Modified: projects/portbuild/scripts/clean-chroot ============================================================================== --- projects/portbuild/scripts/clean-chroot Thu Apr 26 20:11:57 2012 (r234717) +++ projects/portbuild/scripts/clean-chroot Thu Apr 26 20:14:26 2012 (r234718) @@ -3,37 +3,26 @@ # client-side script to clean up a chroot -kill_procs() -{ - dir=$1 - mount=$2 - - pids="XXX" - while [ ! -z "${pids}" ]; do - pids=$(fstat -f "${dir}${mount}" | tail +2 | awk '{print $3}' | sort -u) - if [ ! -z "${pids}" ]; then - echo "clean-chroot: Killing off pids in ${dir}" - ps -p $pids - kill -KILL ${pids} 2> /dev/null - sleep 2 - fi - done -} - cleanup_mount() { chroot=$1 mount=$2 + + chrootnb=${chroot##*/} + #chroot dir + branch=${chroot%/*} + #buildid dir + branch=${branch%/*} + #branch dir + branch=${branch%/*} + #branch name + branch=${branch##*/} + + jname=${branch}-${chrootnb} + + # Stop the jail if running + jls -qj ${jname} > /dev/null 2>&1 && jail -r ${jname} - if [ -d ${chroot}${mount} ]; then - mdir=$(fstat -f ${chroot}${mount} | head -2 | tail -1 | awk '{print $5}') - if [ "${mdir}" = "MOUNT" ]; then - umount -f ${chroot}${mount} || echo "clean-chroot: Cleanup of ${chroot}${mount} on $(hostname) failed!" - fi - if [ "${mdir}" = "${chroot}${mount}" ]; then - kill_procs ${chroot} ${mount} - umount -f ${chroot}${mount} || echo "clean-chroot: Cleanup of ${chroot}${mount} on $(hostname) failed!" - fi - fi + umount -f ${chroot}${mount} || echo "clean-chroot: Cleanup of ${chroot}${mount} on $(hostname) failed!" } arch=$1 @@ -78,9 +67,12 @@ for i in ${ARCHS_REQUIRING_LINPROCFS}; d fi done -for i in /a/ports /usr/src /dev /root/.ccache; do +for i in /a/ports /usr/src /dev; do cleanup_mount ${chroot} ${i} done +if [ ! -z "${ccache_dir}" ]; then + cleanup_mount ${chroot} /root/.ccache +fi if [ "${use_zfs}" = "1" ]; then cleanup_mount ${chroot} "" Modified: projects/portbuild/scripts/cleanup-chroots ============================================================================== --- projects/portbuild/scripts/cleanup-chroots Thu Apr 26 20:11:57 2012 (r234717) +++ projects/portbuild/scripts/cleanup-chroots Thu Apr 26 20:14:26 2012 (r234718) @@ -17,62 +17,34 @@ REMOVE_NOTINUSE_CHROOT_TIME=60 #VERBOSE=1 -kill_procs() -{ - dir=$1 - - pids="XXX" - while [ ! -z "${pids}" ]; do - pids=$(fstat -f "$dir" | tail +2 | awk '{print $3}' | sort -u) - if [ ! -z "${pids}" ]; then - echo "cleanup-chroots: Killing off pids in ${dir} on $(hostname)" - ps -p $pids - kill -KILL ${pids} 2> /dev/null - sleep 2 - echo - fi - done -} - -# clean up a single mount -cleanup_mount() { - chroot=$1 - mount=$2 - - if [ -d ${chroot}${mount} ]; then - mdir=$(fstat -f ${chroot}${mount} | head -2 | tail -1 | awk '{print $5}') - if [ "${mdir}" = "MOUNT" ]; then - umount ${chroot}${mount} || echo "cleanup-chroots: Cleanup of ${chroot}${mount} on $(hostname) failed!" - fi - if [ "${mdir}" = "${chroot}${mount}" ]; then - kill_procs ${chroot}${mount} - umount ${chroot}${mount} || echo "cleanup-chroots: Cleanup of ${chroot}${mount} on $(hostname) failed!" - fi - fi -} - # cleanup all mounts (old NFS, devfs, linux), as well as mds cleanup_all() { chroot=$1 - mounts=$(mount | grep ${chroot} | awk '{print $3}') - if [ ! -z "${mounts}" ]; then - if [ $VERBOSE ]; then - echo "cleanup-chroots: trying to kill off mounts:" - echo ${mounts} - echo - fi - for j in ${mounts}; do - umount ${j} || cleanup_mount ${j} - done - # XXX MCL is this redundant with the above? - umount ${chroot}/compat/linux/proc || cleanup_mount ${chroot}/compat/linux/proc - fi - if [ "${use_zfs}" != "1" -a "${use_md_swap}" = "1" ]; then - chrootnum=$(basename ${chroot}) - umount -f /dev/md${chroot} - mdconfig -d -u ${chrootnum} - fi + chrootnb=${chroot##*/} + #chroot dir + branch=${chroot%/*} + #buildid dir + branch=${branch%/*} + #branch dir + branch=${branch%/*} + #branch name + branch=${branch##*/} + + jname=${branch}-${chrootnb} + + # Stop the jail if running + jls -qj ${jname} > /dev/null 2>&1 && jail -r ${jname} + + # umounting normal device + for MNT in $( mount | awk -v mnt="${chroot}/" 'BEGIN{ gsub(/\//, "\\\/", mnt); } { if ($3 ~ mnt && $1 !~ /\/dev\/md/ ) { print $3 }}' | sort -r ); do + umount -f ${MNT} + done + # umount md device if any + for unit in $(mount | awk -v mnt="${chroot}" 'BEGIN{ gsub(/\//, "\\\/", mnt); } { if ($3 ~ mnt && $1 ~ /\/dev\/md/ ) { sub(/\/dev\/md/, "", $1); print $1 }}' | sort -r); do + umount -f /dev/md${unit} + mdconfig -d -u ${unit} + done } # note: uname is not being overridden (should not need client.conf here) Modified: projects/portbuild/scripts/portbuild ============================================================================== --- projects/portbuild/scripts/portbuild Thu Apr 26 20:11:57 2012 (r234717) +++ projects/portbuild/scripts/portbuild Thu Apr 26 20:14:26 2012 (r234718) @@ -46,11 +46,12 @@ copypkg() bailout() { - chroot=$1 + jname=$1 clean=$2 error=$3 pkgname=$4 + jail -r $jname echo -n "$pkgname failed unexpectedly on $(hostname) at " date @@ -61,13 +62,14 @@ arch=$1 branch=$2 buildid=$3 chroot=$4 +jname=${branch}-${chroot##*/} shift 4 # Default niceness value nice=0 if [ ! -d ${pbd}/${arch}/ ]; then - bailout ${chroot} 0 255 ${pkgname} + bailout ${jname} 0 255 ${pkgname} fi . ${pbd}/${arch}/client.conf . ${pbd}/${arch}/common.conf @@ -81,7 +83,7 @@ fi # if buildenv isn't there, then ${pbd} is probably the default, which for # some reason you have not populated. if [ ! -f ${pbd}/scripts/buildenv ]; then - bailout ${chroot} 0 255 ${pkgname} + bailout ${jname} 0 255 ${pkgname} fi . ${pbd}/scripts/buildenv @@ -186,7 +188,7 @@ echo "building ${pkgname} in ${chroot}" bindist=${buildroot}/${branch}/${buildid}/tarballs/bindist.tar bindistlocal=${buildroot}/${branch}/${buildid}/tarballs/bindist-$(hostname).tar if [ ! -f $bindist ]; then - bailout ${chroot} ${clean} 255 ${pkgname} + bailout ${jname} ${clean} 255 ${pkgname} fi if [ -f ${chroot}/.notready ]; then @@ -194,6 +196,16 @@ if [ -f ${chroot}/.notready ]; then if [ -f ${bindistlocal} ]; then tar -C ${chroot} -xpf ${bindistlocal} fi + # jexec -U root will have the right arch in uname -m and uname -p + LOGIN_ENV=",UNAME_p=${arch},UNAME_m=${arch}" + sed -i "" -e "s/:\(setenv.*\):/:\1${LOGIN_ENV}:/" ${chroot}/etc/login.conf + cap_mkdb ${chroot}/etc/login.conf + + cat >> ${chroot}/etc/make.conf << EOF +ARCH=${arch} +MACHINE=${arch} +MACHINE_ARCH=${arch} +EOF # allow additional (not replacement!) knobs for make.conf if [ -f ${builddir}/make.conf.client ]; then cat ${builddir}/make.conf.client >> ${chroot}/etc/make.conf @@ -208,20 +220,11 @@ if [ -f ${chroot}/.notready ]; then touch ${chroot}/.ready fi -if [ "${use_jail}" = "1" ]; then - # Figure out jail IP addr - chrootpid=$(basename ${chroot}) - ipbase=$((${chrootpid}+2)) - ip1=$(($ipbase /(256*256))) - ip2=$((($ipbase - ($ip1*256*256)) /256)) - ip3=$((($ipbase - ($ip1*256*256) - ($ip2*256)))) -fi - -trap "bailout ${chroot} ${clean} ${error} ${pkgname}" 1 2 3 9 10 11 15 +trap "bailout ${jname} ${clean} ${error} ${pkgname}" 1 2 3 9 10 11 15 rm -rf ${chroot}/tmp/* -cd ${chroot}/tmp || bailout ${chroot} ${clean} 255 ${pkgname} -mkdir -p depends distfiles packages || bailout ${chroot} ${clean} 255 ${pkgname} +cd ${chroot}/tmp || bailout ${jname} ${clean} 255 ${pkgname} +mkdir -p depends distfiles packages || bailout ${jname} ${clean} 255 ${pkgname} echo "building ${pkgname} on $(hostname)" | tee ${chroot}/tmp/${pkgname}.log echo "in directory ${chroot}" | tee -a ${chroot}/tmp/${pkgname}.log @@ -247,12 +250,12 @@ if [ ! -z "${ccache_dir}" ]; then fi fi -mount_fs ${builddir}/ports ${chroot}/a/ports ${CLIENT_NFS_MASTER} || bailout ${chroot} ${clean} 255 ${pkgname} +mount_fs ${builddir}/ports ${chroot}/a/ports ${CLIENT_NFS_MASTER} || bailout ${jname} ${clean} 255 ${pkgname} ln -sf ../a/ports ${chroot}/usr/ports -mkdir -p ${chroot}/usr/src || bailout ${chroot} ${clean} 255 ${pkgname} +mkdir -p ${chroot}/usr/src || bailout ${jname} ${clean} 255 ${pkgname} -mount_fs ${builddir}/src ${chroot}${CLIENT_SRCBASE} ${CLIENT_NFS_MASTER} || bailout ${chroot} ${clean} 255 ${pkgname} +mount_fs ${builddir}/src ${chroot}${CLIENT_SRCBASE} ${CLIENT_NFS_MASTER} || bailout ${jname} ${clean} 255 ${pkgname} # set overrides for uname buildenv.client ${arch} ${chroot}${CLIENT_SRCBASE} @@ -283,26 +286,20 @@ mtree -deU -f ${chroot}/a/ports/Template for i in ${ARCHS_REQUIRING_LINPROCFS}; do if [ ${i} = ${arch} ]; then # JDK ports need linprocfs :( - mkdir -p ${chroot}/compat/linux/proc || bailout ${chroot} ${clean} 255 ${pkgname} - mount -t linprocfs linprocfs ${chroot}/compat/linux/proc || bailout ${chroot} ${clean} 255 ${pkgname} + mkdir -p ${chroot}/compat/linux/proc || bailout ${jname} ${clean} 255 ${pkgname} + mount -t linprocfs linprocfs ${chroot}/compat/linux/proc || bailout ${jname} ${clean} 255 ${pkgname} break fi done -_ldconfig_dirs="/lib /usr/lib /usr/lib/compat" -ldconfig_dirs="" -for i in ${_ldconfig_dirs}; do - if [ -d ${chroot}/${i} ]; then - ldconfig_dirs="${ldconfig_dirs} ${i}" - fi -done -chroot ${chroot} /sbin/ldconfig ${ldconfig_dirs} -for i in ${ARCHS_REQUIRING_AOUT_COMPAT}; do - if [ ${i} = ${arch} ]; then - chroot ${chroot} /sbin/ldconfig -aout /usr/lib/aout /usr/lib/compat/aout - break - fi -done +# Start a jail +# XXX bapt: the network 127.0.0.1 allows ports like postfix do not fail querying a network +# device +jail -c persist name=${jname} ip4.addr=127.0.0.1 ip6=disable path=${chroot} \ + host.hostname=${jname} allow.sysvipc allow.socket_af allow.raw_sockets \ + allow.enforce_statfs allow.chflags_allowed + +jexec -U root ${jname} /usr/sbin/service ldconfig start set x $ED $FD $PD $BD $RD $PKGD shift 1 @@ -316,7 +313,7 @@ while [ $# -gt 0 ]; do # XXX MCL more hard-coding if [ ! -f ${chroot}/tmp/depends/$1 ]; then echo "ERROR: Couldn't copy $1" | tee -a ${chroot}/tmp/${pkgname}.log - bailout ${chroot} ${clean} 255 ${pkgname} + bailout ${jname} ${clean} 255 ${pkgname} fi fi shift @@ -327,7 +324,7 @@ cp -p /tmp/${buildid}/scripts/buildscrip # phase 0, compile pnohang if necessary if [ ! -f /tmp/${buildid}/sources/pnohang ]; then cp -p /tmp/${buildid}/sources/pnohang.c ${chroot} - chroot ${chroot} /usr/bin/gcc -o /pnohang -Wall /pnohang.c 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log + jexec -U root ${jname} /usr/bin/gcc -o /pnohang -Wall /pnohang.c 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log if [ $? -ne 0 ]; then error=255 else @@ -340,8 +337,16 @@ fi if [ "${error}" = 0 ]; then # phase 1, make checksum -# Needs to be chroot not jail so that port can be fetched - chroot ${chroot} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log +# restart the jail with network for the fetch part + jail -r ${jname} + jail -c persist name=${jname} ip4=inherit ip6=inherit path=${chroot} \ + host.hostname=${jname} allow.sysvipc allow.socket_af allow.raw_sockets \ + allow.enforce_statfs allow.chflags_allowed + jexec -U root ${jname} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log + jail -r ${jname} + jail -c persist name=${jname} ip4.addr=127.0.0.1 ip6=disable path=${chroot} \ + host.hostname=${jname} allow.sysvipc allow.socket_af allow.raw_sockets \ + allow.enforce_statfs allow.chflags_allowed if [ -f ${chroot}/tmp/status ]; then error=$(cat ${chroot}/tmp/status) else @@ -354,13 +359,7 @@ if [ "${error}" = 0 ]; then # phase 2, make package ln -sf ${pkgname}.log2 ${chroot}/tmp/make.log - if [ "${use_jail}" = 1 ]; then - ifconfig lo0 alias 127.${ip1}.${ip2}.${ip3}/32 - jail -J ${chroot}/tmp/jail.id ${chroot} jail-${chrootpid} 127.${ip1}.${ip2}.${ip3} /usr/bin/env JAIL_ADDR=127.${ip1}.${ip2}.${ip3} HTTP_PROXY=${http_proxy} /usr/bin/nice -n $nice /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1 - ifconfig lo0 delete 127.${ip1}.${ip2}.${ip3} - else - chroot ${chroot} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1 - fi + jexec -U root ${jname} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1 grep pnohang ${chroot}/tmp/${pkgname}.log2 cat ${chroot}/tmp/${pkgname}.log2 >> ${chroot}/tmp/${pkgname}.log @@ -368,6 +367,7 @@ if [ "${error}" = 0 ]; then error=$(cat ${chroot}/tmp/status) fi +jail -r ${jname} rm -rf ${chroot}/${WRKDIRPREFIX} # Record build completion time for ganglia