Date: Mon, 22 Jul 2013 15:02:56 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r253547 - user/nwhitehorn/condorports Message-ID: <201307221502.r6MF2u6B006350@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Mon Jul 22 15:02:55 2013 New Revision: 253547 URL: http://svnweb.freebsd.org/changeset/base/253547 Log: Give up on distributing subsets of the ports tree. 2500 ports depend on resources elsewhere in the tree with no easy way to track the dependencies. There is still no requirement for a shared filesystem, but the relevant (static, read-only) ports trees must be supplied ahead of time to the build nodes. They could be advertised via classads, for example. This includes a few changes to the scripts that get everything running in a production kind of way. My desktop, in a cluster with itself, has built over a thousand packages in the last few hours. I'll write a few more scripts to automate node setup and some documentation and then kick this over to bapt for integration with poudriere. Modified: user/nwhitehorn/condorports/buildport.sh user/nwhitehorn/condorports/buildportsdag.sh user/nwhitehorn/condorports/package.sub user/nwhitehorn/condorports/prepbuildjail user/nwhitehorn/condorports/reapbuildjail user/nwhitehorn/condorports/stagebuildfiles.sh Modified: user/nwhitehorn/condorports/buildport.sh ============================================================================== --- user/nwhitehorn/condorports/buildport.sh Mon Jul 22 14:23:46 2013 (r253546) +++ user/nwhitehorn/condorports/buildport.sh Mon Jul 22 15:02:55 2013 (r253547) @@ -6,18 +6,15 @@ port=$1 pkg=$2 release=$3 tarballname=$(basename $4) +portsdir=$5 scratchdir=$(pwd) -# Some settings must go in make.conf: -# USE_PACKAGE_DEPENDS=true - set -e -sudo /pkgscripts/prepbuildjail $_CONDOR_SLOT $release +sudo /pkgscripts/prepbuildjail $_CONDOR_SLOT $release $portsdir cat $tarballname | sudo jexec $_CONDOR_SLOT tar xvf - -sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true DISABLE_MAKE_JOBS=true PKGREPOSITORY=/packages PACKAGES=/packages DISTDIR=/distfiles -sudo jexec $_CONDOR_SLOT make -C $port package PKGREPOSITORY=/packages BATCH=true NO_LATEST_LINK=true -ls -lh /scratch/$_CONDOR_SLOT/packages +sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true WRKDIRPREFIX=/obj DISABLE_MAKE_JOBS=true PKGREPOSITORY=/packages PACKAGES=/packages DISTDIR=/distfiles +sudo jexec $_CONDOR_SLOT make -C $port package PKGREPOSITORY=/packages BATCH=true NO_LATEST_LINK=true WRKDIRPREFIX=/obj cp /scratch/$_CONDOR_SLOT/packages/$pkg $scratchdir/$pkg sudo /pkgscripts/reapbuildjail $_CONDOR_SLOT set +e Modified: user/nwhitehorn/condorports/buildportsdag.sh ============================================================================== --- user/nwhitehorn/condorports/buildportsdag.sh Mon Jul 22 14:23:46 2013 (r253546) +++ user/nwhitehorn/condorports/buildportsdag.sh Mon Jul 22 15:02:55 2013 (r253547) @@ -23,13 +23,14 @@ for category in $(make -C $PORTSDIR -V S export UNAME_p=$ARCH PKGNAME=$(make package-name) - TARBALL=$STAGEDIR/$(make -V UNIQUENAME).tgz + TARBALL=$STAGEDIR/stage-$category-$port.tar echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL $DISTHORDE >> $dagjobs echo SCRIPT POST $job postbuild.sh $TARBALL \$RETURN >> $dagjobs echo VARS $job BuildArch=\"$ARCH\" >> $dagjobs echo VARS $job port=\"$job\" >> $dagjobs echo VARS $job pkg=\"$PKGNAME.txz\" >> $dagjobs echo VARS $job pkgdir=\"$PKGSDIR/All\" >> $dagjobs + echo VARS $job portsdir=\"$PORTSDIR\" >> $dagjobs echo VARS $job stagetarball=\"$TARBALL\" >> $dagjobs echo VARS $job release=\"$RELEASE\" >> $dagjobs deps=$(make -V _DEPEND_DIRS) Modified: user/nwhitehorn/condorports/package.sub ============================================================================== --- user/nwhitehorn/condorports/package.sub Mon Jul 22 14:23:46 2013 (r253546) +++ user/nwhitehorn/condorports/package.sub Mon Jul 22 15:02:55 2013 (r253547) @@ -15,5 +15,5 @@ when_to_transfer_output=ON_EXIT transfer_output_remaps="$(pkg)=$(pkgdir)/$(pkg)" transfer_input_files=$(stagetarball) -arguments= $(port) $(pkg) $(release) $(stagetarball) +arguments= $(port) $(pkg) $(release) $(stagetarball) $(portsdir) queue Modified: user/nwhitehorn/condorports/prepbuildjail ============================================================================== --- user/nwhitehorn/condorports/prepbuildjail Mon Jul 22 14:23:46 2013 (r253546) +++ user/nwhitehorn/condorports/prepbuildjail Mon Jul 22 15:02:55 2013 (r253547) @@ -1,6 +1,7 @@ #!/bin/sh slot=$1 release=$2 +ports=$3 #Audit slot and release names case $slot in @@ -9,6 +10,9 @@ esac case $release in *[^a-zA-Z0-9]* ) echo "Non-alphanumeric release $release"; exit 1 esac +case $ports in + *[^a-zA-Z0-9/-]* ) echo "Non-alphanumeric ports $ports"; exit 1 +esac if [ -z "$slot" -o -z "$release" ]; then echo "Zero-length slot or release" exit 1 @@ -20,22 +24,43 @@ if [ -d /scratch/$slot ]; then fi cleanup() { + rm -f /scratch/$slot.lock /pkgscripts/reapbuildjail $slot + exit 1 } set -e trap cleanup EXIT -. /etc/rc.subr - #TODO: localhost networking -mkdir -p /scratch/$slot +set -o noclobber +echo > /scratch/$slot.lock + +set +o noclobber +cat << EOF > /scratch/$slot.conf +$slot { + path="/scratch/$slot"; + host.hostname="$slot.`hostname`"; + persist="true"; + mount.devfs="true"; + mount.fstab="/scratch/$slot.fstab"; +} +EOF # TODO: Setting nosuid seems to break "below" -#mount_unionfs -o below /releases/$release /scratch/$slot -mount_unionfs -o below /scratch/$release /scratch/$slot -devfs_mount_jail /scratch/$slot/dev 4 +echo "/scratch/$release /scratch/$slot unionfs below 0 0" > /scratch/$slot.fstab +if [ -n "$ports" ]; then + mkdir -p /scratch/$slot/$ports + echo "$ports /scratch/$slot/$ports nullfs ro 0 0" >> /scratch/$slot.fstab +fi -jail -c name=$slot path=/scratch/$slot host.hostname=$slot.`hostname` persist=true +mkdir -p /scratch/$slot + +jail -f /scratch/$slot.conf -c $slot +ls /scratch/$slot/dev > /dev/null # Hack around unionfs bugs +if [ -n "$ports" ]; then + ls /scratch/$slot/$ports > /dev/null # And once more +fi trap true EXIT +rm /scratch/$slot.lock Modified: user/nwhitehorn/condorports/reapbuildjail ============================================================================== --- user/nwhitehorn/condorports/reapbuildjail Mon Jul 22 14:23:46 2013 (r253546) +++ user/nwhitehorn/condorports/reapbuildjail Mon Jul 22 15:02:55 2013 (r253547) @@ -16,10 +16,11 @@ if [ ! -d /scratch/$slot ]; then fi set +e -jail -r $slot -umount -f /scratch/$slot/dev -umount -f /scratch/$slot +set -o noclobber +echo > /scratch/$slot.lock +jail -f /scratch/$slot.conf -r $slot chflags -R noschg /scratch/$slot -rm -rf /scratch/$slot +rm -rf /scratch/$slot /scratch/$slot.conf /scratch/$slot.fstab set +e +rm /scratch/$slot.lock Modified: user/nwhitehorn/condorports/stagebuildfiles.sh ============================================================================== --- user/nwhitehorn/condorports/stagebuildfiles.sh Mon Jul 22 14:23:46 2013 (r253546) +++ user/nwhitehorn/condorports/stagebuildfiles.sh Mon Jul 22 15:02:55 2013 (r253547) @@ -9,8 +9,8 @@ DISTHORDE=$5 # Fetch distfiles if unfetched cd $1 -make checksum DISTDIR=$DISTHORDE -make package-links PACKAGES=$2 +make -DBATCH checksum DISTDIR=$DISTHORDE +make -DBATCH package-links PACKAGES=$2 mkdir -p $2/All @@ -29,22 +29,15 @@ ALLFILES=$(for f in `make -V ALLFILES`; DEPENDS=`make all-depends-list` PKGLIST=$(for x in $DEPENDS; do make -C $x PACKAGES=$2 -V PKGFILE; done) -# Take an interesting subset of the ports tree -# TODO: how on Earth to do this robustly? -PORTSDIR=$(make -V PORTSDIR) -PORTS_SUBSET="$PORTSDIR/Mk $PORTSDIR/Templates $PORTSDIR/Tools $PORTSDIR/Keywords $PORTSDIR/UIDs $PORTSDIR/GIDs $PORTSDIR/*/Makefile.inc $DEPENDS $(make -V DIR_DEPENDS) $(make -V MASTERDIR)" # Other important bits? - -tar cvf $TARBALL.tmp $ALLFILES $PKGLIST $PORTS_SUBSET $1 rm -rf $TARBALL-dir -mkdir -p $TARBALL-dir/packages $TARBALL-dir/distfiles +mkdir -p $TARBALL-dir/packages $TARBALL-dir/distfiles/$(make -V DIST_SUBDIR) if [ -n "$ALLFILES" ]; then - ln -s $ALLFILES $TARBALL-dir/distfiles + ln -s $ALLFILES $TARBALL-dir/distfiles/$(make -V DIST_SUBDIR) fi if [ -n "$PKGLIST" ]; then ln -s $PKGLIST $TARBALL-dir/packages fi -tar -r -C $TARBALL-dir -v -L -f $TARBALL.tmp distfiles packages -tar cvzf $TARBALL @$TARBALL.tmp -rm -rf $TARBALL.tmp $TARBALL-dir +tar -c -C $TARBALL-dir -v -L -f $TARBALL distfiles packages +rm -rf $TARBALL-dir set +e
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201307221502.r6MF2u6B006350>