Date: Thu, 29 May 2014 01:01:55 +0000 (UTC) From: Nathan Whitehorn <nwhitehorn@FreeBSD.org> To: src-committers@freebsd.org, svn-src-user@freebsd.org Subject: svn commit: r266825 - user/nwhitehorn/condorports Message-ID: <201405290101.s4T11tN2074424@svn.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: nwhitehorn Date: Thu May 29 01:01:54 2014 New Revision: 266825 URL: http://svnweb.freebsd.org/changeset/base/266825 Log: Freshen scritps and documentation. Still needs some testing. Modified: user/nwhitehorn/condorports/README user/nwhitehorn/condorports/buildportsdag.sh user/nwhitehorn/condorports/prepbuildjail Modified: user/nwhitehorn/condorports/README ============================================================================== --- user/nwhitehorn/condorports/README Thu May 29 00:44:59 2014 (r266824) +++ user/nwhitehorn/condorports/README Thu May 29 01:01:54 2014 (r266825) @@ -23,7 +23,7 @@ Submit node prerequisites: -------------------------- - sysutils/condor, configured to allow job submission - Copies of the relevant ports trees, at the same paths as they are on the - build nodes + build nodes (in /buildshare/ports/XXX) Build node prerequisites: ------------------------- @@ -31,7 +31,9 @@ Build node prerequisites: (adapted for the system in question) - security/sudo, with sudoers file in this directory - A lot of local disk space in /scratch -- A copy of all relevant worlds in /releases/<releasenames> +- A copy of all relevant worlds in /buildshare/releases/<releasenames> +- A copy of all relevant ports trees in /buildshare/ports/XXX +- NB: /buildshare can be read-only and node-private - The prepbuildjail and reapbuildjail scripts installed in /pkgscripts Using: @@ -39,14 +41,14 @@ Using: The basic procedure is: cd condorports -./buildportsdag.sh +./buildportsdag.sh portsdir condor_submit_dag -maxpre 10 ports.dag <Eat one to several meals depending on cluster size> -This will configure a Condor DAG for the ports tree in /usr/ports (the default, -see the options section below), the current system's architecture -(again see options), and release (again), placing the packages in the current -directory (once more). Submitting the DAG to the scheduler will begin the build. +This will configure a Condor DAG for the ports tree in +/buildshare/ports/portsdir, the current system's architecture (see options +below), and release (again), placing the packages in the current directory +(once more). Submitting the DAG to the scheduler will begin the build. NOTE: The -maxpre option is *very* important, as this controls the number of simultaneous make fetch operations to run on the submit node. If you don't @@ -59,7 +61,7 @@ condor_q. Options to buildportsdag.sh are specified using environment variables: - PORTSDIR: Name of ports directory to build in. This *MUST* occur on the same path on both the build and submit nodes. This directory may be read only. - Default: /usr/ports + Default: NONE - PKGSDIR: Name of directory on submit node in which to place built packages. Need not exist on the build nodes and can be on a local disk. Default: $PORTSDIR/packages Modified: user/nwhitehorn/condorports/buildportsdag.sh ============================================================================== --- user/nwhitehorn/condorports/buildportsdag.sh Thu May 29 00:44:59 2014 (r266824) +++ user/nwhitehorn/condorports/buildportsdag.sh Thu May 29 01:01:54 2014 (r266825) @@ -1,13 +1,13 @@ #!/bin/sh -: ${PORTSDIR=/usr/ports} : ${PKGSDIR=$PORTSDIR/packages} : ${DISTHORDE=$PORTSDIR/distfiles} : ${ARCH=`uname -p`} -: ${RELEASE=$ARCH/`uname -r`} +: ${RELEASE=$ARCH-`uname -r | cut -f 1 -d - | tr . -`} : ${STAGEDIR=$(pwd)/stage-$ARCH/$PKGSDIR} : ${__MAKE_CONF=/dev/null}; export __MAKE_CONF export PORT_DBDIR=/var/empty # Avoid host contamination +PORTSDIR=$1 mkdir -p $STAGEDIR @@ -15,9 +15,9 @@ dagjobs=$(pwd)/ports.dagjobs dagdeps=$(pwd)/ports.dagdeps dag=$(pwd)/ports.dag rm -f $dagjobs $dagdeps -for category in $(make -C $PORTSDIR -V SUBDIR); do for port in $(make -C $PORTSDIR/$category -V SUBDIR); do +for category in $(make -C /buildshare/ports/$PORTSDIR -V SUBDIR); do for port in $(make -C /buildshare/ports/$PORTSDIR/$category -V SUBDIR); do echo $category/$port - job=$PORTSDIR/$category/$port + job=/buildshare/ports/$PORTSDIR/$category/$port cd $job echo JOB $job package.sub >> $dagjobs # One fetch for all archs with dummy fetch job? @@ -25,7 +25,7 @@ for category in $(make -C $PORTSDIR -V S export UNAME_p=$ARCH PKGNAME=$(make package-name) TARBALL=$STAGEDIR/stage-$category-$port.tar - echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL $DISTHORDE $PORTSDIR >> $dagjobs + echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL $DISTHORDE /buildshare/ports/$PORTSDIR >> $dagjobs echo SCRIPT POST $job postbuild.sh $TARBALL \$RETURN >> $dagjobs echo VARS $job BuildArch=\"$ARCH\" >> $dagjobs echo VARS $job port=\"$job\" >> $dagjobs Modified: user/nwhitehorn/condorports/prepbuildjail ============================================================================== --- user/nwhitehorn/condorports/prepbuildjail Thu May 29 00:44:59 2014 (r266824) +++ user/nwhitehorn/condorports/prepbuildjail Thu May 29 01:01:54 2014 (r266825) @@ -8,7 +8,7 @@ case $slot in *[^a-zA-Z0-9]* ) echo "Non-alphanumeric slot $slot"; exit 1 esac case $release in - *[^a-zA-Z0-9]* ) echo "Non-alphanumeric release $release"; exit 1 + *[^a-zA-Z0-9-]* ) echo "Non-alphanumeric release $release"; exit 1 esac case $ports in *[^a-zA-Z0-9]* ) echo "Non-alphanumeric ports $ports"; exit 1
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201405290101.s4T11tN2074424>