From owner-svn-src-projects@FreeBSD.ORG Tue Apr 24 21:38:42 2012 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [69.147.83.52]) by hub.freebsd.org (Postfix) with ESMTP id AC2A9106566B; Tue, 24 Apr 2012 21:38:42 +0000 (UTC) (envelope-from bapt@FreeBSD.org) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:4f8:fff6::2c]) by mx1.freebsd.org (Postfix) with ESMTP id 962988FC12; Tue, 24 Apr 2012 21:38:42 +0000 (UTC) Received: from svn.freebsd.org (localhost [127.0.0.1]) by svn.freebsd.org (8.14.4/8.14.4) with ESMTP id q3OLcgUt066103; Tue, 24 Apr 2012 21:38:42 GMT (envelope-from bapt@svn.freebsd.org) Received: (from bapt@localhost) by svn.freebsd.org (8.14.4/8.14.4/Submit) id q3OLcgEx066098; Tue, 24 Apr 2012 21:38:42 GMT (envelope-from bapt@svn.freebsd.org) Message-Id: <201204242138.q3OLcgEx066098@svn.freebsd.org> From: Baptiste Daroussin Date: Tue, 24 Apr 2012 21:38:42 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cc: Subject: svn commit: r234662 - in projects/portbuild: qmanager scripts X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Tue, 24 Apr 2012 21:38:42 -0000 Author: bapt Date: Tue Apr 24 21:38:41 2012 New Revision: 234662 URL: http://svn.freebsd.org/changeset/base/234662 Log: Add support for pkgng Reviewed by: linimon Approved by: linimon Modified: projects/portbuild/qmanager/packagebuild projects/portbuild/scripts/buildscript projects/portbuild/scripts/pdispatch projects/portbuild/scripts/portbuild Modified: projects/portbuild/qmanager/packagebuild ============================================================================== --- projects/portbuild/qmanager/packagebuild Tue Apr 24 20:27:31 2012 (r234661) +++ projects/portbuild/qmanager/packagebuild Tue Apr 24 21:38:41 2012 (r234662) @@ -135,6 +135,8 @@ class Index(object): try: (name, path, prefix, comment, descr, maintainer, categories, bdep, rdep, www, edep, pdep, fdep) = i.rstrip().split("|") + if path == "/usr/ports/ports-mgmt/pkg": + pkgdep = name if targets is None or name in targets: lines.append((name, bdep, rdep, edep, pdep, fdep)) @@ -149,7 +151,14 @@ class Index(object): print "[MASTER] Phase 2" for (name, bdep, rdep, edep, pdep, fdep) in lines: - ports[name].setdeps(bdep, rdep, edep, pdep, fdep) + if not pkgng: + ports[name].setdeps(bdep, rdep, edep, pdep, fdep, "") + continue + + if name == pkgdep: + ports[name].setdeps(bdep, rdep, edep, pdep, fdep, "") + else: + ports[name].setdeps(bdep, rdep, edep, pdep, fdep, pkgdep) lines = None del lines @@ -168,7 +177,7 @@ class Port(object): __slots__ = ["name", "path", "prefix", "comment", "descr", "maintainer", "www", "bdep", "rdep", "edep", "pdep", - "fdep", "alldep", "parents", "depth", "categories"] + "fdep", "pkgdep", "alldep", "parents", "depth", "categories"] self.name = name self.path = path @@ -185,6 +194,7 @@ class Port(object): self.edep = [] self.pdep = [] self.fdep = [] + self.pkgdep = [] self.alldep = [] self.parents = [] @@ -225,6 +235,7 @@ they still need to know about us as depe self.pdep = None self.bdep = None self.rdep = None + self.pkgdep = None self.alldep = None self.parents = None @@ -267,13 +278,18 @@ they still need to know about us as depe pkg.rdep.remove(self) except ValueError: pass + try: + pkg.pkgdep.remove(self) + except ValueError: + pass pkg.alldep.remove(self) sys.exc_clear() self.remove() - def setdeps(self, bdep, rdep, edep, pdep, fdep): + def setdeps(self, bdep, rdep, edep, pdep, fdep, pkgdep): + self.pkgdep = [ports[p] for p in pkgdep.split()] self.fdep = [ports[p] for p in fdep.split()] self.edep = [ports[p] for p in edep.split()] self.pdep = [ports[p] for p in pdep.split()] @@ -281,7 +297,7 @@ they still need to know about us as depe self.rdep = [ports[p] for p in rdep.split()] self.alldep = list(set(chain(self.fdep, self.edep, self.pdep, - self.bdep, self.rdep))) + self.bdep, self.rdep, self.pkgdep))) for p in self.alldep: p.parents.append(self) @@ -443,7 +459,8 @@ class worker(threading.Thread): 'ED':" ".join([p.name + p.sufx for p in pkg.edep]), 'PD':" ".join([p.name + p.sufx for p in pkg.pdep]), 'BD':" ".join([p.name + p.sufx for p in pkg.bdep]), - 'RD':" ".join([p.name + p.sufx for p in pkg.rdep])} + 'RD':" ".join([p.name + p.sufx for p in pkg.rdep]), + 'PKGD':" ".join([p.name + p.sufx for p in pkg.pkgdep])} for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN", "PORTBUILD_CHECKOUT", "PORTBUILD_DATA" ]: if var in os.environ: runenv[var] = os.environ.get(var) @@ -520,7 +537,8 @@ class worker(threading.Thread): wrk.start() def main(arch, branch, buildid, args): - global index, pkg_sufx + global index, pkg_sufx, pkgng + pkgng = False basedir=os.path.realpath(pbd+"/"+arch+"/"+branch+"/builds/"+buildid) buildid=basedir.split("/")[-1] @@ -544,6 +562,9 @@ def main(arch, branch, buildid, args): print "error: pkg_sufx not defined in portbuild.conf" sys.exit(1) + if pkg_sufx == ".txz": + pkgng = True + print "[MASTER] Parsing INDEX..." index = Index(indexfile) index.parse() Modified: projects/portbuild/scripts/buildscript ============================================================================== --- projects/portbuild/scripts/buildscript Tue Apr 24 20:27:31 2012 (r234661) +++ projects/portbuild/scripts/buildscript Tue Apr 24 21:38:41 2012 (r234662) @@ -59,6 +59,11 @@ add_pkg() { pkg_cmd_info="pkg info -e" pkg_cmd_add="pkg add" fi + if [ $use_pkgng = "yes" -a ${i%-*} = "pkg" ]; then + tar xf $i -C . -s ",/.*/,,g" "*/pkg-static" + pkg_cmd_add="./pkg-static add" + pkg_cmd_info="./pkg-static info" + fi eval $pkg_cmd_info $base if [ $? -eq 0 ]; then echo "skipping $base, already added" @@ -75,7 +80,6 @@ add_pkg() { del_pkg() { pkgs=$* - cd /tmp/depends # XXX TODO more hard-coding export PKG_PATH=/tmp/depends @@ -109,7 +113,7 @@ del_pkg() { if [ $use_pkgng = "no" ]; then leftover=$(cd /var/db/pkg && find * -type d -maxdepth 1) else - leftover=$(pkg info -qa) + leftover=$(pkg info -qa | grep -vw "${PKGD}") fi echo "deleted list =\""$dellist"\", packages to delete ="\"$delpkg\" echo "The following packages were left behind (perhaps your dependency list is incomplete):" @@ -144,6 +148,7 @@ PD=$4 FD=$5 BD=$6 RD=$7 +PKGD=$8 #export PATH=/ccache/libexec/ccache/:$PATH #export CCACHE_PATH=/usr/bin:/usr/local/bin @@ -159,11 +164,9 @@ restr=$(make -V RESTRICTED) pkg_sufx=${PKG_SUFX} # Use pkgng if available. -if [ -x /usr/sbin/pkg ]; then - use_pkgng="yes" -else - use_pkgng="no" -fi +#if [ -x /usr/sbin/pkg ]; then +use_pkgng="no" +grep -q WITH_PKGNG /etc/make.conf && use_pkgng="yes" # Keep restricted distfiles in a subdirectory for extra protection # against leakage @@ -190,6 +193,7 @@ if [ $phase = 1 ]; then echo "EXTRACT_DEPENDS=${ED}" echo "BUILD_DEPENDS=${BD}" echo "RUN_DEPENDS=${RD}" + echo "PKG_DEPENDS=${PKGD}" echo "prefixes: LOCALBASE=${L}" @@ -213,6 +217,8 @@ if [ $phase = 1 ]; then ./usr/local/etc/gconf ./var/db/fontconfig EOF + add_pkg $PKGD + # Record a "pristine" mtree. mtree -X /tmp/mtree.preexclude -xcn -k uid,gid,mode -p / > /tmp/mtree.pristine @@ -400,15 +406,15 @@ EOF cd /var/db/pkg leftover=$(find * -type d -maxdepth 1 2> /dev/null) else - leftover=$(pkg info -qa) + leftover=$(pkg info -qa | grep -vw "${PKGD}") fi - if [ -n "$leftover" ]; then - echo "leftover packages:" $leftover - del_pkg $leftover - echo "1" > /tmp/status - cleanup 0 - fi +# if [ -n "$leftover" -a "$leftover" != "${PKGD}" ]; then +# echo "leftover packages:" $leftover +# del_pkg $leftover +# echo "1" > /tmp/status +# cleanup 0 +# fi # Compare the state of the filesystem now to clean system (should again be clean) mtree -X /tmp/mtree.preexclude -x -f /tmp/mtree.pristine -p / | egrep -v "^(${L}/var|${L}/lib/X11/xserver/SecurityPolicy|${L}/share/nls/POSIX|${L}/share/nls/en_US.US-ASCII|etc/services|compat |usr/X11R6 |etc/manpath.config|etc/.*.bak|${L}/info/dir|${L}/lib/X11/fonts/.*/fonts\.|usr/local/man/..( |/man. )|${L}/lib/X11/fonts/TrueType )" > /tmp/list3 Modified: projects/portbuild/scripts/pdispatch ============================================================================== --- projects/portbuild/scripts/pdispatch Tue Apr 24 20:27:31 2012 (r234661) +++ projects/portbuild/scripts/pdispatch Tue Apr 24 21:38:41 2012 (r234662) @@ -143,7 +143,7 @@ rm -f ${builddir}/errors/${pkgname}.log if [ ! -f ${builddir}/ptimeout ]; then /usr/bin/gcc -o ${builddir}/ptimeout -Wall ${pbc}/sources/ptimeout.c fi -${builddir}/ptimeout $timeout ${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${command} ${arch} ${branch} ${buildid} ${chroot} ${flags} \"$ED\" \"$PD\" \"$FD\" \"$BD\" \"$RD\" ${args} 2>&1 +${builddir}/ptimeout $timeout ${ssh_cmd} -a -n ${client_user}@${host} ${sudo_cmd} ${command} ${arch} ${branch} ${buildid} ${chroot} ${flags} \"$ED\" \"$PD\" \"$FD\" \"$BD\" \"$RD\" \"$PKGD\" ${args} 2>&1 error=$? # Pull in the results of the build from the client Modified: projects/portbuild/scripts/portbuild ============================================================================== --- projects/portbuild/scripts/portbuild Tue Apr 24 20:27:31 2012 (r234661) +++ projects/portbuild/scripts/portbuild Tue Apr 24 21:38:41 2012 (r234662) @@ -130,6 +130,7 @@ PD=$2 FD=$3 BD=$4 RD=$5 +PKGD=$6 builddir=${pbd}/${arch}/${branch}/builds/${buildid} @@ -141,8 +142,8 @@ unset __MAKE_CONF # set overrides for make.conf export BACKUP_FTP_SITE=${CLIENT_BACKUP_FTP_SITE} -pkgname=$(basename $6 ${pkg_sufx}) -dirname=$7 +pkgname=$(basename $7 ${pkg_sufx}) +dirname=$8 shift 2 echo $pkgname @@ -303,7 +304,7 @@ for i in ${ARCHS_REQUIRING_AOUT_COMPAT}; fi done -set x $ED $FD $PD $BD $RD +set x $ED $FD $PD $BD $RD $PKGD shift 1 while [ $# -gt 0 ]; do # XXX MCL more hard-coding @@ -340,7 +341,7 @@ fi if [ "${error}" = 0 ]; then # phase 1, make checksum # Needs to be chroot not jail so that port can be fetched - chroot ${chroot} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log + chroot ${chroot} /buildscript ${dirname} 1 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" 2>&1 | tee -a ${chroot}/tmp/${pkgname}.log if [ -f ${chroot}/tmp/status ]; then error=$(cat ${chroot}/tmp/status) else @@ -355,10 +356,10 @@ if [ "${error}" = 0 ]; then if [ "${use_jail}" = 1 ]; then ifconfig lo0 alias 127.${ip1}.${ip2}.${ip3}/32 - jail -J ${chroot}/tmp/jail.id ${chroot} jail-${chrootpid} 127.${ip1}.${ip2}.${ip3} /usr/bin/env JAIL_ADDR=127.${ip1}.${ip2}.${ip3} HTTP_PROXY=${http_proxy} /usr/bin/nice -n $nice /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" > ${chroot}/tmp/${pkgname}.log2 2>&1 + jail -J ${chroot}/tmp/jail.id ${chroot} jail-${chrootpid} 127.${ip1}.${ip2}.${ip3} /usr/bin/env JAIL_ADDR=127.${ip1}.${ip2}.${ip3} HTTP_PROXY=${http_proxy} /usr/bin/nice -n $nice /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1 ifconfig lo0 delete 127.${ip1}.${ip2}.${ip3} else - chroot ${chroot} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" > ${chroot}/tmp/${pkgname}.log2 2>&1 + chroot ${chroot} /usr/bin/nice -n ${nice} /buildscript ${dirname} 2 "$ED" "$PD" "$FD" "$BD" "$RD" "$PKGD" > ${chroot}/tmp/${pkgname}.log2 2>&1 fi grep pnohang ${chroot}/tmp/${pkgname}.log2