Date: Thu, 31 Oct 2013 02:52:10 +0100 From: Matthias Andree <mandree@FreeBSD.org> To: freebsd-ports@freebsd.org Subject: RFT: bsd.stage.mk overhaul v2 Message-ID: <5271B7CA.3000806@FreeBSD.org> In-Reply-To: <5271AF7B.40005@FreeBSD.org> References: <5271AF7B.40005@FreeBSD.org>
next in thread | previous in thread | raw e-mail | index | archive | help
This is a multi-part message in MIME format. --------------090709020900050805090403 Content-Type: text/plain; charset=ISO-8859-15 Content-Transfer-Encoding: 7bit Sorry for following up on myself this quickly, but testing with a really b0rked pkg-plist (news/leafnode), I found a few issues with mtree generation, and found it reasonable to make the pkg-plist parser a bit more conservative. v2 attached and uploaded to <http://people.freebsd.org/~mandree/stage-overhaul-v2.patch>. Diffstat: Scripts/check-stagedir.sh | 113 ++++++++++++++++++++++++++++++++++++ Scripts/qa.sh | 5 - bsd.stage.mk | 103 ++++++-------------------------- 3 files changed, 137 insertions(+), 84 deletions(-) --------------090709020900050805090403 Content-Type: text/x-patch; name="stage-overhaul-v2.patch" Content-Transfer-Encoding: 7bit Content-Disposition: attachment; filename="stage-overhaul-v2.patch" Index: Mk/Scripts/check-stagedir.sh =================================================================== --- Mk/Scripts/check-stagedir.sh (revision 0) +++ Mk/Scripts/check-stagedir.sh (working copy) @@ -0,0 +1,113 @@ +#!/bin/sh +# ports/Mk/Scripts/check-stagedir.sh - called from ports/Mk/bsd.stage.mk + +set -e +export LC_ALL=C + +# lists an mtree file's contents, prefixed to dir. +listmtree() { # mtreefile prefix + { + echo '#mtree' + sed 's/nochange$//;' $1 + } | tar -tf- | sed "s,^,$2/,;s,^$2/\.$,$2,;s,^$,/," +} + +# obtain operating mode from command line +makeplist=0 +case "$1" in + orphans) ;; + makeplist) makeplist=1 ;; + *) echo >&2 "Usage: $0 {orphans|makelist}" ; exit 1 ;; +esac + +# validate environment +envfault=0 +for i in STAGEDIR PREFIX LOCALBASE WRKDIR WRKSRC MTREE_FILE \ + TMPPLIST DATADIR DOCSDIR EXAMPLESDIR +do + if eval test -z "\$$i" ; then + echo >&2 "Environment variable $i undefined. Aborting." + envfault=1 + fi +done +if [ $envfault -ne 0 ] ; then + exit 1 +fi + +set -u + +#### EXPAND TMPPLIST TO ABSOLUTE PATHS, SPLITTING FILES AND DIRS TO +# Use file descriptors 1 and 3 so that the while loop can write +# files to the pipe and dirs to a separate file. +if [ $makeplist = 0 ] ; then + # check for orphans + cwd=${PREFIX} + while read line; do + case $line in + @dirrm*|'@unexec rmdir'*) + line="$(printf %s "$line" \ + | sed -Ee 's/\|\|.*//;s|[0-9]*[[:space:]]*>[&]?[[:space:]]*[^[:space:]]+||g' \ + -e "/^@unexec[[:space:]]+rmdir/s|([^%])%D([^%])|\1${PREFIX}\2|g" \ + -e '/^@unexec[[:space:]]+rmdir/s|"(.*)"[[:space:]]+|\1|g' \ + -e 's/@unexec[[:space:]]+rmdir[[:space:]]+//' \ + -e 's/@dirrm(try)?[[:space:]]+//' \ + -e 's/[[:space:]]+$//')" + case "$line" in + /*) echo >&3 "$line" ;; + *) echo >&3 "$cwd/$line" ;; + esac + ;; + # order matters here - we must check @cwd first because + # otherwise the @cwd* would also match it first, shadowing the + # @cwd) line. + @cwd|@cd) cwd=${PREFIX} ;; + @cwd*|@cd*) set -- $line ; cwd=$2 ;; + @*) ;; + /*) echo "$line" ;; + *) echo "$cwd/$line" ;; + esac + done < ${TMPPLIST} 3>${WRKDIR}/.plist-dirs-unsorted | sort >${WRKDIR}/.plist-files +else + # generate plist - pretend the plist had been empty + : >${WRKDIR}/.plist-dirs-unsorted + : >${WRKDIR}/.plist-files +fi + +### PRODUCE MTREE FILE +{ + listmtree /etc/mtree/BSD.root.dist "" + #listmtree /etc/mtree/BSD.usr.dist /usr + listmtree /etc/mtree/BSD.var.dist /var + + if [ -n "${MTREE_FILE}" ]; then + listmtree "${MTREE_FILE}" "${PREFIX}" + fi + + a=${PREFIX} + while :; do + a=${a%/*} + [ -z "${a}" ] && break + echo ${a} + done +} > ${WRKDIR}/.mtree + +### HANDLE FILES +find ${STAGEDIR} -type f -o -type l | sort | sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files +comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files \ + | sed \ + -e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \ + -e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \ + -e "s,${DATADIR},%%DATADIR%%,g" \ + -e "s,${PREFIX}/,,g" | grep -v "^share/licenses" || [ $? = 1 ] + +### HANDLE DIRS +cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree | sort -u >${WRKDIR}/.traced-dirs +find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | sort >${WRKDIR}/.staged-dirs +comm -13 ${WRKDIR}/.traced-dirs ${WRKDIR}/.staged-dirs \ + | sort -r | sed \ + -e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \ + -e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \ + -e "s,${DATADIR},%%DATADIR%%,g" \ + -e "s,${PREFIX}/,,g" \ + -e 's,^,@dirrmtry ,' \ + -e 's,@dirrmtry \(/.*\),@unexec rmdir >/dev/null 2>\&1 \1 || :,' | grep -v "^@dirrmtry share/licenses" || [ $? = 1 ] Property changes on: Mk/Scripts/check-stagedir.sh ___________________________________________________________________ Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: Mk/Scripts/qa.sh =================================================================== --- Mk/Scripts/qa.sh (revision 332191) +++ Mk/Scripts/qa.sh (working copy) @@ -18,7 +18,7 @@ shebang() { rc=0 for f in `find ${STAGEDIR} -type f`; do - interp=$(sed -n -e '1s/^#![[:space:]]*\([^[:space:]]*\).*/\1/p' $f) + interp=$(sed -n -e '1s/^#![[:space:]]*\([^[:space:]]*\).*/\1/p;2q' $f) case "$interp" in "") ;; /usr/bin/env) ;; @@ -62,7 +62,8 @@ # For now do not raise an error, just warnings stripped() { - [ -x /usr/bin/file ] || return + [ -x /usr/bin/file ] || return # this is fatal + [ -n "${STRIP}" ] || return 0 for f in `find ${STAGEDIR} -type f`; do output=`/usr/bin/file ${f}` case "${output}" in Index: Mk/bsd.stage.mk =================================================================== --- Mk/bsd.stage.mk (revision 332191) +++ Mk/bsd.stage.mk (working copy) @@ -6,10 +6,22 @@ DESTDIRNAME?= DESTDIR MAKE_ARGS+= ${DESTDIRNAME}=${STAGEDIR} -QA_ENV+= STAGEDIR=${STAGEDIR} PREFIX=${PREFIX} \ +QA_ENV+= STAGEDIR=${STAGEDIR} \ + PREFIX=${PREFIX} \ LOCALBASE=${LOCALBASE} \ USESDESKTOPFILEUTILS=${USES:Mdesktop-file-utils} \ - USESSHAREDMIMEINFO=${USES:Mshared-mime-info} + USESSHAREDMIMEINFO=${USES:Mshared-mime-info} \ + "STRIP=${STRIP}" +CO_ENV+= STAGEDIR=${STAGEDIR} \ + PREFIX=${PREFIX} \ + LOCALBASE=${LOCALBASE} \ + WRKDIR=${WRKDIR} \ + WRKSRC=${WRKSRC} \ + MTREE_FILE=${MTREE_FILE} \ + TMPPLIST=${TMPPLIST} \ + DATADIR=${DATADIR} \ + DOCSDIR=${DOCSDIR} \ + EXAMPLESDIR=${EXAMPLESDIR} .if !target(stage-dir) stage-dir: @@ -28,8 +40,8 @@ # Fixes all dead symlinks left by the previous round .if !target(compress-man) compress-man: - @${ECHO_CMD} "====> Compressing man pages" ; \ - mdirs= ; \ + @${ECHO_MSG} "====> Compressing man pages (compress-man)" + @mdirs= ; \ for dir in ${MANDIRS:S/^/${STAGEDIR}/} ; do \ [ -d $$dir ] && mdirs="$$mdirs $$dir" ;\ done ; \ @@ -76,90 +88,17 @@ .if !target(makeplist) makeplist: stage - @if [ -n "${MTREE_FILE}" ]; then \ - { ${ECHO_CMD} "#mtree"; ${CAT} ${MTREE_FILE}; } | ${TAR} tf - | \ - awk '{ sub(/^\.$$/, "", $$1); \ - if ($$1 == "") print "${PREFIX}"; else print "${PREFIX}/"$$1; }' ; \ - fi > ${WRKDIR}/.mtree - @a=${PREFIX}; \ - while :; do \ - a=$${a%/*} ; \ - [ -z "$${a}" ] && break ; \ - ${ECHO_CMD} $${a} >> ${WRKDIR}/.mtree ; \ - done - @${FIND} ${STAGEDIR} -type f -o -type l | ${SORT} | ${SED} -e "s,${STAGEDIR},,g" \ - -e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \ - -e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \ - -e "s,${DATADIR},%%DATADIR%%,g" \ - -e "s,${PREFIX}/,,g" | ${GREP} -v "^share/licenses" || ${TRUE} - @${FIND} ${STAGEDIR} -type d | ${SED} -e "s,${STAGEDIR},,g" \ - | while read line; do \ - ${GREP} -qw "^$${line}$$" ${WRKDIR}/.mtree || { \ - [ -n "$${line}" ] && ${ECHO_CMD} "@dirrmtry $${line}"; \ - }; \ - done | ${SORT} -r | ${SED} \ - -e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \ - -e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \ - -e "s,${DATADIR},%%DATADIR%%,g" \ - -e "s,${PREFIX}/,,g" | ${GREP} -v "^@dirrmtry share/licenses" || ${TRUE} + @${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh makeplist .endif .if !target(check-orphans) check-orphans: stage - @while read line; do \ - cwd=${PREFIX} ; \ - case $$line in \ - @dirrm*) \ - set -- $$line ; \ - case $$2 in \ - /*) ${ECHO_CMD} "dir $$2" ;; \ - *) ${ECHO_CMD} "dir $$cwd/$$2" ;; \ - esac ; \ - ;; \ - @cwd) cwd=${PREFIX} ;; \ - @cwd*) set -- $$line ; \ - cwd=$$2 ;; \ - @*) ;; \ - /*) ${ECHO_CMD} $$line ;; \ - *) ${ECHO_CMD} $$cwd/$$line ;; \ - esac ; \ - done < ${TMPPLIST} > ${WRKDIR}/.expanded-plist - @if [ -n "${MTREE_FILE}" ]; then \ - { ${ECHO_CMD} "#mtree"; ${CAT} ${MTREE_FILE}; } | ${TAR} tf - | \ - awk '{ sub(/^\.$$/, "", $$1); \ - if ($$1 == "") print "${PREFIX}"; else print "${PREFIX}/"$$1; }' ; \ - fi > ${WRKDIR}/.mtree - @a=${PREFIX}; \ - while :; do \ - a=$${a%/*} ; \ - [ -z "$${a}" ] && break ; \ - ${ECHO_CMD} $${a} >> ${WRKDIR}/.mtree ; \ - done - @${FIND} ${STAGEDIR} -type f -o -type l | ${SORT} | ${SED} -e "s,${STAGEDIR},,g" \ - | while read line; do \ - ${GREP} -qw "^$${line}$$" ${WRKDIR}/.expanded-plist || { \ - [ -n "$${line}" ] && ${ECHO_CMD} "$${line}" ; \ - } ; \ - done | ${SED} \ - -e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \ - -e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \ - -e "s,${DATADIR},%%DATADIR%%,g" \ - -e "s,${PREFIX}/,,g" | ${GREP} -v "^share/licenses" || ${TRUE} - @${FIND} ${STAGEDIR} -type d | ${SED} -e "s,${STAGEDIR},,g" \ - | while read line; do \ - ${GREP} -qw "^$${line}$$" ${WRKDIR}/.mtree || \ - ${GREP} -qw "dir\ $${line}$$" ${WRKDIR}/.expanded-plist || { \ - [ -n "$${line}" ] && ${ECHO_CMD} "@dirrmtry $${line}"; \ - } ; \ - done | ${SORT} -r | ${SED} \ - -e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \ - -e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \ - -e "s,${DATADIR},%%DATADIR%%,g" \ - -e "s,${PREFIX}/,,g" | ${GREP} -v "^@dirrmtry share/licenses" || ${TRUE} + @${ECHO_MSG} "====> Items missing from pkg-plist (check-orphans)" + @${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh orphans .endif .if !target(stage-qa) stage-qa: - @${ECHO_CMD} "====> Running Q/A tests" ; \ - ${SETENV} ${QA_ENV} ${SH} ${SCRIPTSDIR}/qa.sh + @${ECHO_MSG} "====> Running Q/A tests (stage-qa)" + @${SETENV} ${QA_ENV} ${SH} ${SCRIPTSDIR}/qa.sh .endif --------------090709020900050805090403--
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?5271B7CA.3000806>