From owner-svn-src-projects@FreeBSD.ORG Wed Nov 5 13:22:20 2014 Return-Path: Delivered-To: svn-src-projects@freebsd.org Received: from mx1.freebsd.org (mx1.freebsd.org [8.8.178.115]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by hub.freebsd.org (Postfix) with ESMTPS id D639AC18; Wed, 5 Nov 2014 13:22:20 +0000 (UTC) Received: from svn.freebsd.org (svn.freebsd.org [IPv6:2001:1900:2254:2068::e6a:0]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (Client did not present a certificate) by mx1.freebsd.org (Postfix) with ESMTPS id C15B0916; Wed, 5 Nov 2014 13:22:20 +0000 (UTC) Received: from svn.freebsd.org ([127.0.1.70]) by svn.freebsd.org (8.14.9/8.14.9) with ESMTP id sA5DMKZS073749; Wed, 5 Nov 2014 13:22:20 GMT (envelope-from gjb@FreeBSD.org) Received: (from gjb@localhost) by svn.freebsd.org (8.14.9/8.14.9/Submit) id sA5DMJHE073744; Wed, 5 Nov 2014 13:22:19 GMT (envelope-from gjb@FreeBSD.org) Message-Id: <201411051322.sA5DMJHE073744@svn.freebsd.org> X-Authentication-Warning: svn.freebsd.org: gjb set sender to gjb@FreeBSD.org using -f From: Glen Barber Date: Wed, 5 Nov 2014 13:22:19 +0000 (UTC) To: src-committers@freebsd.org, svn-src-projects@freebsd.org Subject: svn commit: r274134 - in projects/release-vmimage/release: . amd64 i386 scripts tools X-SVN-Group: projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit X-BeenThere: svn-src-projects@freebsd.org X-Mailman-Version: 2.1.18-1 Precedence: list List-Id: "SVN commit messages for the src " projects" tree" List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Wed, 05 Nov 2014 13:22:20 -0000 Author: gjb Date: Wed Nov 5 13:22:19 2014 New Revision: 274134 URL: https://svnweb.freebsd.org/changeset/base/274134 Log: Initial rewrite to consolidate VM image build scripts into one. There may be some very sharp edges here while refactoring. - Move amd64/mk-vmimage.sh -> scripts/mk-vmimage.sh. - Remove vm-base target from Makefile.vm. - In vm-image target, use getopts flags for argument passing. - Create tools/vmimage.subr, containing default and prototype for the following functions that are used to drive the build, run in this order: vm_install_base() vm_extra_install_base() vm_extra_install_packages() vm_extra_install_ports() vm_extra_enable_services() vm_extra_pre_umount() vm_create_disk() vm_extra_create_disk() - In tools/azure.conf, override: vm_extra_install_base() vm_extra_pre_umount() vm_extra_create_disk() - In tools/openstack.conf, override: vm_extra_install_base() vm_extra_pre_umount() Sponsored by: The FreeBSD Foundation Added: projects/release-vmimage/release/scripts/mk-vmimage.sh - copied, changed from r274130, projects/release-vmimage/release/amd64/mk-vmimage.sh projects/release-vmimage/release/tools/vmimage.subr Deleted: projects/release-vmimage/release/amd64/mk-openstack.sh projects/release-vmimage/release/amd64/mk-vmimage.sh projects/release-vmimage/release/i386/mk-azure.sh projects/release-vmimage/release/i386/mk-openstack.sh projects/release-vmimage/release/i386/mk-vmimage.sh Modified: projects/release-vmimage/release/Makefile.vm projects/release-vmimage/release/tools/azure.conf projects/release-vmimage/release/tools/openstack.conf Modified: projects/release-vmimage/release/Makefile.vm ============================================================================== --- projects/release-vmimage/release/Makefile.vm Wed Nov 5 09:33:02 2014 (r274133) +++ projects/release-vmimage/release/Makefile.vm Wed Nov 5 13:22:19 2014 (r274134) @@ -5,7 +5,7 @@ # Makefile for building virtual machine and cloud provider disk images. # -VMTARGETS= vm-base vm-image +VMTARGETS= vm-image VMFORMATS?= vhd vmdk qcow2 raw VMSIZE?= 20G VMBASE?= vm @@ -37,23 +37,17 @@ CLEANFILES+= ${VMBASE}.${FORMAT} . endfor .endif -vm-base: -.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES) -. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh) - env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ - ${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \ - ${VMBASE}.img ${WORLDDIR} ${.OBJDIR}/${.TARGET} ${VMSIZE} -. endif -.endif - touch ${.TARGET} +vm-base: vm-image -vm-image: vm-base +vm-image: .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES) . if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh) . for FORMAT in ${VMFORMATS} env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ - ${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \ - ${VMBASE}.img ${FORMAT} ${VMBASE}.${FORMAT} + ${.CURDIR}/${TARGET}/mk-vmimage.sh \ + -C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \ + -i ${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \ + -S ${WORLDDIR} -o ${VMBASE}.${FORMAT} . endfor . endif .endif @@ -62,20 +56,21 @@ vm-image: vm-base vm-cloudware: ${CLOUDTARGETS} vm-azure: -.if exists(${.CURDIR}/${TARGET}/mk-azure.sh) - env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \ - AZURE_FORMAT=${AZURE_FORMAT} \ - ${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \ - ${WORLDDIR} ${.TARGET} ${VMSIZE} ${AZUREIMAGE} +.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh) + env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ + ${.CURDIR}/${TARGET}/mk-vmimage.sh \ + -C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \ + -i azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \ + -S ${WORLDDIR} -o ${AZUREIMAGE} -c ${AZURECONF} .endif touch ${.TARGET} vm-openstack: -.if exists(${.CURDIR}/${TARGET}/mk-openstack.sh) +.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh) env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ - OPENSTACKCONF=${OPENSTACKCONF} \ - OPENSTACK_FORMAT=${OPENSTACK_FORMAT} \ - ${.CURDIR}/${TARGET}/mk-openstack.sh ${.TARGET} openstack.img \ - ${WORLDDIR} ${.TARGET} ${VMSIZE} ${OPENSTACKIMAGE} + ${.CURDIR}/${TARGET}/mk-vmimage.sh \ + -C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \ + -i openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \ + -S ${WORLDDIR} -o ${OPENSTACKIMAGE} -c ${OPENSTACKCONF} .endif touch ${.TARGET} Copied and modified: projects/release-vmimage/release/scripts/mk-vmimage.sh (from r274130, projects/release-vmimage/release/amd64/mk-vmimage.sh) ============================================================================== --- projects/release-vmimage/release/amd64/mk-vmimage.sh Wed Nov 5 06:42:07 2014 (r274130, copy source) +++ projects/release-vmimage/release/scripts/mk-vmimage.sh Wed Nov 5 13:22:19 2014 (r274134) @@ -32,164 +32,69 @@ # $FreeBSD$ # -PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" -export PATH - -usage_vm_base() { - echo -n "$(basename ${0}) vm-base " - echo " " - return 0 -} - -usage_vm_image() { - echo -n "$(basename ${0}) vm-image " - echo " " - return 0 -} - -usage() { - echo "Usage:" - echo "$(basename ${0}) [vm-base|vm-image] [...]" - echo - usage_vm_base - echo - usage_vm_image - exit 1 -} - -panic() { - msg="${@}" - printf "${msg}\n" - if [ ! -z "${mddev}" ]; then - mdconfig -d -u ${mddev} - fi - case ${cmd} in - vm-base) - # If the vm-base target fails, the vm-image target - # cannot possibly succeed. Touch the .TARGET file - # so it is not attempted. - touch vm-image - ;; - *) - # FALLTHROUGH - ;; - esac - # Do not allow one failure case to chain through any remaining image - # builds. - return 1 -} - -vm_create_baseimage() { - # Creates the UFS root filesystem for the virtual machine disk, - # written to the formatted disk image with mkimg(1). - # - # Arguments: - # vm-base - - VMBASE="${1}" - WORLDDIR="${2}" - DESTDIR="${3}" - VMSIZE="${4}" - - if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \ - -o -z "${VMSIZE}" ]; then - usage - fi - - i=0 - mkdir -p ${DESTDIR} - truncate -s ${VMSIZE} ${VMBASE} - mddev=$(mdconfig -f ${VMBASE}) - newfs -j /dev/${mddev} - mount /dev/${mddev} ${DESTDIR} - cd ${WORLDDIR} && \ - make DESTDIR=${DESTDIR} \ - installworld installkernel distribution || \ - panic "\n\nCannot install the base system to ${DESTDIR}." - chroot ${DESTDIR} /usr/bin/newaliases - echo '# Custom /etc/fstab for FreeBSD VM images' \ - > ${DESTDIR}/etc/fstab - echo '/dev/gpt/rootfs / ufs rw 2 2' \ - >> ${DESTDIR}/etc/fstab - echo '/dev/gpt/swapfs none swap sw 0 0' \ - >> ${DESTDIR}/etc/fstab - sync - while ! umount ${DESTDIR}; do - i=$(( $i + 1 )) - if [ $i -ge 10 ]; then - # This should never happen. But, it has happened. - msg="Cannot umount(8) ${DESTDIR}\n" - msg="${msg}Something has gone horribly wrong." - panic "${msg}" - fi - sleep 1 +main() { + local arg + while getopts "C:c:d:f:i:o:s:S:" arg; do + case "${arg}" in + C) + VMBUILDCONF="${OPTARG}" + ;; + c) + VMCONFIG="${OPTARG}" + ;; + d) + DESTDIR="${OPTARG}" + ;; + f) + VMFORMAT="${OPTARG}" + ;; + i) + VMBASE="${VMBASE}" + ;; + o) + VMIMAGE="${OPTARG}" + ;; + s) + VMSIZE="${OPTARG}" + ;; + S) + WORLDDIR="${OPTARG}" + ;; + *) + ;; + esac done + shift $(( ${OPTIND} - 1)) - return 0 -} - -vm_create_vmdisk() { - # Creates the virtual machine disk image from the raw disk image. - # - # Arguments: - # vm-image " - - VMBASE="${1}" - FORMAT="${2}" - VMIMAGE="${3}" - - if [ -z "${VMBASE}" -o -z "${FORMAT}" -o -z "${VMIMAGE}" ]; then + if [ -z "${VMBASE}" -o \ + -z "${WORLDDIR}" -o \ + -z "${DESTDIR}" -o \ + -z "${VMSIZE}" -o \ + -z "${VMIMAGE}" -o \ + -z "${VMCONFIG}" ]; + then usage fi - mkimg_version=$(mkimg --version 2>/dev/null | awk '{print $2}') - - # We need mkimg(1) '--version' output, at minimum, to be able to - # tell what virtual machine disk image formats are available. - # Bail if mkimg(1) reports an empty '--version' value. - if [ -z "${mkimg_version}" ]; then - msg="Cannot determine mkimg(1) version.\n" - msg="${msg}Cannot continue without a known mkimg(1) version." - panic "${msg}" + if [ -z "${VMBUILDCONF}" ] || [ ! -e "${VMBUILDCONF}" ]; then + echo "Must provide the path to vmimage.subr." + return 1 fi - if ! mkimg --formats 2>/dev/null | grep -q ${FORMAT}; then - panic "'${FORMAT}' is not supported by this mkimg(1).\n" - fi + . "${VMBUILDCONF}" - case ${FORMAT} in - vhd) - mkimg_format=vhdf - ;; - *) - mkimg_format=${FORMAT} - ;; - esac - - mkimg -f ${mkimg_format} -s gpt \ - -b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \ - -p freebsd-swap/swapfs::1G \ - -p freebsd-ufs/rootfs:=${VMBASE} \ - -o ${VMIMAGE} - - return 0 -} - -main() { - cmd="${1}" - shift 1 + if [ ! -z "${VMCONFIG}" ] && [ -e "${VMCONFIG}" ]; then + . "${VMCONFIG}" + fi - case ${cmd} in - vm-base) - eval vm_create_baseimage "$@" || return 0 - ;; - vm-image) - eval vm_create_vmdisk "$@" || return 0 - ;; - *|\?) - usage - ;; - esac + vm_install_base + vm_extra_install_base + vm_extra_install_packages + vm_extra_install_ports + vm_extra_enable_services + vm_extra_pre_umount + vm_create_disk + vm_extra_create_disk return 0 } Modified: projects/release-vmimage/release/tools/azure.conf ============================================================================== --- projects/release-vmimage/release/tools/azure.conf Wed Nov 5 09:33:02 2014 (r274133) +++ projects/release-vmimage/release/tools/azure.conf Wed Nov 5 13:22:19 2014 (r274134) @@ -12,3 +12,40 @@ export VM_EXTRA_PACKAGES= # Example: #export VM_RC_LIST="apache24" export VM_RC_LIST= + +vm_extra_install_base() { + fetch -o ${DESTDIR}/usr/sbin/waagent \ + http://people.freebsd.org/~gjb/waagent + chmod +x ${DESTDIR}/usr/sbin/waagent + rm -f ${DESTDIR}/etc/resolv.conf + + return 0 +} + +vm_extra_pre_umount() { + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \ + python python2 python27 py27-asn1 sudo bash + chroot ${DESTDIR} /usr/sbin/waagent -verbose -install + yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision + echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf + echo 'ifconfig_hn0="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf + echo 'waagent_enable="YES"' >> ${DESTDIR}/etc/rc.conf + echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf + echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf + + return 0 +} + +vm_extra_create_disk() { + if [ ! -x "/usr/local/bin/qemu-img" ]; then + env ASSUME_ALWAYS_YES=yes pkg install -y emulators/qemu-devel + fi + + mv ${VMIMAGE} ${VMIMAGE}.raw + size=$(qemu-img info -f raw --output json ${VMIMAGE}.raw | awk '/virtual-size/ {print $2}' | tr -d ',') + size=$(( ( ${size} / ( 1024 * 1024 ) + 1 ) * ( 1024 * 1024 ) )) + qemu-img resize ${VMIMAGE}.raw ${size} + qemu-img convert -f raw -o subformat=fixed -O vpc ${VMIMAGE}.raw ${VMIMAGE} + + return 0 +} Modified: projects/release-vmimage/release/tools/openstack.conf ============================================================================== --- projects/release-vmimage/release/tools/openstack.conf Wed Nov 5 09:33:02 2014 (r274133) +++ projects/release-vmimage/release/tools/openstack.conf Wed Nov 5 13:22:19 2014 (r274134) @@ -8,3 +8,19 @@ export VM_EXTRA_PACKAGES="net/cloud-init # Set to a list of third-party software to enable in rc.conf(5). export VM_RC_LIST="cloudinit" + +vm_extra_install_base() { + fetch -o ${DESTDIR}/usr/sbin/waagent \ + http://people.freebsd.org/~gjb/waagent + chmod +x ${DESTDIR}/usr/sbin/waagent + rm -f ${DESTDIR}/etc/resolv.conf + + return 0 +} + +vm_extra_pre_umount() { + echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf + echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf + + return 0 +} Added: projects/release-vmimage/release/tools/vmimage.subr ============================================================================== --- /dev/null 00:00:00 1970 (empty, because file is newly added) +++ projects/release-vmimage/release/tools/vmimage.subr Wed Nov 5 13:22:19 2014 (r274134) @@ -0,0 +1,150 @@ +#!/bin/sh +# +# $FreeBSD$ +# +# +# Common functions for virtual machine image build scripts. +# + +export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin" +trap "cleanup" INT QUIT TRAP ABRT TERM + +mkimg_bootcode="/boot/pmbr" +mkimg_partitions="-p freebsd-boot/bootfs:=/boot/gptboot" +mkimg_partitions="${mkimg_partitions} -p freebsd-swap/swapfs::1G" +mkimg_partitions="${mkimg_partitions} freebsd-ufs/rootfs:=${VMBASE}" + +usage() { + echo "${0} usage:" + echo "${@}" + return 1 +} + +err() { + printf "${@}\n" + cleanup + return 1 +} + +cleanup() { + if [ ! -z "${mddev}" ]; then + mdconfig -d -u ${mddev} + fi + umount ${DESTDIR}/dev + umount ${DESTDIR} + + return 0 +} + +vm_create_base() { + # Creates the UFS root filesystem for the virtual machine disk, + # written to the formatted disk image with mkimg(1). + + mkdir -p ${DESTDIR} + truncate -s ${VMSIZE} ${VMBASE} + mddev=$(mdconfig -f ${VMBASE}) + newfs -j /dev/${mddev} + mount /dev/${mddev} ${DESTDIR} + + return 0 +} + +vm_install_base() { + # Installs the FreeBSD userland/kernel to the virtual machine disk. + + cd ${WORLDDIR} && \ + make DESTDIR=${DESTDIR} \ + installworld installkernel distribution || \ + err "\n\nCannot install the base system to ${DESTDIR}." + + echo '# Custom /etc/fstab for FreeBSD VM images' \ + > ${DESTDIR}/etc/fstab + echo '/dev/gpt/rootfs / ufs rw 1 1' \ + >> ${DESTDIR}/etc/fstab + echo '/dev/gpt/swapfs none swap sw 0 0' \ + >> ${DESTDIR}/etc/fstab + + chroot ${DESTDIR} /usr/bin/newaliases + chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart + + return 0 +} + +vm_extra_install_base() { + # Prototype. When overridden, runs extra post-installworld commands + # as needed, based on the target virtual machine image or cloud + # provider image target. + + return 0 +} + +vm_extra_enable_services() { + if [ ! -z "${VM_RC_LIST}" ]; then + for _rcvar in ${VM_RC_LIST}; do + echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf + done + fi + + return 0 +} + +vm_extra_install_packages() { + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \ + /usr/sbin/pkg bootstrap -y + if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then + chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \ + /usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES} + fi + + return 0 +} + +vm_extra_install_ports() { + # Prototype. When overridden, installs additional ports within the + # virtual machine environment. + + return 0 +} + +vm_umount_base() { + i=0 + sync + while ! umount ${DESTDIR}/dev ${DESTDIR}; do + i=$(( $i + 1 )) + if [ $i -ge 10 ]; then + # This should never happen. But, it has happened. + msg="Cannot umount(8) ${DESTDIR}\n" + msg="${msg}Something has gone horribly wrong." + err "${msg}" + fi + sleep 1 + done + + return 0 +} + +vm_create_disk() { + if [ -z "${mkimg_paritions}" ]; then + err "No partition types specified. Skipping." + return 1 + fi + echo "Creating image... Please wait." + echo + mkimg -f ${mkimg_format} -s ${mkimg_scheme} \ + ${mkimg_bootcode} \ + ${mkimg_partitions} \ + ${mkimg_outfile} + + mkimg -b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \ + -p freebsd-swap/swapfs::1G \ + -p freebsd-ufs/rootfs:=${VMBASE} \ + -o ${VMIMAGE}.raw + + return 0 +} + +vm_extra_create_disk() { + + return 0 +} +