Skip site navigation (1)Skip section navigation (2)
Date:      Wed, 5 Nov 2014 13:22:19 +0000 (UTC)
From:      Glen Barber <gjb@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-projects@freebsd.org
Subject:   svn commit: r274134 - in projects/release-vmimage/release: . amd64 i386 scripts tools
Message-ID:  <201411051322.sA5DMJHE073744@svn.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: gjb
Date: Wed Nov  5 13:22:19 2014
New Revision: 274134
URL: https://svnweb.freebsd.org/changeset/base/274134

Log:
  Initial rewrite to consolidate VM image build scripts into one.
  There may be some very sharp edges here while refactoring.
  
   - Move amd64/mk-vmimage.sh -> scripts/mk-vmimage.sh.
   - Remove vm-base target from Makefile.vm.
   - In vm-image target, use getopts flags for argument passing.
   - Create tools/vmimage.subr, containing default and prototype
     for the following functions that are used to drive the build,
     run in this order:
  
     vm_install_base()
     vm_extra_install_base()
     vm_extra_install_packages()
     vm_extra_install_ports()
     vm_extra_enable_services()
     vm_extra_pre_umount()
     vm_create_disk()
     vm_extra_create_disk()
  
   - In tools/azure.conf, override:
  
     vm_extra_install_base()
     vm_extra_pre_umount()
     vm_extra_create_disk()
  
   - In tools/openstack.conf, override:
  
     vm_extra_install_base()
     vm_extra_pre_umount()
  
  Sponsored by:	The FreeBSD Foundation

Added:
  projects/release-vmimage/release/scripts/mk-vmimage.sh
     - copied, changed from r274130, projects/release-vmimage/release/amd64/mk-vmimage.sh
  projects/release-vmimage/release/tools/vmimage.subr
Deleted:
  projects/release-vmimage/release/amd64/mk-openstack.sh
  projects/release-vmimage/release/amd64/mk-vmimage.sh
  projects/release-vmimage/release/i386/mk-azure.sh
  projects/release-vmimage/release/i386/mk-openstack.sh
  projects/release-vmimage/release/i386/mk-vmimage.sh
Modified:
  projects/release-vmimage/release/Makefile.vm
  projects/release-vmimage/release/tools/azure.conf
  projects/release-vmimage/release/tools/openstack.conf

Modified: projects/release-vmimage/release/Makefile.vm
==============================================================================
--- projects/release-vmimage/release/Makefile.vm	Wed Nov  5 09:33:02 2014	(r274133)
+++ projects/release-vmimage/release/Makefile.vm	Wed Nov  5 13:22:19 2014	(r274134)
@@ -5,7 +5,7 @@
 # Makefile for building virtual machine and cloud provider disk images.
 #
 
-VMTARGETS=	vm-base vm-image
+VMTARGETS=	vm-image
 VMFORMATS?=	vhd vmdk qcow2 raw
 VMSIZE?=	20G
 VMBASE?=	vm
@@ -37,23 +37,17 @@ CLEANFILES+=	${VMBASE}.${FORMAT}
 . endfor
 .endif
 
-vm-base:
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
-		${VMBASE}.img ${WORLDDIR} ${.OBJDIR}/${.TARGET} ${VMSIZE}
-. endif
-.endif
-	touch ${.TARGET}
+vm-base:	vm-image
 
-vm-image: vm-base
+vm-image:
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 . if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 .  for FORMAT in ${VMFORMATS}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
-		${VMBASE}.img ${FORMAT} ${VMBASE}.${FORMAT}
+		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-i ${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \
+		-S ${WORLDDIR} -o ${VMBASE}.${FORMAT}
 .  endfor
 . endif
 .endif
@@ -62,20 +56,21 @@ vm-image: vm-base
 vm-cloudware:	${CLOUDTARGETS}
 
 vm-azure:
-.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \
-		AZURE_FORMAT=${AZURE_FORMAT} \
-		${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \
-		${WORLDDIR} ${.TARGET} ${VMSIZE} ${AZUREIMAGE}
+.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
+		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-i azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \
+		-S ${WORLDDIR} -o ${AZUREIMAGE} -c ${AZURECONF}
 .endif
 	touch ${.TARGET}
 
 vm-openstack:
-.if exists(${.CURDIR}/${TARGET}/mk-openstack.sh)
+.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		OPENSTACKCONF=${OPENSTACKCONF} \
-		OPENSTACK_FORMAT=${OPENSTACK_FORMAT} \
-		${.CURDIR}/${TARGET}/mk-openstack.sh ${.TARGET} openstack.img \
-		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OPENSTACKIMAGE}
+		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-i openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \
+		-S ${WORLDDIR} -o ${OPENSTACKIMAGE} -c ${OPENSTACKCONF}
 .endif
 	touch ${.TARGET}

Copied and modified: projects/release-vmimage/release/scripts/mk-vmimage.sh (from r274130, projects/release-vmimage/release/amd64/mk-vmimage.sh)
==============================================================================
--- projects/release-vmimage/release/amd64/mk-vmimage.sh	Wed Nov  5 06:42:07 2014	(r274130, copy source)
+++ projects/release-vmimage/release/scripts/mk-vmimage.sh	Wed Nov  5 13:22:19 2014	(r274134)
@@ -32,164 +32,69 @@
 # $FreeBSD$
 #
 
-PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
-export PATH
-
-usage_vm_base() {
-	echo -n "$(basename ${0}) vm-base <base image> <source tree>"
-	echo	" <dest dir> <disk image size>"
-	return 0
-}
-
-usage_vm_image() {
-	echo -n "$(basename ${0}) vm-image <base image> <image format>"
-	echo	" <output image>"
-	return 0
-}
-
-usage() {
-	echo "Usage:"
-	echo "$(basename ${0}) [vm-base|vm-image] [...]"
-	echo
-	usage_vm_base
-	echo
-	usage_vm_image
-	exit 1
-}
-
-panic() {
-	msg="${@}"
-	printf "${msg}\n"
-	if [ ! -z "${mddev}" ]; then
-		mdconfig -d -u ${mddev}
-	fi
-	case ${cmd} in
-		vm-base)
-			# If the vm-base target fails, the vm-image target
-			# cannot possibly succeed.  Touch the .TARGET file
-			# so it is not attempted.
-			touch vm-image
-			;;
-		*)
-			# FALLTHROUGH
-			;;
-	esac
-	# Do not allow one failure case to chain through any remaining image
-	# builds.
-	return 1
-}
-
-vm_create_baseimage() {
-	# Creates the UFS root filesystem for the virtual machine disk,
-	# written to the formatted disk image with mkimg(1).
-	#
-	# Arguments:
-	# vm-base <base image> <source tree> <dest dir> <disk image size>
-
-	VMBASE="${1}"
-	WORLDDIR="${2}"
-	DESTDIR="${3}"
-	VMSIZE="${4}"
-
-	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
-		-o -z "${VMSIZE}" ]; then
-			usage
-	fi
-
-	i=0
-	mkdir -p ${DESTDIR}
-	truncate -s ${VMSIZE} ${VMBASE}
-	mddev=$(mdconfig -f ${VMBASE})
-	newfs -j /dev/${mddev}
-	mount /dev/${mddev} ${DESTDIR}
-	cd ${WORLDDIR} && \
-		make DESTDIR=${DESTDIR} \
-		installworld installkernel distribution || \
-		panic "\n\nCannot install the base system to ${DESTDIR}."
-	chroot ${DESTDIR} /usr/bin/newaliases
-	echo '# Custom /etc/fstab for FreeBSD VM images' \
-		> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
-		>> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
-		>> ${DESTDIR}/etc/fstab
-	sync
-	while ! umount ${DESTDIR}; do
-		i=$(( $i + 1 ))
-		if [ $i -ge 10 ]; then
-			# This should never happen.  But, it has happened.
-			msg="Cannot umount(8) ${DESTDIR}\n"
-			msg="${msg}Something has gone horribly wrong."
-			panic "${msg}"
-		fi
-		sleep 1
+main() {
+	local arg
+	while getopts "C:c:d:f:i:o:s:S:" arg; do
+		case "${arg}" in
+			C)
+				VMBUILDCONF="${OPTARG}"
+				;;
+			c)
+				VMCONFIG="${OPTARG}"
+				;;
+			d)
+				DESTDIR="${OPTARG}"
+				;;
+			f)
+				VMFORMAT="${OPTARG}"
+				;;
+			i)
+				VMBASE="${VMBASE}"
+				;;
+			o)
+				VMIMAGE="${OPTARG}"
+				;;
+			s)
+				VMSIZE="${OPTARG}"
+				;;
+			S)
+				WORLDDIR="${OPTARG}"
+				;;
+			*)
+				;;
+		esac
 	done
+	shift $(( ${OPTIND} - 1))
 
-	return 0
-}
-
-vm_create_vmdisk() {
-	# Creates the virtual machine disk image from the raw disk image.
-	#
-	# Arguments:
-	# vm-image <base image> <image format> <output image>"
-
-	VMBASE="${1}"
-	FORMAT="${2}"
-	VMIMAGE="${3}"
-
-	if [ -z "${VMBASE}" -o -z "${FORMAT}" -o -z "${VMIMAGE}" ]; then
+	if [ -z "${VMBASE}" -o \
+		-z "${WORLDDIR}" -o \
+		-z "${DESTDIR}" -o \
+		-z "${VMSIZE}" -o \
+		-z "${VMIMAGE}" -o \
+		-z "${VMCONFIG}" ];
+	then
 		usage
 	fi
 
-	mkimg_version=$(mkimg --version 2>/dev/null | awk '{print $2}')
-
-	# We need mkimg(1) '--version' output, at minimum, to be able to
-	# tell what virtual machine disk image formats are available.
-	# Bail if mkimg(1) reports an empty '--version' value.
-	if [ -z "${mkimg_version}" ]; then
-		msg="Cannot determine mkimg(1) version.\n"
-		msg="${msg}Cannot continue without a known mkimg(1) version."
-		panic "${msg}"
+	if [ -z "${VMBUILDCONF}" ] || [ ! -e "${VMBUILDCONF}" ]; then
+		echo "Must provide the path to vmimage.subr."
+		return 1
 	fi
 
-	if ! mkimg --formats 2>/dev/null | grep -q ${FORMAT}; then
-		panic "'${FORMAT}' is not supported by this mkimg(1).\n"
-	fi
+	. "${VMBUILDCONF}"
 
-	case ${FORMAT} in
-		vhd)
-			mkimg_format=vhdf
-			;;
-		*)
-			mkimg_format=${FORMAT}
-			;;
-	esac
-
-	mkimg -f ${mkimg_format} -s gpt \
-		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}
-
-	return 0
-}
-
-main() {
-	cmd="${1}"
-	shift 1
+	if [ ! -z "${VMCONFIG}" ] && [ -e "${VMCONFIG}" ]; then
+		. "${VMCONFIG}"
+	fi
 
-	case ${cmd} in
-		vm-base)
-			eval vm_create_baseimage "$@" || return 0
-			;;
-		vm-image)
-			eval vm_create_vmdisk "$@" || return 0
-			;;
-		*|\?)
-			usage
-			;;
-	esac
+	vm_install_base
+	vm_extra_install_base
+	vm_extra_install_packages
+	vm_extra_install_ports
+	vm_extra_enable_services
+	vm_extra_pre_umount
+	vm_create_disk
+	vm_extra_create_disk
 
 	return 0
 }

Modified: projects/release-vmimage/release/tools/azure.conf
==============================================================================
--- projects/release-vmimage/release/tools/azure.conf	Wed Nov  5 09:33:02 2014	(r274133)
+++ projects/release-vmimage/release/tools/azure.conf	Wed Nov  5 13:22:19 2014	(r274134)
@@ -12,3 +12,40 @@ export VM_EXTRA_PACKAGES=
 # Example:
 #export VM_RC_LIST="apache24"
 export VM_RC_LIST=
+
+vm_extra_install_base() {
+	fetch -o ${DESTDIR}/usr/sbin/waagent \
+		http://people.freebsd.org/~gjb/waagent
+	chmod +x ${DESTDIR}/usr/sbin/waagent
+	rm -f ${DESTDIR}/etc/resolv.conf
+
+	return 0
+}
+
+vm_extra_pre_umount() {
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
+		python python2 python27 py27-asn1 sudo bash
+	chroot ${DESTDIR} /usr/sbin/waagent -verbose -install
+	yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision
+	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
+	echo 'ifconfig_hn0="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
+	echo 'waagent_enable="YES"' >> ${DESTDIR}/etc/rc.conf
+	echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf
+	echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf
+
+	return 0
+}
+
+vm_extra_create_disk() {
+	if [ ! -x "/usr/local/bin/qemu-img" ]; then
+		env ASSUME_ALWAYS_YES=yes pkg install -y emulators/qemu-devel
+	fi
+
+	mv ${VMIMAGE} ${VMIMAGE}.raw
+	size=$(qemu-img info -f raw --output json ${VMIMAGE}.raw | awk '/virtual-size/ {print $2}' | tr -d ',')
+	size=$(( ( ${size} / ( 1024 * 1024 ) + 1 ) * ( 1024 * 1024 ) ))
+	qemu-img resize ${VMIMAGE}.raw ${size}
+	qemu-img convert -f raw -o subformat=fixed -O vpc ${VMIMAGE}.raw ${VMIMAGE}
+
+	return 0
+}

Modified: projects/release-vmimage/release/tools/openstack.conf
==============================================================================
--- projects/release-vmimage/release/tools/openstack.conf	Wed Nov  5 09:33:02 2014	(r274133)
+++ projects/release-vmimage/release/tools/openstack.conf	Wed Nov  5 13:22:19 2014	(r274134)
@@ -8,3 +8,19 @@ export VM_EXTRA_PACKAGES="net/cloud-init
 
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="cloudinit"
+
+vm_extra_install_base() {
+	fetch -o ${DESTDIR}/usr/sbin/waagent \
+		http://people.freebsd.org/~gjb/waagent
+	chmod +x ${DESTDIR}/usr/sbin/waagent
+	rm -f ${DESTDIR}/etc/resolv.conf
+
+	return 0
+}
+
+vm_extra_pre_umount() {
+	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
+	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
+
+	return 0
+}

Added: projects/release-vmimage/release/tools/vmimage.subr
==============================================================================
--- /dev/null	00:00:00 1970	(empty, because file is newly added)
+++ projects/release-vmimage/release/tools/vmimage.subr	Wed Nov  5 13:22:19 2014	(r274134)
@@ -0,0 +1,150 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+#
+# Common functions for virtual machine image build scripts.
+#
+
+export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+trap "cleanup" INT QUIT TRAP ABRT TERM
+
+mkimg_bootcode="/boot/pmbr"
+mkimg_partitions="-p freebsd-boot/bootfs:=/boot/gptboot"
+mkimg_partitions="${mkimg_partitions} -p freebsd-swap/swapfs::1G"
+mkimg_partitions="${mkimg_partitions} freebsd-ufs/rootfs:=${VMBASE}"
+
+usage() {
+	echo "${0} usage:"
+	echo "${@}"
+	return 1
+}
+
+err() {
+	printf "${@}\n"
+	cleanup
+	return 1
+}
+
+cleanup() {
+	if [ ! -z "${mddev}" ]; then
+		mdconfig -d -u ${mddev}
+	fi
+	umount ${DESTDIR}/dev
+	umount ${DESTDIR}
+
+	return 0
+}
+
+vm_create_base() {
+	# Creates the UFS root filesystem for the virtual machine disk,
+	# written to the formatted disk image with mkimg(1).
+
+	mkdir -p ${DESTDIR}
+	truncate -s ${VMSIZE} ${VMBASE}
+	mddev=$(mdconfig -f ${VMBASE})
+	newfs -j /dev/${mddev}
+	mount /dev/${mddev} ${DESTDIR}
+
+	return 0
+}
+
+vm_install_base() {
+	# Installs the FreeBSD userland/kernel to the virtual machine disk.
+
+	cd ${WORLDDIR} && \
+		make DESTDIR=${DESTDIR} \
+		installworld installkernel distribution || \
+		err "\n\nCannot install the base system to ${DESTDIR}."
+
+	echo '# Custom /etc/fstab for FreeBSD VM images' \
+		> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/rootfs   /       ufs     rw      1       1' \
+		>> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/swapfs  none    swap    sw      0       0' \
+		>> ${DESTDIR}/etc/fstab
+
+	chroot ${DESTDIR} /usr/bin/newaliases
+	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
+
+	return 0
+}
+
+vm_extra_install_base() {
+	# Prototype.  When overridden, runs extra post-installworld commands
+	# as needed, based on the target virtual machine image or cloud
+	# provider image target.
+
+	return 0
+}
+
+vm_extra_enable_services() {
+	if [ ! -z "${VM_RC_LIST}" ]; then
+		for _rcvar in ${VM_RC_LIST}; do
+			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
+		done
+	fi
+
+	return 0
+}
+
+vm_extra_install_packages() {
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+		/usr/sbin/pkg bootstrap -y
+	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
+		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
+	fi
+
+	return 0
+}
+
+vm_extra_install_ports() {
+	# Prototype.  When overridden, installs additional ports within the
+	# virtual machine environment.
+
+	return 0
+}
+
+vm_umount_base() {
+	i=0
+	sync
+	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
+		i=$(( $i + 1 ))
+		if [ $i -ge 10 ]; then
+			# This should never happen.  But, it has happened.
+			msg="Cannot umount(8) ${DESTDIR}\n"
+			msg="${msg}Something has gone horribly wrong."
+			err "${msg}"
+		fi
+		sleep 1
+	done
+
+	return 0
+}
+
+vm_create_disk() {
+	if [ -z "${mkimg_paritions}" ]; then
+		err "No partition types specified.  Skipping."
+		return 1
+	fi
+	echo "Creating image...  Please wait."
+	echo
+	mkimg -f ${mkimg_format} -s ${mkimg_scheme} \
+		${mkimg_bootcode} \
+		${mkimg_partitions} \
+		${mkimg_outfile}
+
+	mkimg 	-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
+		-p freebsd-swap/swapfs::1G \
+		-p freebsd-ufs/rootfs:=${VMBASE} \
+		-o ${VMIMAGE}.raw
+
+	return 0
+}
+
+vm_extra_create_disk() {
+
+	return 0
+}
+



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201411051322.sA5DMJHE073744>