Skip site navigation (1)Skip section navigation (2)
Date:      Fri, 16 Feb 2018 17:08:42 +0000 (UTC)
From:      Glen Barber <gjb@FreeBSD.org>
To:        src-committers@freebsd.org, svn-src-user@freebsd.org
Subject:   svn commit: r329385 - user/gjb/thermite
Message-ID:  <201802161708.w1GH8grH006852@repo.freebsd.org>

next in thread | raw e-mail | index | archive | help
Author: gjb
Date: Fri Feb 16 17:08:42 2018
New Revision: 329385
URL: https://svnweb.freebsd.org/changeset/base/329385

Log:
  Incorporate a parallelization option for snapshot/release builds
  when the 'parallel' option is set to 'parallel', which changes the
  invoked build_release() function to parallelbuild_release().
  
  Note, this has significant impact on the machine, most notably load
  averages of 220+ when the -CURRENT builds fire off simultanously.
  
  This is toggleable in main.conf, as it could likely set machines on
  fire.
  
  In my initial tests, build times were reduced as follows:
  
  - 12-CURRENT: ~22 hours -> ~4 hours
  - 11-STABLE: ~10 hours -> ~3 hours
  - 10-STABLE: ~5 hours -> ~2 hours [1]
  
  While here, unset WITH_VMIMAGES when re-sourcing the build config
  file, so it does not pollute arm builds, discovered by accident.
  
  Sponsored by:	The FreeBSD Foundation

Modified:
  user/gjb/thermite/main.conf
  user/gjb/thermite/thermite.sh

Modified: user/gjb/thermite/main.conf
==============================================================================
--- user/gjb/thermite/main.conf	Fri Feb 16 16:56:09 2018	(r329384)
+++ user/gjb/thermite/main.conf	Fri Feb 16 17:08:42 2018	(r329385)
@@ -13,6 +13,10 @@ scriptdir="$(dirname $(realpath ${0}))"
 srcdir="${scriptdir}/../release"
 logdir="${scriptdir}/../logs"
 chroots="${scriptdir}/../chroots"
+## To build all architectures in parallel, set the 'parallel' variable to
+## 'parallel'.  Note, this puts extreme CPU load on the machine.
+#parallel="parallel"
+parallel=""
 
 ftpdir="/snap/ftp"
 
@@ -48,6 +52,7 @@ unset EMBEDDEDBUILD
 unset EMBEDDEDPORTS
 unset EMBEDDED_TARGET
 unset EMBEDDED_TARGET_ARCH
+unset WITH_VMIMAGES
 
 # Check if it is a 'releng' hostname.  Do not set this special
 # variable if it is not.

Modified: user/gjb/thermite/thermite.sh
==============================================================================
--- user/gjb/thermite/thermite.sh	Fri Feb 16 16:56:09 2018	(r329384)
+++ user/gjb/thermite/thermite.sh	Fri Feb 16 17:08:42 2018	(r329385)
@@ -321,6 +321,25 @@ build_release() {
 	unset _build _conf
 }
 
+# Run the release builds in parallel.  CAUSES INSANE CPU LOAD.
+parallelbuild_release() {
+	_build="${rev}-${arch}-${kernel}-${type}"
+	_conf="${scriptdir}/${_build}.conf"
+	source_config || return 0
+	(
+	info "Building release: ${_build}"
+	set >> ${logdir}/${_build}.log
+	env -i __BUILDCONFDIR="${__BUILDCONFDIR}" \
+		/bin/sh ${srcdir}/release.sh -c ${_conf} \
+		>> ${logdir}/${_build}.log 2>&1
+
+	ftp_stage
+	ls -1 ${CHROOTDIR}/R/* >> ${logdir}/${_build}.log
+	send_logmail ${logdir}/${_build}.log ${_build}
+	) &
+	unset _build _conf
+}
+
 # Upload AWS EC2 AMI images.
 upload_ec2_ami() {
 	_build="${rev}-${arch}-${kernel}-${type}"
@@ -524,7 +543,8 @@ main() {
 	zfs_bootstrap
 	runall build_chroots
 	runall install_chroots
-	runall build_release
+	runall ${parallel}build_release
+	wait
 	runall upload_ec2_ami
 	runall upload_gce_image
 	runall upload_vagrant_image



Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201802161708.w1GH8grH006852>