From owner-p4-projects@FreeBSD.ORG Mon Aug 20 03:47:05 2007 Return-Path: Delivered-To: p4-projects@freebsd.org Received: by hub.freebsd.org (Postfix, from userid 32767) id 12B9D16A468; Mon, 20 Aug 2007 03:47:05 +0000 (UTC) Delivered-To: perforce@FreeBSD.org Received: from mx1.freebsd.org (mx1.freebsd.org [IPv6:2001:4f8:fff6::34]) by hub.freebsd.org (Postfix) with ESMTP id B4F7516A419 for ; Mon, 20 Aug 2007 03:47:04 +0000 (UTC) (envelope-from ivoras@FreeBSD.org) Received: from repoman.freebsd.org (repoman.freebsd.org [IPv6:2001:4f8:fff6::29]) by mx1.freebsd.org (Postfix) with ESMTP id A29F513C467 for ; Mon, 20 Aug 2007 03:47:04 +0000 (UTC) (envelope-from ivoras@FreeBSD.org) Received: from repoman.freebsd.org (localhost [127.0.0.1]) by repoman.freebsd.org (8.14.1/8.14.1) with ESMTP id l7K3l46k043446 for ; Mon, 20 Aug 2007 03:47:04 GMT (envelope-from ivoras@FreeBSD.org) Received: (from perforce@localhost) by repoman.freebsd.org (8.14.1/8.14.1/Submit) id l7K3l4GN043443 for perforce@freebsd.org; Mon, 20 Aug 2007 03:47:04 GMT (envelope-from ivoras@FreeBSD.org) Date: Mon, 20 Aug 2007 03:47:04 GMT Message-Id: <200708200347.l7K3l4GN043443@repoman.freebsd.org> X-Authentication-Warning: repoman.freebsd.org: perforce set sender to ivoras@FreeBSD.org using -f From: Ivan Voras To: Perforce Change Reviews Cc: Subject: PERFORCE change 125384 for review X-BeenThere: p4-projects@freebsd.org X-Mailman-Version: 2.1.5 Precedence: list List-Id: p4 projects tree changes List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 20 Aug 2007 03:47:05 -0000 http://perforce.freebsd.org/chv.cgi?CH=125384 Change 125384 by ivoras@ivoras_finstall on 2007/08/20 03:46:03 Finished partitioning and image copy / install primitives. Affected files ... .. //depot/projects/soc2007/ivoras_finstall/installer/basewin.py#5 edit .. //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#12 edit .. //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#3 edit .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_1.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_2.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_3.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_4.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/makeimage/makeimage.py#11 edit .. //depot/projects/soc2007/ivoras_finstall/makeimage/maptree.py#1 add .. //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#7 edit .. //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#10 edit Differences ... ==== //depot/projects/soc2007/ivoras_finstall/installer/basewin.py#5 (text+ko) ==== @@ -45,6 +45,7 @@ given label control""" label.set_text(self._get_text(file_name, dir)) label.set_use_markup(True) + label.set_line_wrap(True) def _set_label(self, label, txt): @@ -52,6 +53,7 @@ markup tags""" label.set_text(txt) label.set_use_markup(True) + label.set_line_wrap(True) def _get_text(self, file_name, dir="text"): ==== //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#12 (text+ko) ==== @@ -313,15 +313,11 @@ the following separate file systems: / swap - /usr/local + /usr /var /home and a symlink from /usr/ports to /usr/local/ports. - This is a slight deviation from the way FreeBSD is usually partitioned, but I feel - that having the whole base system on the root partition is a good thing for a novice - user. The only thing not belonging here is the ports tree, which can get huge, so - we'll move it to /usr/local. - We'll also not use the traditional BSD layout of bsdlabels (a=root, b=swap), except + We'll not use the traditional BSD layout of bsdlabels (a=root, b=swap), except for the unfortunately special "c" label. """ @@ -363,7 +359,7 @@ # Calculate the partitioning scheme var_size = min(2048, int(self.trackdata["drive_size"] * 0.1)) - usrlocal_size = min(10240, int(self.trackdata["drive_size"] * 0.3)) + usr_size = min(15*1024, int(self.trackdata["drive_size"] * 0.3)) if fs in ("UFS+SU", "UFS+GJ", "Ext2"): # "Normal" file systems, unix-like @@ -373,53 +369,54 @@ var_fs = fs parts = [] self.trackdata["new_parts"] = parts # a reference + parts.append({ "base" : self.trackdata["drive"], "type" : "fdisk", + "size" : self.trackdata["drive_size"], + "name" : "%ss1" % self.trackdata["drive"], + "mount" : None, + "fs" : None, + "flags" : "init,active,boot" + }) + parts.append({ + "base" : parts[0]["name"], + "type" : "bsdlabel", "size" : 512, - "name" : "%ss1" % self.trackdata["drive"], + "name" : "%sa" % parts[0]["name"], "mount" : "/", "fs" : "UFS+SU", - "flags" : "init,active,boot" + "flags" : "init,boot" }) parts.append({ - "base" : self.trackdata["drive"], - "type" : "fdisk", + "base" : parts[0]["name"], + "type" : "bsdlabel", "size" : physmem, - "name" : "%ss2" % self.trackdata["drive"], + "name" : "%sb" % parts[0]["name"], "mount" : "swap", "fs" : "swap" }) parts.append({ - "base" : self.trackdata["drive"], - "type" : "fdisk", - "size" : self.trackdata["drive_size"] - parts[0]["size"] - parts[1]["size"], # i.e. all the rest - "name" : "%ss3" % self.trackdata["drive"], - "mount" : None, - "fs" : None - }) - parts.append({ - "base" : parts[2]["name"], + "base" : parts[0]["name"], "type" : "bsdlabel", "size" : var_size, - "name" : "%sa" % parts[2]["name"], + "name" : "%sd" % parts[0]["name"], "mount" : "/var", - "fs" : var_fs, - "flags" : "init" + "fs" : var_fs }) parts.append({ - "base" : parts[2]["name"], + "base" : parts[0]["name"], "type" : "bsdlabel", - "size" : usrlocal_size, - "name" : "%sb" % parts[2]["name"], - "mount" : "/usr/local", + "size" : usr_size, + "name" : "%se" % parts[0]["name"], + "mount" : "/usr", "fs" : fs }) parts.append({ - "base" : parts[2]["name"], + "base" : parts[0]["name"], "type" : "bsdlabel", - "size" : parts[2]["size"] - parts[3]["size"] - parts[4]["size"] -2, # TODO: why is the bsdlabel 2679 sectors smaller than it should be? - "name" : "%sd" % parts[2]["name"], + "size" : parts[0]["size"] - parts[1]["size"] - parts[2]["size"] - parts[3]["size"] - parts[4]["size"] - 6, + "name" : "%sf" % parts[0]["name"], "mount" : "/home", "fs" : fs }) @@ -428,28 +425,37 @@ self.trackdata["new_parts"] = parts # The root file system will be UFS, but all the rest we'll set to ZFS, # including the swap. Since we'll boot from the root file system, - # we need to encapsulate ZFS in fdisk partition + # we need to encapsulate ZFS in a fdisk partition. parts.append({ "base" : self.trackdata["drive"], "type" : "fdisk", "size" : 512, "name" : "%ss1" % self.trackdata["drive"], + "mount" : None, + "fs" : None, + "flags" : "init,active,boot" + }) + parts.append({ + "base" : parts[0]["name"], + "type" : "bsdlabel", + "size" : 512, + "name" : "%sa" % parts[0]["name"], "mount" : "/", "fs" : "UFS+SU", - "flags" : "init,active,boot" + "flags" : "init,boot" }) parts.append({ "base" : self.trackdata["drive"], "type" : "fdisk", - "size" : self.trackdata["drive_size"] - parts[0]["size"], + "size" : self.trackdata["drive_size"] - parts[1]["size"], "name" : "%ss2" % self.trackdata["drive"], "mount" : None, "fs" : None }) parts.append({ - "base" : parts[1]["name"], + "base" : parts[2]["name"], "type" : "zpool", - "size" : parts[1]["size"], + "size" : None, "name" : "FreeBSD", "mount" : None, "fs" : None @@ -473,9 +479,9 @@ parts.append({ "base" : parts[2]["name"], "type" : "zfs", - "size" : usrlocal_size, - "name" : "%s/usrlocal" % parts[2]["name"], - "mount" : "/usr/local", + "size" : usrl_size, + "name" : "%s/usr" % parts[2]["name"], + "mount" : "/usr", "fs" : "ZFS" }) parts.append({ @@ -515,30 +521,76 @@ def ninstall_on_load(self): self._load_label(self["label2"], "ninstall.txt") - self.trackdata["install_list"] = "Creating file systems...\n" - self._set_label(self["label3"], self.trackdata["install_list"]) + self.trackdata["install_list"] = [] + self.add_install_list("Creating file systems") + self.trackdata["advert_n"] = 1 + self.next_progress_advert() self.trackdata["part_job"] = self.server.StartPartitionJob(self.trackdata["new_parts"]) gobject.timeout_add(500, self.part_progress) + + def add_install_list(self, msg): + self.trackdata["install_list"].append(msg) + label = "" + for i, s in enumerate(self.trackdata["install_list"]): + if i == len(self.trackdata["install_list"])-1: + label += "+ %s\n" % s + else: + label += "+ %s\n" % s + self._set_label(self["progresstext"], label) + + + def next_progress_advert(self): + filename = "progress_%d.txt" % self.trackdata["advert_n"] + if not os.path.exists("text/%s" % filename): + self.trackdata["advert_n"] = 1 + self.next_progress_advert() + return + self._load_label(self["ads"], filename) + self.trackdata["advert_n"] += 1 + def part_progress(self): try: pcnt = self.server.QueryJobProgress(self.trackdata["part_job"]) except Exception, e: - print "Exception ===================================================================" - print e code, result = self.server.QueryJobError(self.trackdata["part_job"]) print code, result + self._show_message(result[-1000:], "Error during backend function (part_job)") return False self["progressbar"].set_fraction(float(pcnt) / 100) if pcnt == 100: result = self.server.QueryJobResult(self.trackdata["part_job"]) - print result + self.server.DismantleJob(self.trackdata["part_job"]) + del self.trackdata["part_job"] + # Start a new job - base install + self.trackdata["install_job"] = self.server.StartInstallJob(self.trackdata["new_parts"]) + self["progressbar"].set_fraction(0) + self.add_install_list("Installing FreeBSD base system...") + gobject.timeout_add(1000, self.install_progress) + self.next_progress_advert() + return False + return True + + + def install_progress(self): + try: + pcnt = self.server.QueryJobProgress(self.trackdata["install_job"]) + except Exception, e: + code, result = self.server.QueryJobError(self.trackdata["install_job"]) + print code, result + self._show_message(result[-1000:], "Error during backend function (install_job)") + return False + self["progressbar"].set_fraction(float(pcnt) / 100) + if pcnt > 0 and pcnt % 15 == 0: + self.next_progress_advert() + if pcnt == 100: return False return True + my_dir = os.path.split(sys.argv[0])[0] if my_dir != "": # attempt to chdir into the directory where the script is located ==== //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#3 (text+ko) ==== @@ -1,6 +1,6 @@ - + GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK @@ -34,10 +34,12 @@ GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK + 520 True GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK 0 label + True False @@ -48,7 +50,9 @@ True GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK - + + 520 + 80 True GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK 0 @@ -56,6 +60,9 @@ True True + + False + @@ -77,7 +84,18 @@ - + + 520 + True + GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK + 0 + Ads / billboards during progress + True + True + + + 2 + ==== //depot/projects/soc2007/ivoras_finstall/makeimage/makeimage.py#11 (text+ko) ==== @@ -26,6 +26,7 @@ from time import strftime from getopt import getopt, GetoptError from util import nukedir, execute, printmsg, cmdout, readline, initutils, getpkgdeps, getpkgfullname +from maptree import write_maptree class MakeImageException(Exception): pass @@ -177,7 +178,10 @@ execute("make distribution DESTDIR=%s" % DESTDIR) execute("rm %s/boot/kernel/*.symbols" % DESTDIR) os.chdir(DESTDIR) - execute("mtree -c > livecd.mtree") + f = file("maptree.txt", "w") + write_maptree(".", f) + f.close() + execute("mtree -c > base.mtree") os.chdir(STARTDIR) else: if not os.path.exists(DESTDIR) or not os.path.exists("%s/COPYRIGHT" % DESTDIR): ==== //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#7 (text+ko) ==== @@ -29,6 +29,7 @@ cmd_sysctl = "/sbin/sysctl" cmd_geom = "/sbin/geom" cmd_mount = "/sbin/mount" +cmd_umount = "/sbin/umount" cmd_fdisk = "/sbin/fdisk" cmd_bsdlabel = "/sbin/bsdlabel" cmd_file = "/usr/bin/file -s" @@ -37,6 +38,7 @@ cmd_kldstat = "/sbin/kldstat" cmd_kldload = "/sbin/kldload" cmd_mke2fs = "/usr/local/sbin/mke2fs" +cmd_boot0cfg = "/usr/sbin/boot0cfg" file_dmesg = "/var/run/dmesg.boot" @@ -161,7 +163,7 @@ def create_fdisk_partition(dev, index, offset_sectors, size_sectors, flags=[]): - global cmd_fdisk + global cmd_fdisk, cmd_boot0cfg temp_fname, f = make_temp_script() # Create the script f.write("p %d 165 %d %d\n" % (index, offset_sectors, size_sectors)) @@ -179,7 +181,7 @@ return (code, output, None) if 'boot' in flags: # Make the device bootable - code, o = exec_cmd("%s -B %s" % (cmd_fdisk, dev), "y\ny\n") + code, o = exec_cmd("%s -B %s" % (cmd_boot0cfg, dev)) output += o if code != 0: return (code, o, None) @@ -283,6 +285,27 @@ return exec_cmd("%s -J %s.journal" % (cmd_newfs, dev)) +def mount(dev, mount, fs_type, mkdir_mount=True): + global cmd_mount + if not dev.startswith("/dev/"): + dev = "/dev/"+dev + if fs_type in ("UFS", "UFS+SU"): + cmd = "%s %s %s" % (cmd_mount, dev, mount) + elif fs_type == "UFS+GJ": + cmd = "%s -o async %s.journal %s" % (cmd_mount, dev, mount) + elif fs_type == "Ext2": + cmd = "%s -t ext2fs %s %s" % (cmd_mount, dev, mount) + else: + return (0, "Unsupported file system: "+str(fs_type)) + if not os.path.exists(mount) and mkdir_mount: + os.makedirs(mount) + return exec_cmd(cmd) + + +def umount(mount): + global cmd_umount + return exec_cmd("%s %s" % (cmd_umount, mount)) + if __name__ == "__main__": xml = get_geom_xml() for cls in xml["mesh"]["class"]: ==== //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#10 (text+ko) ==== @@ -21,7 +21,7 @@ # SysToolD Engine: implements SysToolD XML-RPC methods -import os, sys +import os, sys, shutil import re import logging, warnings from threading import Thread, Lock @@ -72,7 +72,8 @@ class PartitionJob(SysToolJob): - """A partitioning SysTool job. This one accept a list of partitions + """ + A partitioning SysTool job. This one accept a list of partitions to create and creates them one by one. The list cannot be arbitrary, it is evaluated sequentially. After creation, the partitions are also formatted with the file system specified (if any). @@ -163,6 +164,100 @@ if self.result == None: self.result = buf.getvalue() self.finished = True + del self.part_spec + + +class InstallJob(SysToolJob): + """ + The install job. The job progress is: + - Mount partitions to a temporary tree + - Parse mtree record and use it as a list of files to install + """ + def __init__(self, part_spec): + SysToolJob.__init__(self) + self.part_spec = part_spec + + def run(self): + install_root = "/dist" + if not os.path.exists(install_root): + os.mkdir(install_root) + buf = StringIO() + # mount partitions + for part in self.part_spec: + if part["fs"] == None or part["fs"] == "swap": + continue + if part["mount"] == "/": + mount = install_root + else: + mount = "%s/%s" % (install_root, part["mount"]) + code, result = freebsd.mount(part["name"], mount, part["fs"]) + buf.write(result) + if code != 0: + self.error = code + self.result = buf.getvalue() + self.finished = True + return + maptree_filename = "/maptree.txt" + if not os.path.exists(maptree_filename): + self.error = 1 + self.result = "No "+maptree_filename + self.finished = True + return + maptree_size = os.path.getsize(maptree_filename) + f = file(maptree_filename, "r") + for line in f: + self.percent_complete = self._calc_percent(f.tell(), maptree_size) + rec = line.strip().split("|") + rec_file = rec[-1] + dest_file = "%s/%s" % (install_root, rec_file) + src_file = "/%s" % rec_file + if rec[0] == "F": + if not os.path.exists(src_file): + logging.info("File not found: "+src_file) + continue + shutil.copyfile(src_file, dest_file) + shutil.copystat(src_file, dest_file) + logging.info("copy %s to %s" % (src_file, dest_file)) + elif rec[0] == "D": + if not os.path.exists(dest_file): + os.mkdir(dest_file) + elif rec[0] == "H": + if os.path.exists(dest_file): + os.unlink(dest_file) + try: + os.link("%s/%s" % (install_root, rec[1]), dest_file) + except: + logging.error("Cannot hardlink '/%s' to '%s'" % (rec[1], dest_file)) + try: + shutil.copyfile("/%s" % rec[1], dest_file) + except: + logging.error("Cannot replace hardlink from '/%s' to '%s' with file copy" % (rec[1], dest_file)) + elif rec[0] == "L": + if os.path.exists(dest_file): + if os.path.islink(dest_file): + os.unlink(dest_file) + os.symlink("/%s" % rec[1], dest_file) + else: + os.symlink("/%s" % rec[1], dest_file) + else: + self.error = 1 + self.result = "Unknown record type: "+line + break + for part in reversed(self.part_spec): + if part["fs"] == None or part["fs"] == "swap": + continue + if part["mount"] == "/": + mount = install_root + else: + mount = "%s/%s" % (install_root, part["mount"]) + code, result = freebsd.umount(mount) + if code != 0: + self.error = 1 + if self.result != None: + self.result += "\n"+result + else: + self.result = result + self.finished = True class SysToolEngine: @@ -383,6 +478,20 @@ @logexception + def StartInstallJob(self, part_spec): + """ + Starts the install job. Returns an integer job_id. + """ + self.job_list_lock.acquire() + job = InstallJob(part_spec) + self.job_list.append(job) + job_id = len(self.job_list) + job.start() + self.job_list_lock.release() + return job_id + + + @logexception def QueryJobProgress(self, job_id): """ Queries the progress of a job, returns percent complete or None @@ -424,3 +533,16 @@ return None return (job.error, job.result) + + @logexception + def DismantleJob(self, job_id): + """ + Dismantles a job. After this method is called, job_id is + no longer valid. + """ + self.job_list_lock.acquire() + del self.job_list[job_id-1] + self.job_list_lock.release() + return True + +