Date: Mon, 20 Aug 2007 03:47:04 GMT From: Ivan Voras <ivoras@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 125384 for review Message-ID: <200708200347.l7K3l4GN043443@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=125384 Change 125384 by ivoras@ivoras_finstall on 2007/08/20 03:46:03 Finished partitioning and image copy / install primitives. Affected files ... .. //depot/projects/soc2007/ivoras_finstall/installer/basewin.py#5 edit .. //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#12 edit .. //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#3 edit .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_1.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_2.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_3.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/installer/text/progress_4.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/makeimage/makeimage.py#11 edit .. //depot/projects/soc2007/ivoras_finstall/makeimage/maptree.py#1 add .. //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#7 edit .. //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#10 edit Differences ... ==== //depot/projects/soc2007/ivoras_finstall/installer/basewin.py#5 (text+ko) ==== @@ -45,6 +45,7 @@ given label control""" label.set_text(self._get_text(file_name, dir)) label.set_use_markup(True) + label.set_line_wrap(True) def _set_label(self, label, txt): @@ -52,6 +53,7 @@ markup tags""" label.set_text(txt) label.set_use_markup(True) + label.set_line_wrap(True) def _get_text(self, file_name, dir="text"): ==== //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#12 (text+ko) ==== @@ -313,15 +313,11 @@ the following separate file systems: / swap - /usr/local + /usr /var /home and a symlink from /usr/ports to /usr/local/ports. - This is a slight deviation from the way FreeBSD is usually partitioned, but I feel - that having the whole base system on the root partition is a good thing for a novice - user. The only thing not belonging here is the ports tree, which can get huge, so - we'll move it to /usr/local. - We'll also not use the traditional BSD layout of bsdlabels (a=root, b=swap), except + We'll not use the traditional BSD layout of bsdlabels (a=root, b=swap), except for the unfortunately special "c" label. """ @@ -363,7 +359,7 @@ # Calculate the partitioning scheme var_size = min(2048, int(self.trackdata["drive_size"] * 0.1)) - usrlocal_size = min(10240, int(self.trackdata["drive_size"] * 0.3)) + usr_size = min(15*1024, int(self.trackdata["drive_size"] * 0.3)) if fs in ("UFS+SU", "UFS+GJ", "Ext2"): # "Normal" file systems, unix-like @@ -373,53 +369,54 @@ var_fs = fs parts = [] self.trackdata["new_parts"] = parts # a reference + parts.append({ "base" : self.trackdata["drive"], "type" : "fdisk", + "size" : self.trackdata["drive_size"], + "name" : "%ss1" % self.trackdata["drive"], + "mount" : None, + "fs" : None, + "flags" : "init,active,boot" + }) + parts.append({ + "base" : parts[0]["name"], + "type" : "bsdlabel", "size" : 512, - "name" : "%ss1" % self.trackdata["drive"], + "name" : "%sa" % parts[0]["name"], "mount" : "/", "fs" : "UFS+SU", - "flags" : "init,active,boot" + "flags" : "init,boot" }) parts.append({ - "base" : self.trackdata["drive"], - "type" : "fdisk", + "base" : parts[0]["name"], + "type" : "bsdlabel", "size" : physmem, - "name" : "%ss2" % self.trackdata["drive"], + "name" : "%sb" % parts[0]["name"], "mount" : "swap", "fs" : "swap" }) parts.append({ - "base" : self.trackdata["drive"], - "type" : "fdisk", - "size" : self.trackdata["drive_size"] - parts[0]["size"] - parts[1]["size"], # i.e. all the rest - "name" : "%ss3" % self.trackdata["drive"], - "mount" : None, - "fs" : None - }) - parts.append({ - "base" : parts[2]["name"], + "base" : parts[0]["name"], "type" : "bsdlabel", "size" : var_size, - "name" : "%sa" % parts[2]["name"], + "name" : "%sd" % parts[0]["name"], "mount" : "/var", - "fs" : var_fs, - "flags" : "init" + "fs" : var_fs }) parts.append({ - "base" : parts[2]["name"], + "base" : parts[0]["name"], "type" : "bsdlabel", - "size" : usrlocal_size, - "name" : "%sb" % parts[2]["name"], - "mount" : "/usr/local", + "size" : usr_size, + "name" : "%se" % parts[0]["name"], + "mount" : "/usr", "fs" : fs }) parts.append({ - "base" : parts[2]["name"], + "base" : parts[0]["name"], "type" : "bsdlabel", - "size" : parts[2]["size"] - parts[3]["size"] - parts[4]["size"] -2, # TODO: why is the bsdlabel 2679 sectors smaller than it should be? - "name" : "%sd" % parts[2]["name"], + "size" : parts[0]["size"] - parts[1]["size"] - parts[2]["size"] - parts[3]["size"] - parts[4]["size"] - 6, + "name" : "%sf" % parts[0]["name"], "mount" : "/home", "fs" : fs }) @@ -428,28 +425,37 @@ self.trackdata["new_parts"] = parts # The root file system will be UFS, but all the rest we'll set to ZFS, # including the swap. Since we'll boot from the root file system, - # we need to encapsulate ZFS in fdisk partition + # we need to encapsulate ZFS in a fdisk partition. parts.append({ "base" : self.trackdata["drive"], "type" : "fdisk", "size" : 512, "name" : "%ss1" % self.trackdata["drive"], + "mount" : None, + "fs" : None, + "flags" : "init,active,boot" + }) + parts.append({ + "base" : parts[0]["name"], + "type" : "bsdlabel", + "size" : 512, + "name" : "%sa" % parts[0]["name"], "mount" : "/", "fs" : "UFS+SU", - "flags" : "init,active,boot" + "flags" : "init,boot" }) parts.append({ "base" : self.trackdata["drive"], "type" : "fdisk", - "size" : self.trackdata["drive_size"] - parts[0]["size"], + "size" : self.trackdata["drive_size"] - parts[1]["size"], "name" : "%ss2" % self.trackdata["drive"], "mount" : None, "fs" : None }) parts.append({ - "base" : parts[1]["name"], + "base" : parts[2]["name"], "type" : "zpool", - "size" : parts[1]["size"], + "size" : None, "name" : "FreeBSD", "mount" : None, "fs" : None @@ -473,9 +479,9 @@ parts.append({ "base" : parts[2]["name"], "type" : "zfs", - "size" : usrlocal_size, - "name" : "%s/usrlocal" % parts[2]["name"], - "mount" : "/usr/local", + "size" : usrl_size, + "name" : "%s/usr" % parts[2]["name"], + "mount" : "/usr", "fs" : "ZFS" }) parts.append({ @@ -515,30 +521,76 @@ def ninstall_on_load(self): self._load_label(self["label2"], "ninstall.txt") - self.trackdata["install_list"] = "Creating file systems...\n" - self._set_label(self["label3"], self.trackdata["install_list"]) + self.trackdata["install_list"] = [] + self.add_install_list("Creating file systems") + self.trackdata["advert_n"] = 1 + self.next_progress_advert() self.trackdata["part_job"] = self.server.StartPartitionJob(self.trackdata["new_parts"]) gobject.timeout_add(500, self.part_progress) + + def add_install_list(self, msg): + self.trackdata["install_list"].append(msg) + label = "" + for i, s in enumerate(self.trackdata["install_list"]): + if i == len(self.trackdata["install_list"])-1: + label += "<b>+ %s</b>\n" % s + else: + label += "+ %s\n" % s + self._set_label(self["progresstext"], label) + + + def next_progress_advert(self): + filename = "progress_%d.txt" % self.trackdata["advert_n"] + if not os.path.exists("text/%s" % filename): + self.trackdata["advert_n"] = 1 + self.next_progress_advert() + return + self._load_label(self["ads"], filename) + self.trackdata["advert_n"] += 1 + def part_progress(self): try: pcnt = self.server.QueryJobProgress(self.trackdata["part_job"]) except Exception, e: - print "Exception ===================================================================" - print e code, result = self.server.QueryJobError(self.trackdata["part_job"]) print code, result + self._show_message(result[-1000:], "Error during backend function (part_job)") return False self["progressbar"].set_fraction(float(pcnt) / 100) if pcnt == 100: result = self.server.QueryJobResult(self.trackdata["part_job"]) - print result + self.server.DismantleJob(self.trackdata["part_job"]) + del self.trackdata["part_job"] + # Start a new job - base install + self.trackdata["install_job"] = self.server.StartInstallJob(self.trackdata["new_parts"]) + self["progressbar"].set_fraction(0) + self.add_install_list("Installing FreeBSD base system...") + gobject.timeout_add(1000, self.install_progress) + self.next_progress_advert() + return False + return True + + + def install_progress(self): + try: + pcnt = self.server.QueryJobProgress(self.trackdata["install_job"]) + except Exception, e: + code, result = self.server.QueryJobError(self.trackdata["install_job"]) + print code, result + self._show_message(result[-1000:], "Error during backend function (install_job)") + return False + self["progressbar"].set_fraction(float(pcnt) / 100) + if pcnt > 0 and pcnt % 15 == 0: + self.next_progress_advert() + if pcnt == 100: return False return True + my_dir = os.path.split(sys.argv[0])[0] if my_dir != "": # attempt to chdir into the directory where the script is located ==== //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#3 (text+ko) ==== @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <!DOCTYPE glade-interface SYSTEM "glade-2.0.dtd"> -<!--Generated with glade3 3.2.2 on Sat Aug 18 15:28:34 2007 by ivoras@finstall.cosmos--> +<!--Generated with glade3 3.2.2 on Sun Aug 19 04:11:55 2007 by ivoras@finstall.cosmos--> <glade-interface> <widget class="GtkWindow" id="window1"> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> @@ -34,10 +34,12 @@ <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> <child> <widget class="GtkLabel" id="label2"> + <property name="width_request">520</property> <property name="visible">True</property> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> <property name="xalign">0</property> <property name="label" translatable="yes">label</property> + <property name="wrap">True</property> </widget> <packing> <property name="expand">False</property> @@ -48,7 +50,9 @@ <property name="visible">True</property> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> <child> - <widget class="GtkLabel" id="label3"> + <widget class="GtkLabel" id="progresstext"> + <property name="width_request">520</property> + <property name="height_request">80</property> <property name="visible">True</property> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> <property name="xalign">0</property> @@ -56,6 +60,9 @@ <property name="use_markup">True</property> <property name="wrap">True</property> </widget> + <packing> + <property name="expand">False</property> + </packing> </child> <child> <widget class="GtkProgressBar" id="progressbar"> @@ -77,7 +84,18 @@ </packing> </child> <child> - <placeholder/> + <widget class="GtkLabel" id="ads"> + <property name="width_request">520</property> + <property name="visible">True</property> + <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> + <property name="xalign">0</property> + <property name="label" translatable="yes">Ads / billboards during progress</property> + <property name="use_markup">True</property> + <property name="wrap">True</property> + </widget> + <packing> + <property name="position">2</property> + </packing> </child> </widget> <packing> ==== //depot/projects/soc2007/ivoras_finstall/makeimage/makeimage.py#11 (text+ko) ==== @@ -26,6 +26,7 @@ from time import strftime from getopt import getopt, GetoptError from util import nukedir, execute, printmsg, cmdout, readline, initutils, getpkgdeps, getpkgfullname +from maptree import write_maptree class MakeImageException(Exception): pass @@ -177,7 +178,10 @@ execute("make distribution DESTDIR=%s" % DESTDIR) execute("rm %s/boot/kernel/*.symbols" % DESTDIR) os.chdir(DESTDIR) - execute("mtree -c > livecd.mtree") + f = file("maptree.txt", "w") + write_maptree(".", f) + f.close() + execute("mtree -c > base.mtree") os.chdir(STARTDIR) else: if not os.path.exists(DESTDIR) or not os.path.exists("%s/COPYRIGHT" % DESTDIR): ==== //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#7 (text+ko) ==== @@ -29,6 +29,7 @@ cmd_sysctl = "/sbin/sysctl" cmd_geom = "/sbin/geom" cmd_mount = "/sbin/mount" +cmd_umount = "/sbin/umount" cmd_fdisk = "/sbin/fdisk" cmd_bsdlabel = "/sbin/bsdlabel" cmd_file = "/usr/bin/file -s" @@ -37,6 +38,7 @@ cmd_kldstat = "/sbin/kldstat" cmd_kldload = "/sbin/kldload" cmd_mke2fs = "/usr/local/sbin/mke2fs" +cmd_boot0cfg = "/usr/sbin/boot0cfg" file_dmesg = "/var/run/dmesg.boot" @@ -161,7 +163,7 @@ def create_fdisk_partition(dev, index, offset_sectors, size_sectors, flags=[]): - global cmd_fdisk + global cmd_fdisk, cmd_boot0cfg temp_fname, f = make_temp_script() # Create the script f.write("p %d 165 %d %d\n" % (index, offset_sectors, size_sectors)) @@ -179,7 +181,7 @@ return (code, output, None) if 'boot' in flags: # Make the device bootable - code, o = exec_cmd("%s -B %s" % (cmd_fdisk, dev), "y\ny\n") + code, o = exec_cmd("%s -B %s" % (cmd_boot0cfg, dev)) output += o if code != 0: return (code, o, None) @@ -283,6 +285,27 @@ return exec_cmd("%s -J %s.journal" % (cmd_newfs, dev)) +def mount(dev, mount, fs_type, mkdir_mount=True): + global cmd_mount + if not dev.startswith("/dev/"): + dev = "/dev/"+dev + if fs_type in ("UFS", "UFS+SU"): + cmd = "%s %s %s" % (cmd_mount, dev, mount) + elif fs_type == "UFS+GJ": + cmd = "%s -o async %s.journal %s" % (cmd_mount, dev, mount) + elif fs_type == "Ext2": + cmd = "%s -t ext2fs %s %s" % (cmd_mount, dev, mount) + else: + return (0, "Unsupported file system: "+str(fs_type)) + if not os.path.exists(mount) and mkdir_mount: + os.makedirs(mount) + return exec_cmd(cmd) + + +def umount(mount): + global cmd_umount + return exec_cmd("%s %s" % (cmd_umount, mount)) + if __name__ == "__main__": xml = get_geom_xml() for cls in xml["mesh"]["class"]: ==== //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#10 (text+ko) ==== @@ -21,7 +21,7 @@ # SysToolD Engine: implements SysToolD XML-RPC methods -import os, sys +import os, sys, shutil import re import logging, warnings from threading import Thread, Lock @@ -72,7 +72,8 @@ class PartitionJob(SysToolJob): - """A partitioning SysTool job. This one accept a list of partitions + """ + A partitioning SysTool job. This one accept a list of partitions to create and creates them one by one. The list cannot be arbitrary, it is evaluated sequentially. After creation, the partitions are also formatted with the file system specified (if any). @@ -163,6 +164,100 @@ if self.result == None: self.result = buf.getvalue() self.finished = True + del self.part_spec + + +class InstallJob(SysToolJob): + """ + The install job. The job progress is: + - Mount partitions to a temporary tree + - Parse mtree record and use it as a list of files to install + """ + def __init__(self, part_spec): + SysToolJob.__init__(self) + self.part_spec = part_spec + + def run(self): + install_root = "/dist" + if not os.path.exists(install_root): + os.mkdir(install_root) + buf = StringIO() + # mount partitions + for part in self.part_spec: + if part["fs"] == None or part["fs"] == "swap": + continue + if part["mount"] == "/": + mount = install_root + else: + mount = "%s/%s" % (install_root, part["mount"]) + code, result = freebsd.mount(part["name"], mount, part["fs"]) + buf.write(result) + if code != 0: + self.error = code + self.result = buf.getvalue() + self.finished = True + return + maptree_filename = "/maptree.txt" + if not os.path.exists(maptree_filename): + self.error = 1 + self.result = "No "+maptree_filename + self.finished = True + return + maptree_size = os.path.getsize(maptree_filename) + f = file(maptree_filename, "r") + for line in f: + self.percent_complete = self._calc_percent(f.tell(), maptree_size) + rec = line.strip().split("|") + rec_file = rec[-1] + dest_file = "%s/%s" % (install_root, rec_file) + src_file = "/%s" % rec_file + if rec[0] == "F": + if not os.path.exists(src_file): + logging.info("File not found: "+src_file) + continue + shutil.copyfile(src_file, dest_file) + shutil.copystat(src_file, dest_file) + logging.info("copy %s to %s" % (src_file, dest_file)) + elif rec[0] == "D": + if not os.path.exists(dest_file): + os.mkdir(dest_file) + elif rec[0] == "H": + if os.path.exists(dest_file): + os.unlink(dest_file) + try: + os.link("%s/%s" % (install_root, rec[1]), dest_file) + except: + logging.error("Cannot hardlink '/%s' to '%s'" % (rec[1], dest_file)) + try: + shutil.copyfile("/%s" % rec[1], dest_file) + except: + logging.error("Cannot replace hardlink from '/%s' to '%s' with file copy" % (rec[1], dest_file)) + elif rec[0] == "L": + if os.path.exists(dest_file): + if os.path.islink(dest_file): + os.unlink(dest_file) + os.symlink("/%s" % rec[1], dest_file) + else: + os.symlink("/%s" % rec[1], dest_file) + else: + self.error = 1 + self.result = "Unknown record type: "+line + break + for part in reversed(self.part_spec): + if part["fs"] == None or part["fs"] == "swap": + continue + if part["mount"] == "/": + mount = install_root + else: + mount = "%s/%s" % (install_root, part["mount"]) + code, result = freebsd.umount(mount) + if code != 0: + self.error = 1 + if self.result != None: + self.result += "\n"+result + else: + self.result = result + self.finished = True class SysToolEngine: @@ -383,6 +478,20 @@ @logexception + def StartInstallJob(self, part_spec): + """ + Starts the install job. Returns an integer job_id. + """ + self.job_list_lock.acquire() + job = InstallJob(part_spec) + self.job_list.append(job) + job_id = len(self.job_list) + job.start() + self.job_list_lock.release() + return job_id + + + @logexception def QueryJobProgress(self, job_id): """ Queries the progress of a job, returns percent complete or None @@ -424,3 +533,16 @@ return None return (job.error, job.result) + + @logexception + def DismantleJob(self, job_id): + """ + Dismantles a job. After this method is called, job_id is + no longer valid. + """ + self.job_list_lock.acquire() + del self.job_list[job_id-1] + self.job_list_lock.release() + return True + +
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200708200347.l7K3l4GN043443>