Date: Sun, 19 Aug 2007 03:09:10 GMT From: Ivan Voras <ivoras@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 125330 for review Message-ID: <200708190309.l7J39AEh043092@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=125330 Change 125330 by ivoras@ivoras_finstall on 2007/08/19 03:08:11 Finished framework for asynchronous backend jobs Finished backend partitioning and file system creation primitives Affected files ... .. //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#11 edit .. //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#2 edit .. //depot/projects/soc2007/ivoras_finstall/installer/text/ninstall.txt#1 add .. //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#6 edit .. //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#9 edit Differences ... ==== //depot/projects/soc2007/ivoras_finstall/installer/finstall.py#11 (text+ko) ==== @@ -26,7 +26,7 @@ import logging from types import MethodType from xmlrpclib import ServerProxy -import gtk, gtk.gdk, gtk.glade +import gobject, gtk, gtk.gdk, gtk.glade from basewin import BaseWin from helpdialog import HelpDialog @@ -45,7 +45,7 @@ { "tile" : "nparts" }, { "tile" : "ndefaultfs" }, { "tile" : "nverify" }, - { "tile" : "ninstall" } + { "tile" : "ninstall", "glade" : "installprogress.glade" } ] @@ -79,8 +79,12 @@ def _load_tile(self, tile_name): """Loads a tile by it's name and integrates it in the wizard window""" + glade_name = "glade/%s.glade" % tile_name + for t in self.step_track: + if t["tile"] == tile_name and "glade" in t: + glade_name = "glade/%s" % t["glade"] self._clear_container(self.xml.get_widget("vbox_container")) - self.tile_xml = gtk.glade.XML("glade/%s.glade" % tile_name) + self.tile_xml = gtk.glade.XML(glade_name) self.tile_handlers = self._get_event_handlers(tile_name) self.tile_xml.signal_autoconnect(self.tile_handlers) w = self.tile_xml.get_widget("vbox_container").get_children()[0] @@ -284,7 +288,11 @@ part_list = parts.keys() part_list.sort() for part in part_list: - list.append([part, parts[part]["fs_type"], "%d MB" % parts[part]["mediasize"]]) + if parts[part]["mediasize"] < 2048: + valid_target = "NO" + else: + valid_target = "Yes" + list.append([part, parts[part]["fs_type"], "%d MB" % parts[part]["mediasize"], valid_target]) self["parttree"].set_model(list) self["parttree"].append_column(self._make_column("Partition", 0, True)) self["parttree"].append_column(self._make_column("File system", 1)) @@ -371,7 +379,8 @@ "size" : 512, "name" : "%ss1" % self.trackdata["drive"], "mount" : "/", - "fs" : "UFS+SU" + "fs" : "UFS+SU", + "flags" : "init,active,boot" }) parts.append({ "base" : self.trackdata["drive"], @@ -395,7 +404,8 @@ "size" : var_size, "name" : "%sa" % parts[2]["name"], "mount" : "/var", - "fs" : var_fs + "fs" : var_fs, + "flags" : "init" }) parts.append({ "base" : parts[2]["name"], @@ -408,7 +418,7 @@ parts.append({ "base" : parts[2]["name"], "type" : "bsdlabel", - "size" : parts[2]["size"] - parts[3]["size"] - parts[4]["size"], + "size" : parts[2]["size"] - parts[3]["size"] - parts[4]["size"] -2, # TODO: why is the bsdlabel 2679 sectors smaller than it should be? "name" : "%sd" % parts[2]["name"], "mount" : "/home", "fs" : fs @@ -425,7 +435,8 @@ "size" : 512, "name" : "%ss1" % self.trackdata["drive"], "mount" : "/", - "fs" : "UFS+SU" + "fs" : "UFS+SU", + "flags" : "init,active,boot" }) parts.append({ "base" : self.trackdata["drive"], @@ -495,6 +506,37 @@ return True + def nverify_on_next(self): + self["button_cancel"].set_sensitive(False) + self["button_previous"].set_sensitive(False) + self["button_next"].set_sensitive(False) + return True + + + def ninstall_on_load(self): + self._load_label(self["label2"], "ninstall.txt") + self.trackdata["install_list"] = "Creating file systems...\n" + self._set_label(self["label3"], self.trackdata["install_list"]) + self.trackdata["part_job"] = self.server.StartPartitionJob(self.trackdata["new_parts"]) + gobject.timeout_add(500, self.part_progress) + + + def part_progress(self): + try: + pcnt = self.server.QueryJobProgress(self.trackdata["part_job"]) + except Exception, e: + print "Exception ===================================================================" + print e + code, result = self.server.QueryJobError(self.trackdata["part_job"]) + print code, result + return False + self["progressbar"].set_fraction(float(pcnt) / 100) + if pcnt == 100: + result = self.server.QueryJobResult(self.trackdata["part_job"]) + print result + return False + return True + my_dir = os.path.split(sys.argv[0])[0] ==== //depot/projects/soc2007/ivoras_finstall/installer/glade/installprogress.glade#2 (text+ko) ==== @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <!DOCTYPE glade-interface SYSTEM "glade-2.0.dtd"> -<!--Generated with glade3 3.2.2 on Fri Aug 17 01:01:37 2007 by ivoras@finstall.cosmos--> +<!--Generated with glade3 3.2.2 on Sat Aug 18 15:28:34 2007 by ivoras@finstall.cosmos--> <glade-interface> <widget class="GtkWindow" id="window1"> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> @@ -36,8 +36,12 @@ <widget class="GtkLabel" id="label2"> <property name="visible">True</property> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> + <property name="xalign">0</property> <property name="label" translatable="yes">label</property> </widget> + <packing> + <property name="expand">False</property> + </packing> </child> <child> <widget class="GtkVBox" id="vbox3"> @@ -47,11 +51,14 @@ <widget class="GtkLabel" id="label3"> <property name="visible">True</property> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> + <property name="xalign">0</property> <property name="label" translatable="yes">label</property> + <property name="use_markup">True</property> + <property name="wrap">True</property> </widget> </child> <child> - <widget class="GtkProgressBar" id="progressbar1"> + <widget class="GtkProgressBar" id="progressbar"> <property name="visible">True</property> <property name="events">GDK_POINTER_MOTION_MASK | GDK_POINTER_MOTION_HINT_MASK | GDK_BUTTON_PRESS_MASK | GDK_BUTTON_RELEASE_MASK</property> <property name="text" translatable="yes"></property> ==== //depot/projects/soc2007/ivoras_finstall/pybackend/freebsd.py#6 (text+ko) ==== @@ -21,19 +21,33 @@ # Interface to (most) FreeBSD's low-level utilities -import os, sys +import os, sys, popen2 +import logging import re import xmldict cmd_sysctl = "/sbin/sysctl" cmd_geom = "/sbin/geom" cmd_mount = "/sbin/mount" +cmd_fdisk = "/sbin/fdisk" +cmd_bsdlabel = "/sbin/bsdlabel" cmd_file = "/usr/bin/file -s" +cmd_newfs = "/sbin/newfs" +cmd_gjournal = "/sbin/gjournal" +cmd_kldstat = "/sbin/kldstat" +cmd_kldload = "/sbin/kldload" +cmd_mke2fs = "/usr/local/sbin/mke2fs" file_dmesg = "/var/run/dmesg.boot" +bsdlabel_map = ('a', 'b', 'd', 'e', 'f', 'g', 'h') # maps bsdlabel # to a letter + def get_sysctl(name, b_flag = False): + """ + Returns a sysctl value. If second parameter is True, the sysctl is treated + as binary buffer (or a string) + """ global cmd_sysctl if b_flag: str = os.popen("%s -b %s" % (cmd_sysctl, name)).read().strip() @@ -98,10 +112,181 @@ return m.group(1) +def make_temp_script(): + """Creates a temporary file to hold scripts for various utilities""" + fname = os.tempnam() + f = file(fname, "w+") + return (fname, f) + + +def tolist(e): + if type(e) == type([]): + return e + else: + return [e] + + +def geom_sector_size(dev): + xml = get_geom_xml() + for cls in xml["mesh"]["class"]: + if "geom" in cls: + for geom in tolist(cls["geom"]): + if "provider" in geom: + for provider in tolist(geom["provider"]): + if provider["name"].data == dev and "sectorsize" in provider: + return int(provider["sectorsize"].data) + return None + + +def exec_cmd(cmd, input = None): + """ + Convenience function that executes the command specified by + the cmd argument and returns its exit status and its output from both stdout + and stderr. Optionally, a simple string can be send to the process' + stdin. + """ + logging.info("Executing %s" %cmd) + p = popen2.Popen4(cmd) + if input != None: + p.tochild.write(input) + p.tochild.flush() + buf = "" + while p.poll() == -1: + buf += p.fromchild.read(4096) + buf += p.fromchild.read() + p.fromchild.close() + p.tochild.close() + w = p.wait() + return (os.WEXITSTATUS(w), buf) + + +def create_fdisk_partition(dev, index, offset_sectors, size_sectors, flags=[]): + global cmd_fdisk + temp_fname, f = make_temp_script() + # Create the script + f.write("p %d 165 %d %d\n" % (index, offset_sectors, size_sectors)) + if "active" in flags: + f.write("a %d\n" % index) + f.close() + # Execute the script + if "init" in flags: + cmd = "%s -i -f %s %s" % (cmd_fdisk, temp_fname, dev) + else: + cmd = "%s -f %s %s" % (cmd_fdisk, temp_fname, dev) + code, output = exec_cmd(cmd) + os.unlink(temp_fname) + if code != 0: + return (code, output, None) + if 'boot' in flags: + # Make the device bootable + code, o = exec_cmd("%s -B %s" % (cmd_fdisk, dev), "y\ny\n") + output += o + if code != 0: + return (code, o, None) + # Make fdisk report what it has done + cmd = "%s -p %s" % (cmd_fdisk, dev) + code, dump = exec_cmd(cmd) + if code != 0: + return (code, None, None) + m = re.search(r"p %d .+ (\d+) (\d+)" % index, dump) + if m == None: + return (1, None, None) + p_start = int(m.group(1)) + p_length = int(m.group(2)) + return (0, output, p_start+p_length) + + +def create_bsdlabel_partition(dev, index, offset_sectors, size_sectors, flags=[], fs_type="4.2BSD"): + global cmd_bsdlabel, bsdlabel_map + output = "" + if "init" in flags: + # Initialize the bsdlabel table + code, o = exec_cmd("%s -w %s" % (cmd_bsdlabel, dev)) + output += o + if code != 0: + return (code, output, None) + if "boot" in flags: + # Install boot loader + code, o = exec_cmd("%s -B %s" % (cmd_bsdlabel, dev)) + output += o + if code != 0: + return (code, output, None) + # Read the current label + code, label = exec_cmd("%s %s" % (cmd_bsdlabel, dev)) + if code != 0: + return (code, label, None) + label = label.replace("\t", " ") # get rid of multiple whitespace + while label.find(" ") != -1: + label = label.replace(" ", " ") + label_found = False + lines = [x.strip() for x in label.split("\n")] + for i, line in enumerate(lines): + if line.find(":") == -1: + continue + letter, rest = [x.strip() for x in line.split(":", 1)] + rest = [x.strip() for x in rest.split(" ")] + if bsdlabel_map[index] == letter: # modify the label "in place" + label_found = True + lines[i] = "%s: %d %d %s" % (letter, size_sectors, offset_sectors, fs_type) + if not label_found: + lines.append("%s: %d %d %s" % (bsdlabel_map[index], size_sectors, offset_sectors, fs_type)) + # Create the bsdlabel script and run it through bsdlabel + script_fname, f = make_temp_script() + f.write("\n".join(lines)) + f.write("\n") + f.close() + code, output = exec_cmd("%s -R %s %s" % (cmd_bsdlabel, dev, script_fname)) + #os.unlink(script_fname) + if code != 0: + return (code, output, None) + # Verify the result & fetch the last_offset + code, label = exec_cmd("%s %s" % (cmd_bsdlabel, dev)) + if code != 0: + return (code, label, None) + m = re.search(r"\s+%s:\s+(\d+)\s+(\d+)" % bsdlabel_map[index], label) + if m == None: + return (1, "Cannot parse the label:\n"+label, None) + p_length = int(m.group(1)) + p_offset = int(m.group(2)) + return (0, output, p_offset+p_length) + + +def newfs_simple(dev, fs_type): + """Creates a "simple" file system on the given device""" + global cmd_newfs, cmd_mke2fs + if not dev.startswith("/dev/"): + dev = "/dev/"+dev + if fs_type == "UFS": + cmd = "%s %s" % (cmd_newfs, dev) + elif fs_type == "UFS+SU": + cmd = "%s -U %s" % (cmd_newfs, dev) + elif fs_type == "Ext2": + cmd = "%s %s" % (cmd_mke2fs, dev) + else: + return (1, "Unknown file system type "+fs_type) + return exec_cmd(cmd) + + +def newfs_ufsgj(dev): + """Create a gjournaled UFS on the given device""" + global cmd_newfs, cmd_gjournal, cmd_kldstat, cmd_kldload + if not dev.startswith("/dev/"): + dev = "/dev/"+dev + code, kldlist = exec_cmd(cmd_kldstat) + if kldlist.find("geom_journal") == -1: + code, output = exec_cmd("%s geom_journal" % cmd_kldload) + if code != 0: + return (code, "Cannot kldload geom_journal: "+output) + code, output = exec_cmd("%s label -f %s" % (cmd_gjournal, dev)) + if code != 0: + return (code, "Canno label gjournal: "+output) + return exec_cmd("%s -J %s.journal" % (cmd_newfs, dev)) + + if __name__ == "__main__": xml = get_geom_xml() for cls in xml["mesh"]["class"]: if cls["name"].data == "DISK": for geom in cls["geom"]: - print geom["name"].data, geom["provider"]["mediasize"].data + print geom["name"].data, geom["provider"]["mediasize"].data, geom_sector_size(geom["name"].data) ==== //depot/projects/soc2007/ivoras_finstall/pybackend/systoolengine.py#9 (text+ko) ==== @@ -23,8 +23,9 @@ import os, sys import re -import logging +import logging, warnings from threading import Thread, Lock +from StringIO import StringIO import globals import freebsd @@ -52,6 +53,10 @@ return [e] +class SysToolJobException(Exception): + pass + + class SysToolJob(Thread): """A generic asynchronous SysTool job""" @@ -68,16 +73,98 @@ class PartitionJob(SysToolJob): """A partitioning SysTool job. This one accept a list of partitions - to create and creates them one by one.""" + to create and creates them one by one. The list cannot be arbitrary, + it is evaluated sequentially. After creation, the partitions are also + formatted with the file system specified (if any). + Elements of part_spec list are dictionaries which contain keys: + base - base device on which to create the part. + type - partition type + size - size in MB + name - expected name of the created part. + fs - file system to create on the part. + type can be: fdisk | bsdlabel | zvol | zfs, + and fs can be: UFS+SU | UFS+GJ | Ext2 | ZFS + If error is encountered during any of the operations, the job is + terminated, self.finished, self.error and self.result are set. + """ def __init__(self, part_spec): SysToolJob.__init__(self) self.part_spec = part_spec + def run(self): + # A thread + buf = StringIO() + fdisk_part_nr = 1 # fdisk partitions start from 1 + fdisk_last_offset = 1 # in sectors + bsdlabel_nr = 0 # bsdlabels start from 0 + bsdlabel_last_offset = 1 for i, part in enumerate(self.part_spec): self.percent_complete = self._calc_percent(i, len(self.part_spec)) + if "flags" in part: + flags = [x.strip() for x in part["flags"].split(",")] + else: + flags = [] + if part["type"] == "fdisk": + # Create a fdisk partition + if "init" in flags: + fdisk_part_nr = 1 + fdisk_last_offset = 1 + if fdisk_part_nr > 4: + self.error = 2 + self.result = "Too many fdisk partitions" + break + sector_size = freebsd.geom_sector_size(part["base"]) + sectors_per_mb = (1024*1024) / sector_size + part_size_sectors = sectors_per_mb * part["size"] + code, result, last_offset = freebsd.create_fdisk_partition(part["base"], fdisk_part_nr, fdisk_last_offset, part_size_sectors, flags) + buf.write(result) + if code != 0: # error + self.error = code + self.result = buf.getvalue() + break # Bail out on first error + fdisk_last_offset = last_offset + fdisk_part_nr += 1 + elif part["type"] == "bsdlabel": + # Create a bsdlabel partition + if "init" in flags: + bsdlabel_nr = 0 + bsdlabel_last_offset = 1 + if bsdlabel_nr > 7: + self.error = 2 + self.result = "Too many bsdlabel partitions" + break + sector_size = freebsd.geom_sector_size(part["base"]) + sectors_per_mb = (1024*1024) / sector_size + part_size_sector = sectors_per_mb * part["size"] + code, result, last_offset = freebsd.create_bsdlabel_partition(part["base"], bsdlabel_nr, bsdlabel_last_offset, part_size_sector, flags) + buf.write(result) + if code != 0: + self.error = code + self.result = buf.getvalue() + break + bsdlabel_last_offset = last_offset + bsdlabel_nr += 1 + elif part["type"] == "zpool": + pass + if part["fs"] in ("UFS", "UFS+SU", "Ext2"): + code, result = freebsd.newfs_simple(part["name"], part["fs"]) + buf.write(result) + if code != 0: + self.error = code + break + elif part["fs"] == "UFS+GJ": + code, result = freebsd.newfs_ufsgj(part["name"]) + buf.write(result) + if code != 0: + self.error = code + break + if self.result == None: + self.result = buf.getvalue() + self.finished = True + class SysToolEngine: def __init__(self): @@ -85,6 +172,7 @@ self.root_live = "" # Live file system root (for binaries!) self.job_list = [] self.job_list_lock = Lock() + warnings.simplefilter('ignore', RuntimeWarning) def GetId(self): @@ -281,8 +369,10 @@ @logexception def StartPartitionJob(self, part_spec): - """Starts a job that executes a partition spec list. Returns - an integer job_id""" + """ + Starts a job that executes a partition spec list. Returns + an integer job_id + """ self.job_list_lock.acquire() job = PartitionJob(part_spec) self.job_list.append(job) @@ -294,21 +384,28 @@ @logexception def QueryJobProgress(self, job_id): - """Queries the progress of a job, returns percent complete or None - if the job is in error""" + """ + Queries the progress of a job, returns percent complete or None + if the job is in error + """ self.job_list_lock.acquire() job = self.job_list[job_id-1] self.job_list_lock.release() if job.error != None: - return None - return job.percent_complete + raise SysToolJobException, "Error %d, %s" % (job.error, job.result) + if not job.finished: + return job.percent_complete + else: + return 100 @logexception def QueryJobResult(self, job_id): - """Queries the result of a job, if the job is finished. Returns + """ + Queries the result of a job, if the job is finished. Returns a string with the job's status report or None if the job is not - yet finished.""" + yet finished. + """ self.job_list_lock.acquire() job = self.job_list[job_id-1] self.job_list_lock.release() @@ -316,3 +413,14 @@ return None return job.result + + @logexception + def QueryJobError(self, job_id): + """Queries job error status.""" + self.job_list_lock.acquire() + job = self.job_list[job_id-1] + self.job_list_lock.release() + if job.error == None: + return None + return (job.error, job.result) +
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200708190309.l7J39AEh043092>