New method for storing log files.

This is a new method developed to store the log files for builds.  It
doesn't use git anymore, but instead uses a set of sqlite databases +
scripts + xz in order to make the files smaller and more organized.
Also, it is now possible to safely remove old build logs.

lib/fsdb.py

@@ -10,7 +10,7 @@ class SaveGDBResults (steps.MasterShellCommand):
     descriptionDone = 'saved build results'
 
     def __init__ (self, **kwargs):
-        steps.MasterShellCommand.__init__ (self, **kwargs)
+        steps.MasterShellCommand.__init__ (self, command = None, **kwargs)
         self.command = [ os.path.expanduser ("~/scripts/update-logs.sh"),
                          "--commit", util.Property ('got_revision'),
                          "--builder", util.Property ('buildername'),

lib/gdbcommand.py

@@ -1,6 +1,6 @@
 # GDB .sum-fetching command.
 
-from buildbot.process.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION
+from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION
 from buildbot.plugins import steps, util
 from sumfiles import DejaResults, get_web_base
 from gdbgitdb import switch_to_branch
@@ -22,16 +22,27 @@ def create_copy_command (props):
 
     con = sqlite3.connect (db_file)
     c = con.cursor ()
-    c.execute ('SELECT commitid WHERE branch = "%s" AND trysched = 0 FROM logs ORDER BY timestamp DESC LIMIT 1' % branch)
+    c.execute ('SELECT commitid FROM logs WHERE branch = "%s" AND trysched = 0 ORDER BY timestamp DESC LIMIT 1' % branch)
+    comm = c.fetchone ()
     con.close ()
 
-    commit = c.fetchone ()[0]
+    if comm:
-
+        commit = comm[0]
-    from_path = os.path.join (get_web_base (), commit[:2], commit, 'gdb.sum')
-    if istry and istry == 'yes':
-        to_path = os.path.join (get_web_base (), 'try', rev[:2], rev, 'previous_gdb.sum')
     else:
-        to_path = os.path.join (get_web_base (), rev[:2], rev, 'previous_gdb.sum')
+        return [ 'true' ]
+
+    from_path = os.path.join (get_web_base (), builder, commit[:2], commit, 'gdb.sum')
+    if istry and istry == 'yes':
+        to_path = os.path.join (get_web_base (), builder, 'try', rev[:2], rev)
+    else:
+        to_path = os.path.join (get_web_base (), builder, rev[:2], rev)
+
+    if not os.path.exists (to_path):
+        old_umask = os.umask (0022)
+        os.makedirs (to_path)
+        os.umask (old_umask)
+
+    to_path = os.path.join (to_path, 'previous_gdb.sum.xz')
 
     command += [ from_path, to_path ]
 
@@ -79,11 +90,20 @@ class GdbCatSumfileCommand(steps.ShellCommand):
         con = sqlite3.connect (db_file)
         c = con.cursor ()
         c.execute ('SELECT commitid WHERE branch = "%s" AND trysched = 0 FROM logs ORDER BY timestamp DESC LIMIT 1' % branch)
+        prev = c.fetchone ()
         con.close ()
-        prevcommit = c.fetchone ()[0]
 
-        # Switch to the right branch inside the BUILDER repo
+        if prev:
-#        switch_to_branch (builder, branch, force_switch = False)
+            prevcommit = prev[0]
+        else:
+            # This takes care of our very first build.
+            parser.write_sum_file (cur_results, builder, branch, rev)
+            # If there was no previous baseline, then this run
+            # gets the honor.
+            if baseline is None:
+                baseline = cur_results
+            parser.write_baseline (baseline, builder, branch, rev)
+            return SUCCESS
 
         baseline = parser.read_baseline (builder, branch, prevcommit)
         old_sum = parser.read_sum_file (builder, branch, prevcommit)

lib/sumfiles.py

@@ -74,7 +74,9 @@ class DejaResults(object):
 
         bdir = os.path.join (gdb_web_base, builder, rev[:2], rev)
         if not os.path.exists (bdir):
+            old_umask = os.umask (0022)
             os.makedirs (bdir)
+            os.umask (old_umask)
         fname = os.path.join (bdir, filename)
         keys = sum_dict[0].keys ()
         mode = 'w'
@@ -126,10 +128,11 @@ class DejaResults(object):
                     self.parse_sum_line (result, line,
                                          is_racy_file = is_racy_file)
         elif os.path.exists (fname + '.xz'):
-            with lzma.open (fname, 'r') as f:
+            f = lzma.LZMAFile (fname, 'r')
             for line in f:
                 self.parse_sum_line (result, line,
                                      is_racy_file = is_racy_file)
+            f.close ()
         else:
             return None
         return result
@@ -140,8 +143,8 @@ class DejaResults(object):
     def read_baseline(self, builder, branch, rev):
         return self._read_sum_file (builder, branch, rev, 'baseline')
 
-    def read_xfail (self, builder, branch, rev):
+    def read_xfail (self, builder, branch):
-        return self._read_sum_file (builder, branch, rev, 'xfail', is_xfail_file = True)
+        return self._read_sum_file (builder, branch, None, 'xfail', is_xfail_file = True)
 
     def read_old_sum_file (self, builder, branch, rev):
         return self._read_sum_file (builder, branch, rev, 'previous_gdb.sum')

master.cfg

@@ -15,14 +15,13 @@ from buildbot.schedulers.timed import Nightly
 from buildbot.schedulers.trysched import Try_Jobdir
 from buildbot.schedulers.forcesched import ForceScheduler
 from buildbot.process import factory
-from buildbot.plugins import util, reporters, steps, worker
+from buildbot.plugins import util, steps, buildslave
-from buildbot.reporters import utils
 from buildbot.changes.filter import ChangeFilter
-from buildbot.process.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION
+from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION
 from buildbot.interfaces import IEmailLookup
 from zope.interface import implements
 from gdbcommand import CopyOldGDBSumFile, GdbCatSumfileCommand
-from gdbgitdb import SaveGDBResults, get_builder_commit_id
+from fsdb import SaveGDBResults
 from racyanalyze import GDBAnalyzeRacyTests
 from urllib import quote
 
@@ -254,7 +253,8 @@ subsequent commits are made after X, by different people."""
 def MessageGDBTesters (mode, name, build, results, master_status):
     """This function is responsible for composing the message that will be
 send to the gdb-testers mailing list."""
-    res_url = "http://gdb-build.sergiodj.net/cgit"
+    res_url = "http://gdb-build.sergiodj.net/results"
+    sourcestamp = build.getSourceStamps ()[0]
     branch = build.getSourceStamps ()[0].branch
     cur_change = build.getSourceStamps ()[0].changes[0]
     properties = build.getProperties ()
@@ -447,7 +447,7 @@ send to the gdb-testers mailing list."""
 
     # URL to find more info about what went wrong.
     text += "\nTestsuite log (gdb.sum and gdb.log) URL(s):\n"
-    text += "\t<%s/%s/%s/%s/>\n" % (res_url, name, sourcestamp.revision[:2],
+    text += "\t<%s/%s/try/%s/%s/>\n" % (res_url, name, sourcestamp.revision[:2],
                                         sourcestamp.revision)
 
     # commit_id = get_builder_commit_id (name, sourcestamp.revision,
@@ -582,8 +582,7 @@ mn_try = MyMailNotifier(fromaddr = GDB_MAIL_FROM,
                         lookup = LookupEmailTryBuild (),
                         tags = [ "MAIL", "TRY"  ],
                         extraHeaders = { 'X-GDB-Buildbot' : '1',
-                                         'In-Reply-To' : WithProperties ("<%s-try@gdb-build>",
+                                         'In-Reply-To' : util.Interpolate ("<%(prop:got_revision)s-try@gdb-build>")})
-                                                                         'got_revision') })
 
 c['status'].append (mn)
 c['status'].append (mn_try)
@@ -941,8 +940,8 @@ The parameters of the class are:
                                                 descriptionDone = 'analyzed test results',
                                                 doStepIf = scheduler_is_not_racy_do,
                                                 hideStepIf = scheduler_is_racy_hide))
-            self.addStep (steps.FileUpload (workersrc = util.Interpolate ("%(prop:builddir)s/build/gdb/testsuite/gdb.log"),
+            self.addStep (steps.FileUpload (slavesrc = util.Interpolate ("%(prop:builddir)s/build/gdb/testsuite/gdb.log"),
-                                            masterdest = util.Interpolate ("public_html/results/%(prop:buildername)s/tmp/$(prop:got_revision)s/gdb.log"),
+                                            masterdest = util.Interpolate ("~/results/%(prop:buildername)s/tmp/%(prop:got_revision)s/gdb.log"),
                                             mode = 0644,
                                             doStepIf = scheduler_is_not_racy_do,
                                             hideStepIf = True))
@@ -1164,7 +1163,7 @@ def load_config (c):
     config = load (open ("lib/config.json"))
     passwd = load (open ("lib/passwords.json"))
 
-    c['slaves'] = [BuildSlave (slave['name'], passwd[slave['name']],
+    c['slaves'] = [buildslave.BuildSlave (slave['name'], passwd[slave['name']],
                                           max_builds = 1,
                                           notify_on_missing = [ str (slave['admin']) ],
                                           missing_timeout = 300,

scripts/update-logs.sh

@@ -65,7 +65,7 @@ DIR=$BASE_DIR/$BUILDER/
 
 if test ! -d $DIR ; then
     msg "$DIR is not a valid directory.  Creeating it..."
-    mkdir -p $DIR
+    (umask 0022 && mkdir --verbose -p $DIR)
 fi
 
 cd $DIR
@@ -86,7 +86,8 @@ if test "$IS_TRY_SCHED" = "yes" ; then
     ISTRY=1
 fi
 
-if test -d $CDIR ; then
+CDIR_EXISTS=`sqlite3 $DB_NAME "SELECT commitid FROM logs WHERE commitid = '${COMMIT}'"`
+if test -n "$CDIR_EXISTS" ; then
     # If this is a try build, the user is doing a rebuild.
     # If this is a normal build, someone triggered a rebuild.
     # Either way, we need to delete the current log dir.
@@ -94,8 +95,10 @@ if test -d $CDIR ; then
     rm --verbose -rf $CDIR
 fi
 
+if test ! -d $CDIR ; then
     msg "Creating directory structure $CDIR..."
-mkdir --verbose -p $CDIR
+    (umask 0022 && mkdir --verbose -p $CDIR)
+fi
 cd $CDIR
 
 TMP_DIR=$DIR/tmp/$COMMIT/
@@ -104,21 +107,30 @@ msg "Moving log files to $PWD..."
 mv --verbose $TMP_DIR/* .
 rmdir $TMP_DIR
 msg "Compressing log files..."
-xz --verbose --compress *
+find . -type f ! -name "*.xz" | xargs xz --verbose --compress
 
 PREV_COMMIT=`sqlite3 $DB_NAME "SELECT commitid FROM logs WHERE branch = '$BRANCH' AND trysched = 0 ORDER BY timestamp DESC LIMIT 1"`
 
+if test "$IS_TRY_SCHED" != "yes" ; then
+    PREV_2DIG=`echo $PREV_COMMIT | sed 's/^\(..\).*$/\1/'`
+    ln -s $DIR/$PREV_2DIG/$PREV_COMMIT PREVIOUS_COMMIT
+    ln -s $DIR/$CDIR $DIR/$PREV_2DIG/$PREV_COMMIT/NEXT_COMMIT
+fi
+
 msg "Update database..."
 sqlite3 $DB_NAME "INSERT INTO logs(commitid, branch, trysched) VALUES('$COMMIT', '$BRANCH', $ISTRY)"
 
 msg "Creating README.txt..."
 cat > README.txt <<EOF
-== README -- Logs for commit $COMMIT ==
+=== README ===
+
+Logs for: $COMMIT
 
 Branch tested: $BRANCH
 
 Previous commit: $PREV_COMMIT
 
+Patch: <http://sourceware.org/git/?p=binutils-gdb.git;a=commitdiff;h=${COMMIT}>
 EOF
 
 exit 0