Implement multi-branch testing

lib/config.json

@@ -109,7 +109,7 @@
 		 ],
 
     "schedulers" : [ { "type" : "AnyBranchScheduler", "name" : "master",
-		       "change_filter" : "master_filter",
+		       "change_filter" : "all_gdb_filter",
 		       "builderNames" : [ "Fedora-x86_64-m64",
 					  "Fedora-x86_64-m32",
 					  "Fedora-x86_64-native-gdbserver-m64",

lib/gdbcommand.py

@@ -3,6 +3,7 @@
 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
 from buildbot.steps.shell import ShellCommand
 from sumfiles import DejaResults
+from gdbgitdb import switch_to_branch
 
 class GdbCatSumfileCommand(ShellCommand):
     name = 'regressions'
@@ -18,6 +19,10 @@ class GdbCatSumfileCommand(ShellCommand):
         branch = self.getProperty('branch')
         if branch is None:
             branch = 'master'
+
+        # Switch to the right branch inside the BUILDER repo
+        switch_to_branch (builder, branch)
+
         parser = DejaResults()
         cur_results = parser.read_sum_text(self.getLog('stdio').getText())
         if not istry or istry == 'no':
@@ -26,15 +31,16 @@ class GdbCatSumfileCommand(ShellCommand):
             baseline = parser.read_sum_file(builder, rev)
         result = SUCCESS
         if baseline is not None:
-            report = parser.compute_regressions(builder, cur_results, baseline)
+            report = parser.compute_regressions (builder, branch,
+                                                 cur_results, baseline)
             if report is not '':
-                self.addCompleteLog('regressions', report)
+                self.addCompleteLog ('regressions', report)
                 result = FAILURE
         if not istry or istry == 'no':
-            parser.write_sum_file(cur_results, builder, rev)
+            parser.write_sum_file (cur_results, builder, branch)
             # If there was no previous baseline, then this run
             # gets the honor.
             if baseline is None:
                 baseline = cur_results
-            parser.write_baseline(baseline, builder, branch)
+            parser.write_baseline (baseline, builder, branch)
         return result

lib/gdbgitdb.py

@@ -23,6 +23,27 @@ log files of the COMMIT that was tested."""
 
     return None
 
+def switch_to_branch (builder, branch):
+    """Switch (or create) to BRANCH on BUILDER repo."""
+    repodir = os.path.join (get_web_base (), builder)
+    repo = git.Repo.init (path = repodir)
+
+    if 'master' not in repo.heads:
+        with open (os.path.join (repodir, 'README'), 'w') as f:
+            f.write ("git repo for GDB test results")
+        with open (os.path.join (repodir, '.gitignore'), 'w') as f:
+            f.write ("*xfails*\n")
+        repo.index.add (['README', '.gitignore'])
+        repo.index.commit ('Initial commit')
+        repo.index.write ()
+
+    if branch not in repo.heads:
+        myhead = repo.create_head (branch)
+    else:
+        myhead = repo.heads[branch]
+
+    myhead.checkout ()
+
 class SaveGDBResults (ShellCommand):
     name = 'save build results'
     description = 'saving build results'
@@ -40,7 +61,7 @@ class SaveGDBResults (ShellCommand):
         repodir = get_web_base ()
         builder_dir = os.path.join (repodir, builder)
         # TODO: Include timestamp in the tag name?
-        full_tag = "%s-%s" % (builder, rev)
+        full_tag = "%s-%s-%s" % (builder, rev, branch)
 
         if branch is None:
             branch = 'master'
@@ -55,7 +76,9 @@ class SaveGDBResults (ShellCommand):
         if 'master' not in repo.heads:
             with open (os.path.join (repodir, 'README'), 'w') as f:
                 f.write ("git repo for GDB test results")
-            repo.index.add (['README'])
+            with open (os.path.join (repodir, '.gitignore'), 'w') as f:
+                f.write ("*xfail*\n")
+            repo.index.add (['README', '.gitignore'])
             repo.index.commit ('Initial commit')
             repo.index.write ()
 
@@ -68,9 +91,9 @@ class SaveGDBResults (ShellCommand):
             myhead.checkout ()
             repo.index.add (['%s/gdb.sum' % builder,
                              '%s/gdb.log' % builder,
-                             '%s/%s/baseline' % (builder, branch)])
+                             '%s/baseline' % builder])
             if repo.is_dirty ():
-                repo.index.commit ('Log files for %s' % full_tag)
+                repo.index.commit ('Log files for %s -- branch %s' % (full_tag, branch))
                 repo.index.write ()
             repo.create_tag (full_tag)
         return SUCCESS
@@ -81,7 +104,7 @@ class SaveGDBResults (ShellCommand):
         istry = self.getProperty ('isTryBuilder')
         branch = self.getProperty ('branch')
         repodir = os.path.join (get_web_base (), builder)
-        full_tag = "%s-%s" % (datetime.now ().strftime ("%Y%m%d-%H%M%S"), rev)
+        full_tag = "%s-%s-%s" % (datetime.now ().strftime ("%Y%m%d-%H%M%S"), rev, branch)
 
         if branch is None:
             branch = 'master'
@@ -94,20 +117,33 @@ class SaveGDBResults (ShellCommand):
         if 'master' not in repo.heads:
             with open (os.path.join (repodir, 'README'), 'w') as f:
                 f.write ("git repo for GDB test results -- %s" % builder)
-            repo.index.add (['README'])
+            with open (os.path.join (repodir, '.gitignore'), 'w') as f:
+                f.write ("*xfail*\n")
+            repo.index.add (['README', '.gitignore'])
             repo.index.commit ('Initial commit')
             repo.index.write ()
 
+        if branch not in repo.heads:
+            myhead = repo.create_head (branch)
+        else:
+            myhead = repo.heads[branch]
+
+        myhead.checkout ()
         if full_tag not in repo.tags:
             repo.index.add (['gdb.sum',
                              'gdb.log',
-                             '%s/baseline' % branch])
+                             'baseline'])
             if repo.is_dirty ():
-                repo.index.commit ('Log files for %s' % full_tag)
+                repo.index.commit ('Log files for %s -- branch %s' % (full_tag, branch))
                 repo.index.write ()
             repo.create_tag (full_tag)
+            # Returning the HEAD to master
+            repo.heads['master'].checkout ()
         return SUCCESS
 
     def evaluateCommand (self, cmd):
         # We can change this scheme for the other one if needed
+
+        # FIXME: the _evaluateCommand_builder_branch function needs
+        # adjustment because of the multi-branch testing...
         return self._evaluateCommand_single_repo (cmd)

lib/sumfiles.py

@@ -46,29 +46,26 @@ class DejaResults(object):
                 test_name = nname
             out_dict[test_name] = result
 
-    def _write_sum_file(self, sum_dict, subdir, revision, filename):
+    def _write_sum_file(self, sum_dict, subdir, rev_or_branch, filename):
         global gdb_web_base
-        if revision:
+        if not rev_or_branch:
-            bdir = os.path.join(gdb_web_base, subdir, revision)
-        else:
             bdir = os.path.join (gdb_web_base, subdir)
-        if not os.path.isdir(bdir):
+        else:
-            os.makedirs(bdir, 0755)
+            bdir = os.path.join (gdb_web_base, subdir, rev_or_branch)
-        fname = os.path.join(bdir, filename)
+        if not os.path.isdir (bdir):
-        keys = sum_dict.keys()
+            os.makedirs (bdir, 0755)
-        keys.sort()
+        fname = os.path.join (bdir, filename)
-        f = open(fname, 'w')
+        keys = sum_dict.keys ()
+        keys.sort ()
+        with open (fname, 'w') as f:
             for k in keys:
-            f.write(sum_dict[k] + ': ' + k + '\n')
+                f.write (sum_dict[k] + ': ' + k + '\n')
-        f.close()
 
-    def write_sum_file(self, sum_dict, builder, revision):
+    def write_sum_file(self, sum_dict, builder, branch):
         self._write_sum_file (sum_dict, builder, None, 'gdb.sum')
-#        self._write_sum_file(sum_dict, builder, revision, 'gdb.sum')
 
     def write_baseline(self, sum_dict, builder, branch):
-        self._write_sum_file(sum_dict, os.path.join(builder, branch),
+        self._write_sum_file(sum_dict, builder, None, 'baseline')
-                             None, 'baseline')
 
     # Read a .sum file.
     # The builder name is BUILDER.
@@ -76,50 +73,48 @@ class DejaResults(object):
     # revision; to read the baseline file for a branch, use `read_baseline'.
     # Returns a dictionary holding the .sum contents, or None if the
     # file did not exist.
-    def _read_sum_file(self, subdir, revision, filename):
+    def _read_sum_file(self, subdir, rev_or_branch, filename):
         global gdb_web_base
-        if not revision:
+        if not rev_or_branch:
-            fname = os.path.join(gdb_web_base, subdir, filename)
+            fname = os.path.join (gdb_web_base, subdir, filename)
         else:
-            fname = os.path.join (gdb_web_base, subdir, revision, filename)
+            fname = os.path.join (gdb_web_base, subdir, rev_or_branch, filename)
-        if os.path.exists(fname):
+        if os.path.exists (fname):
             result = {}
-            f = open(fname, 'r')
+            with open (fname, 'r') as f:
                 for line in f:
                     self.parse_sum_line (result, line)
-            f.close()
         else:
             result = None
         return result
 
-    def read_sum_file (self, builder, revision):
+    def read_sum_file (self, builder, branch):
         return self._read_sum_file (builder, None, 'gdb.sum')
-#        return self._read_sum_file (builder, revision, 'gdb.sum')
 
     def read_baseline(self, builder, branch):
-        return self._read_sum_file(os.path.join(builder, branch),
+        return self._read_sum_file (builder, None, 'baseline')
-                                   None, 'baseline')
 
-    def read_xfail (self, builder):
+    def read_xfail (self, builder, branch):
-        return self._read_sum_file (builder, None, 'xfail')
+        return self._read_sum_file (builder, os.path.join ('xfails', branch),
+                                    'xfail')
 
     # Parse some text as a .sum file and return the resulting
     # dictionary.
-    def read_sum_text(self, text):
+    def read_sum_text (self, text):
-        cur_file = StringIO(text)
+        cur_file = StringIO (text)
         cur_results = {}
-        for line in cur_file.readlines():
+        for line in cur_file.readlines ():
-            self.parse_sum_line(cur_results, line)
+            self.parse_sum_line (cur_results, line)
         return cur_results
 
     # Compute regressions between RESULTS and BASELINE on BUILDER.
     # BASELINE will be modified if any new PASSes are seen.
     # Returns a regression report, as a string.
-    def compute_regressions(self, builder, results, baseline):
+    def compute_regressions (self, builder, branch, results, baseline):
-        our_keys = results.keys()
+        our_keys = results.keys ()
-        our_keys.sort()
+        our_keys.sort ()
         result = ''
-        xfails = self.read_xfail (builder)
+        xfails = self.read_xfail (builder, branch)
         if xfails is None:
             xfails = {}
         for key in our_keys:

master.cfg

@@ -33,6 +33,7 @@ import os.path
 import urllib
 from json import load
 import random
+import re
 
 ####################################
 ####################################
@@ -62,12 +63,22 @@ c['protocols'] = {'pb': {'port': 16123}}
 # the 'change_source' setting tells the buildmaster how it should find out
 # about source code changes.
 
+# RE representing which branches to track on the GDB repository
+branches_to_watch = re.compile ("(refs/heads/)?(master|gdb-\d+\.\d+-branch)")
+
+# Function which decides whether BRANCH should be used or not
+def should_watch_branch (branch):
+    if re.match (branches_to_watch, branch):
+        return True
+    else:
+        return False
+
 from buildbot.changes.gitpoller import GitPoller
 c['change_source'] = []
 c['change_source'].append(GitPoller(
 	repourl = r'git://sourceware.org/git/binutils-gdb.git',
 	workdir = os.path.expanduser (os.path.join ('~/', 'buildbot-master-binutils-gdb')),
-	branches = [ r'master' ],
+	branches = should_watch_branch,
 	pollinterval = 60 * 3))
 
 # 'status' is a list of Status Targets. The results of each build will be
@@ -127,7 +138,7 @@ send to the gdb-testers mailing list."""
     git_url = "http://gdb-build.sergiodj.net/cgit"
 
     # Subject
-    subj = "Failures on %s" % name
+    subj = "Failures on %s, branch %s" % (name, build.getSourceStamps ()[0].branch)
 
     # Body
     text = ""
@@ -581,7 +592,7 @@ class RunTestGDBIndexBuildBSD (RunTestGDBIndexBuild, RunTestGDBBSD_Common):
     pass
 
 # For now, we only support testing the "master" branch.
-master_filter = ChangeFilter (branch = [ r'master' ])
+all_gdb_filter = ChangeFilter (branch_fn = should_watch_branch)
 
 ###############################
 #### Configuration loading ####