Implementing racy testcase detection

Several modifications were necessary.  Mainly, the Nightly scheduler has
been added.  I've also created a new class in order to help detect and
notify about the racy testcases found.

lib/config.json

@@ -318,6 +318,51 @@
 					  "FreeBSD-x86_64-m64",
 					  "FreeBSD-x86_64-cc-with-index" ]
 
+		     },
+
+		     { "type" : "Nightly", "name" : "racy",
+		       "dayOfWeek" : "6", "hour" : "3", "minute" : "0",
+		       "builderNames" : [ "Fedora-x86_64-m64",
+					  "Fedora-x86_64-m32",
+					  "Fedora-x86_64-native-gdbserver-m64",
+					  "Fedora-x86_64-native-gdbserver-m32",
+					  "Fedora-x86_64-native-extended-gdbserver-m64",
+					  "Fedora-x86_64-native-extended-gdbserver-m32",
+					  "Fedora-x86_64-cc-with-index",
+					  "Fedora-i686",
+					  "Fedora-x86_64-cxx-build-m64",
+
+					  "Debian-x86_64-m64",
+					  "Debian-x86_64-native-gdbserver-m64",
+					  "Debian-x86_64-native-extended-gdbserver-m64",
+					  "Debian-i686-native-gdbserver",
+					  "Debian-i686-native-extended-gdbserver",
+					  "Debian-i686",
+					  "Debian-s390x-m64",
+					  "Debian-s390x-native-gdbserver-m64",
+					  "Debian-s390x-native-extended-gdbserver-m64",
+
+					  "Fedora-s390x-m64",
+
+					  "Fedora-ppc64be-m64",
+					  "Fedora-ppc64be-native-gdbserver-m64",
+					  "Fedora-ppc64be-native-extended-gdbserver-m64",
+					  "Fedora-ppc64be-cc-with-index",
+
+					  "Fedora-ppc64le-m64",
+					  "Fedora-ppc64le-native-gdbserver-m64",
+					  "Fedora-ppc64le-native-extended-gdbserver-m64",
+					  "Fedora-ppc64le-cc-with-index",
+
+					  "AIX-POWER7-plain",
+
+					  "RHEL-s390x-m64",
+
+					  "NetBSD-x86_64-m64",
+
+					  "FreeBSD-x86_64-m64",
+					  "FreeBSD-x86_64-cc-with-index" ]
+
 		     }
 		   ]
 }

lib/racyanalyze.py

@@ -0,0 +1,42 @@
+from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
+from buildbot.steps.shell import ShellCommand
+from sumfiles import DejaResults
+import smtplib
+import socket
+from email.mime.text import MIMEText
+
+class GDBAnalyzeRacyTests (ShellCommand):
+    """Analyze the racy tests"""
+    command = ['cat', 'racy.sum']
+
+    def __init__ (self, **kwargs):
+        ShellCommand.__init__ (self, **kwargs)
+
+    def evaluateCommand (self, cmd):
+        builder = self.getProperty('buildername')
+        branch = self.getProperty('branch')
+
+        p = DejaResults ()
+
+        racy_tests = p.read_sum_text (self.getLog ('stdio').getText ())
+        xfails = p.read_xfail (builder, branch)
+
+        unique_tests = racy_tests[1]['NONE'] - xfails[1]['FAIL']
+
+        msg = "============================\n"
+        for t in unique_tests:
+            msg += "FAIL: %s\n" % t
+        msg += "============================\n"
+
+        mail = MIMEText (msg)
+        mail['Subject'] = 'Failures on %s, branch %s' % (builder, branch)
+        mail['From'] = 'GDB BuildBot Racy Detector <gdb-buildbot@sergiodj.net>'
+        mail['To'] = 'gdb-buildbot@sergiodj.net'
+
+        s = smtplib.SMTP ('localhost')
+        s.sendmail ('gdb-buildbot@sergiodj.net',
+                    [ 'GDB BuildBot Racy Detector <gdb-buildbot@sergiodj.net>' ],
+                    mail.as_string ())
+        s.quit ()
+
+        return SUCCESS

lib/sumfiles.py

@@ -8,7 +8,7 @@ from StringIO import StringIO
 from collections import OrderedDict
 
 # Helper regex for parse_sum_line.
-sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$')
+sum_matcher = re.compile('^(.?(PASS|FAIL))?(: )?(.*)$')
 
 # You must call set_web_base at startup to set this.
 gdb_web_base = None
@@ -34,11 +34,19 @@ class DejaResults(object):
     # If the line does not appear to be about a test, ignore it.
     def parse_sum_line(self, out_dict, line):
         global sum_matcher
+
         line = line.rstrip()
         m = re.match(sum_matcher, line)
         if m:
             result = m.group(1)
-            test_name = m.group(3)
+            if not result:
+                # On racy.sum files, there is no result to parse.
+                result = 'NONE'
+            test_name = m.group(4)
+            # Remove tail parentheses
+            test_name = re.sub ('(\s+)?\(.*$', '', test_name)
+            if result not in out_dict[1].keys ():
+                out_dict[1][result] = set ()
             if test_name in out_dict:
                 i = 2
                 while True:
@@ -47,7 +55,10 @@ class DejaResults(object):
                         break
                     i = i + 1
                 test_name = nname
-            out_dict[test_name] = result
+            # Add the testname to the dictionary...
+            out_dict[0][test_name] = result
+            # and to the set.
+            out_dict[1][result].add (test_name)
 
     def _write_sum_file(self, sum_dict, subdir, rev_or_branch, filename,
                         header = None):
@@ -59,7 +70,7 @@ class DejaResults(object):
         if not os.path.isdir (bdir):
             os.makedirs (bdir, 0755)
         fname = os.path.join (bdir, filename)
-        keys = sum_dict.keys ()
+        keys = sum_dict[0].keys ()
         mode = 'w'
         if header:
             with open (fname, 'w') as f:
@@ -67,7 +78,7 @@ class DejaResults(object):
             mode = 'a'
         with open (fname, mode) as f:
             for k in keys:
-                f.write (sum_dict[k] + ': ' + k + '\n')
+                f.write (sum_dict[0][k] + ': ' + k + '\n')
 
     def write_sum_file(self, sum_dict, builder, branch):
         self._write_sum_file (sum_dict, builder, None, 'gdb.sum')
@@ -89,7 +100,12 @@ class DejaResults(object):
         else:
             fname = os.path.join (gdb_web_base, subdir, rev_or_branch, filename)
         if os.path.exists (fname):
-            result = OrderedDict ()
+            result = []
+            # result[0] is the OrderedDict containing all the tests
+            # and results.
+            result.append (OrderedDict ())
+            # result[1] is a dictionary containing sets of tests
+            result.append (dict ())
             with open (fname, 'r') as f:
                 for line in f:
                     self.parse_sum_line (result, line)
@@ -114,33 +130,43 @@ class DejaResults(object):
     # dictionary.
     def read_sum_text (self, text):
         cur_file = StringIO (text)
-        cur_results = OrderedDict ()
+        cur_results = []
+        cur_results.append (OrderedDict ())
+        cur_results.append (dict ())
         for line in cur_file.readlines ():
             self.parse_sum_line (cur_results, line)
         return cur_results
 
+    # Parse some text as the racy.sum file and return the resulting
+    # dictionary.
+    def read_racy_sum_text (self, text):
+        return self.read_sum_text (text)
+
     # Compute regressions between RESULTS and BASELINE on BUILDER.
     # BASELINE will be modified if any new PASSes are seen.
     # Returns a regression report, as a string.
-    def compute_regressions (self, builder, branch, results, baseline):
+    def compute_regressions (self, builder, branch, results, old_res):
-        our_keys = results.keys ()
+        our_keys = results[0].keys ()
         result = ''
         xfails = self.read_xfail (builder, branch)
         if xfails is None:
             xfails = {}
+        else:
+            xfails = xfails[0]
         for key in our_keys:
             # An XFAIL entry means we have an unreliable test.
             if key in xfails:
                 continue
             # A transition to PASS means we should update the baseline.
-            if results[key] == 'PASS':
+            if results[0][key] == 'PASS':
-                if key not in baseline or baseline[key] != 'PASS':
+                if key not in old_res[0] or old_res[0][key] != 'PASS':
-                    baseline[key] = 'PASS'
+                    old_res[0][key] = 'PASS'
-            # A regression is just a transition to FAIL.
-            if results[key] != 'FAIL':
                 continue
-            if key not in baseline:
+            # A regression is just a transition to FAIL.
+            if results[0][key] != 'FAIL':
+                continue
+            if key not in old_res[0]:
                 result = result + 'new FAIL: ' + key + '\n'
-            elif baseline[key] != 'FAIL':
+            elif old_res[0][key] != 'FAIL':
-                result = result + baseline[key] + ' -> FAIL: ' + key + '\n'
+                result = result + old_res[0][key] + ' -> FAIL: ' + key + '\n'
         return result

master.cfg

@@ -11,6 +11,7 @@
 
 
 from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
+from buildbot.schedulers.timed import Nightly
 from buildbot.schedulers.forcesched import ForceScheduler
 from buildbot.process import factory
 from buildbot.process.properties import WithProperties
@@ -26,6 +27,7 @@ from buildbot.buildslave import BuildSlave
 from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION
 from gdbcommand import CopyOldGDBSumFile, GdbCatSumfileCommand
 from gdbgitdb import SaveGDBResults, get_builder_commit_id
+from racyanalyze import GDBAnalyzeRacyTests
 from urllib import quote
 
 from sumfiles import DejaResults, set_web_base
@@ -384,7 +386,17 @@ send to the gdb-testers mailing list."""
              'subject' : subj }
 
 from buildbot.status import mail
-mn = mail.MailNotifier(fromaddr = GDB_MAIL_FROM,
+
+class MyMailNotifier (mail.MailNotifier):
+    """Extend the regular MailNotifier class in order to filter e-mails by
+scheduler."""
+    def isMailNeeded (self, build):
+        if build.properties.getProperty ('scheduler') is not 'racy':
+            return mail.MailNotifier.isMailNeeded (self, build)
+        else:
+            return False
+
+mn = MyMailNotifier(fromaddr = GDB_MAIL_FROM,
                     sendToInterestedUsers = False,
                     extraRecipients = [ GDB_MAIL_TO ],
                     mode = ('failing'),
@@ -415,6 +427,7 @@ c['db'] = {
 ## does.
 
 class CloneOrUpdateGDBMasterRepo (Git):
+
     """This build step updates the so-called "master" git repository.  For
 each buildslave, we have one master GDB git repository, which is then
 used to create each builder's GDB git repository.  In other words,
@@ -560,6 +573,12 @@ class CleanupBreakageLockfile (ShellCommand):
         return SUCCESS
 
 
+def scheduler_is_racy (step):
+    return step.build.properties.getProperty ('scheduler') == 'racy'
+
+def scheduler_is_not_racy (step):
+    return step.build.properties.getProperty ('scheduler') != 'racy'
+
 #######################
 #### Build Factory ####
 #######################
@@ -701,16 +720,39 @@ The parameters of the class are:
                 self.extra_make_check_flags.append (r'FORCE_PARALLEL=1')
 
             self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags,
-                                          self.test_env))
+                                          self.test_env,
+                                          doStepIf = scheduler_is_not_racy,
+                                          hideStepIf = scheduler_is_racy))
 
             self.addStep (GdbCatSumfileCommand (workdir = WithProperties (r'%s/build/gdb/testsuite',
                                                                           r'builddir'),
-                                                description = r'analyze test results'))
+                                                description = r'analyze test results',
+                                                doStepIf = scheduler_is_not_racy,
+                                                hideStepIf = scheduler_is_racy))
             self.addStep (FileUpload (slavesrc = WithProperties (r"%s/build/gdb/testsuite/gdb.log",
                                                                  r'builddir'),
                                       masterdest = WithProperties (r"public_html/results/%s/gdb.log",
-                                                                   r'buildername')))
+                                                                   r'buildername'),
-            self.addStep (SaveGDBResults ())
+                                      doStepIf = scheduler_is_not_racy,
+                                      hideStepIf = scheduler_is_racy))
+            self.addStep (SaveGDBResults (doStepIf = scheduler_is_not_racy,
+                                          hideStepIf = scheduler_is_racy))
+
+            ##################### Racy ######################
+
+            self.extra_make_check_flags.append ('RACY_ITER=5')
+
+            self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags,
+                                          self.test_env,
+                                          doStepIf = scheduler_is_racy,
+                                          hideStepIf = scheduler_is_not_racy))
+
+            self.addStep (GDBAnalyzeRacyTests (workdir = WithProperties ('%s/build/gdb/testsuite',
+                                                                         'builddir'),
+                                               description = 'analyzing racy tests',
+                                               descriptionDone = 'analyzed racy tests',
+                                               doStepIf = scheduler_is_racy,
+                                               hideStepIf = scheduler_is_not_racy))
 
 
 ##################
@@ -906,10 +948,18 @@ def load_config (c):
 
     c['schedulers'] = []
     for s in config['schedulers']:
-        if "change_filter" in s:
+        # Ugh :-/.  There should be no special case here...
-            s['change_filter'] = globals ()[s['change_filter']]
+        if s['type'] is not 'Nightly':
             s['treeStableTimer'] = None
             s['fileIsImportant'] = DefaultGDBfileIsImportant
+        else:
+            s['dayOfWeek'] = int (s['dayOfWeek'])
+            s['hour'] = int (s['hour'])
+            s['minute'] = int (s['minute'])
+            s['onlyIfChanged'] = True
+            s['branch'] = None
+        if "change_filter" in s:
+            s['change_filter'] = globals ()[s['change_filter']]
         kls = globals ()[s.pop ('type')]
         s = dict (map (lambda key_value_pair : (str (key_value_pair[0]),
                                                 key_value_pair[1]),