From 6c86aad97173392835afde470d74bcb37a92b2e8 Mon Sep 17 00:00:00 2001 From: Sergio Durigan Junior Date: Tue, 15 Mar 2016 18:04:36 -0400 Subject: [PATCH] Implementing racy testcase detection Several modifications were necessary. Mainly, the Nightly scheduler has been added. I've also created a new class in order to help detect and notify about the racy testcases found. --- lib/config.json | 45 ++++++++++++++++++++++++++ lib/racyanalyze.py | 42 ++++++++++++++++++++++++ lib/sumfiles.py | 60 ++++++++++++++++++++++++---------- master.cfg | 80 +++++++++++++++++++++++++++++++++++++--------- 4 files changed, 195 insertions(+), 32 deletions(-) create mode 100644 lib/racyanalyze.py diff --git a/lib/config.json b/lib/config.json index 5adb7f8..995b7ca 100644 --- a/lib/config.json +++ b/lib/config.json @@ -318,6 +318,51 @@ "FreeBSD-x86_64-m64", "FreeBSD-x86_64-cc-with-index" ] + }, + + { "type" : "Nightly", "name" : "racy", + "dayOfWeek" : "6", "hour" : "3", "minute" : "0", + "builderNames" : [ "Fedora-x86_64-m64", + "Fedora-x86_64-m32", + "Fedora-x86_64-native-gdbserver-m64", + "Fedora-x86_64-native-gdbserver-m32", + "Fedora-x86_64-native-extended-gdbserver-m64", + "Fedora-x86_64-native-extended-gdbserver-m32", + "Fedora-x86_64-cc-with-index", + "Fedora-i686", + "Fedora-x86_64-cxx-build-m64", + + "Debian-x86_64-m64", + "Debian-x86_64-native-gdbserver-m64", + "Debian-x86_64-native-extended-gdbserver-m64", + "Debian-i686-native-gdbserver", + "Debian-i686-native-extended-gdbserver", + "Debian-i686", + "Debian-s390x-m64", + "Debian-s390x-native-gdbserver-m64", + "Debian-s390x-native-extended-gdbserver-m64", + + "Fedora-s390x-m64", + + "Fedora-ppc64be-m64", + "Fedora-ppc64be-native-gdbserver-m64", + "Fedora-ppc64be-native-extended-gdbserver-m64", + "Fedora-ppc64be-cc-with-index", + + "Fedora-ppc64le-m64", + "Fedora-ppc64le-native-gdbserver-m64", + "Fedora-ppc64le-native-extended-gdbserver-m64", + "Fedora-ppc64le-cc-with-index", + + "AIX-POWER7-plain", + + "RHEL-s390x-m64", + + "NetBSD-x86_64-m64", + + "FreeBSD-x86_64-m64", + "FreeBSD-x86_64-cc-with-index" ] + } ] } diff --git a/lib/racyanalyze.py b/lib/racyanalyze.py new file mode 100644 index 0000000..79d7792 --- /dev/null +++ b/lib/racyanalyze.py @@ -0,0 +1,42 @@ +from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION +from buildbot.steps.shell import ShellCommand +from sumfiles import DejaResults +import smtplib +import socket +from email.mime.text import MIMEText + +class GDBAnalyzeRacyTests (ShellCommand): + """Analyze the racy tests""" + command = ['cat', 'racy.sum'] + + def __init__ (self, **kwargs): + ShellCommand.__init__ (self, **kwargs) + + def evaluateCommand (self, cmd): + builder = self.getProperty('buildername') + branch = self.getProperty('branch') + + p = DejaResults () + + racy_tests = p.read_sum_text (self.getLog ('stdio').getText ()) + xfails = p.read_xfail (builder, branch) + + unique_tests = racy_tests[1]['NONE'] - xfails[1]['FAIL'] + + msg = "============================\n" + for t in unique_tests: + msg += "FAIL: %s\n" % t + msg += "============================\n" + + mail = MIMEText (msg) + mail['Subject'] = 'Failures on %s, branch %s' % (builder, branch) + mail['From'] = 'GDB BuildBot Racy Detector ' + mail['To'] = 'gdb-buildbot@sergiodj.net' + + s = smtplib.SMTP ('localhost') + s.sendmail ('gdb-buildbot@sergiodj.net', + [ 'GDB BuildBot Racy Detector ' ], + mail.as_string ()) + s.quit () + + return SUCCESS diff --git a/lib/sumfiles.py b/lib/sumfiles.py index be9bbf7..41fa434 100644 --- a/lib/sumfiles.py +++ b/lib/sumfiles.py @@ -8,7 +8,7 @@ from StringIO import StringIO from collections import OrderedDict # Helper regex for parse_sum_line. -sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$') +sum_matcher = re.compile('^(.?(PASS|FAIL))?(: )?(.*)$') # You must call set_web_base at startup to set this. gdb_web_base = None @@ -34,11 +34,19 @@ class DejaResults(object): # If the line does not appear to be about a test, ignore it. def parse_sum_line(self, out_dict, line): global sum_matcher + line = line.rstrip() m = re.match(sum_matcher, line) if m: result = m.group(1) - test_name = m.group(3) + if not result: + # On racy.sum files, there is no result to parse. + result = 'NONE' + test_name = m.group(4) + # Remove tail parentheses + test_name = re.sub ('(\s+)?\(.*$', '', test_name) + if result not in out_dict[1].keys (): + out_dict[1][result] = set () if test_name in out_dict: i = 2 while True: @@ -47,7 +55,10 @@ class DejaResults(object): break i = i + 1 test_name = nname - out_dict[test_name] = result + # Add the testname to the dictionary... + out_dict[0][test_name] = result + # and to the set. + out_dict[1][result].add (test_name) def _write_sum_file(self, sum_dict, subdir, rev_or_branch, filename, header = None): @@ -59,7 +70,7 @@ class DejaResults(object): if not os.path.isdir (bdir): os.makedirs (bdir, 0755) fname = os.path.join (bdir, filename) - keys = sum_dict.keys () + keys = sum_dict[0].keys () mode = 'w' if header: with open (fname, 'w') as f: @@ -67,7 +78,7 @@ class DejaResults(object): mode = 'a' with open (fname, mode) as f: for k in keys: - f.write (sum_dict[k] + ': ' + k + '\n') + f.write (sum_dict[0][k] + ': ' + k + '\n') def write_sum_file(self, sum_dict, builder, branch): self._write_sum_file (sum_dict, builder, None, 'gdb.sum') @@ -89,7 +100,12 @@ class DejaResults(object): else: fname = os.path.join (gdb_web_base, subdir, rev_or_branch, filename) if os.path.exists (fname): - result = OrderedDict () + result = [] + # result[0] is the OrderedDict containing all the tests + # and results. + result.append (OrderedDict ()) + # result[1] is a dictionary containing sets of tests + result.append (dict ()) with open (fname, 'r') as f: for line in f: self.parse_sum_line (result, line) @@ -114,33 +130,43 @@ class DejaResults(object): # dictionary. def read_sum_text (self, text): cur_file = StringIO (text) - cur_results = OrderedDict () + cur_results = [] + cur_results.append (OrderedDict ()) + cur_results.append (dict ()) for line in cur_file.readlines (): self.parse_sum_line (cur_results, line) return cur_results + # Parse some text as the racy.sum file and return the resulting + # dictionary. + def read_racy_sum_text (self, text): + return self.read_sum_text (text) + # Compute regressions between RESULTS and BASELINE on BUILDER. # BASELINE will be modified if any new PASSes are seen. # Returns a regression report, as a string. - def compute_regressions (self, builder, branch, results, baseline): - our_keys = results.keys () + def compute_regressions (self, builder, branch, results, old_res): + our_keys = results[0].keys () result = '' xfails = self.read_xfail (builder, branch) if xfails is None: xfails = {} + else: + xfails = xfails[0] for key in our_keys: # An XFAIL entry means we have an unreliable test. if key in xfails: continue # A transition to PASS means we should update the baseline. - if results[key] == 'PASS': - if key not in baseline or baseline[key] != 'PASS': - baseline[key] = 'PASS' - # A regression is just a transition to FAIL. - if results[key] != 'FAIL': + if results[0][key] == 'PASS': + if key not in old_res[0] or old_res[0][key] != 'PASS': + old_res[0][key] = 'PASS' continue - if key not in baseline: + # A regression is just a transition to FAIL. + if results[0][key] != 'FAIL': + continue + if key not in old_res[0]: result = result + 'new FAIL: ' + key + '\n' - elif baseline[key] != 'FAIL': - result = result + baseline[key] + ' -> FAIL: ' + key + '\n' + elif old_res[0][key] != 'FAIL': + result = result + old_res[0][key] + ' -> FAIL: ' + key + '\n' return result diff --git a/master.cfg b/master.cfg index e939953..8b79f5b 100644 --- a/master.cfg +++ b/master.cfg @@ -11,6 +11,7 @@ from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler +from buildbot.schedulers.timed import Nightly from buildbot.schedulers.forcesched import ForceScheduler from buildbot.process import factory from buildbot.process.properties import WithProperties @@ -26,6 +27,7 @@ from buildbot.buildslave import BuildSlave from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION from gdbcommand import CopyOldGDBSumFile, GdbCatSumfileCommand from gdbgitdb import SaveGDBResults, get_builder_commit_id +from racyanalyze import GDBAnalyzeRacyTests from urllib import quote from sumfiles import DejaResults, set_web_base @@ -384,15 +386,25 @@ send to the gdb-testers mailing list.""" 'subject' : subj } from buildbot.status import mail -mn = mail.MailNotifier(fromaddr = GDB_MAIL_FROM, - sendToInterestedUsers = False, - extraRecipients = [ GDB_MAIL_TO ], - mode = ('failing'), - messageFormatter = MessageGDBTesters, - tags = [ "MAIL" ], - extraHeaders = { 'X-GDB-Buildbot' : '1', - 'In-Reply-To' : WithProperties ("<%s@gdb-build>", - 'got_revision') }) + +class MyMailNotifier (mail.MailNotifier): + """Extend the regular MailNotifier class in order to filter e-mails by +scheduler.""" + def isMailNeeded (self, build): + if build.properties.getProperty ('scheduler') is not 'racy': + return mail.MailNotifier.isMailNeeded (self, build) + else: + return False + +mn = MyMailNotifier(fromaddr = GDB_MAIL_FROM, + sendToInterestedUsers = False, + extraRecipients = [ GDB_MAIL_TO ], + mode = ('failing'), + messageFormatter = MessageGDBTesters, + tags = [ "MAIL" ], + extraHeaders = { 'X-GDB-Buildbot' : '1', + 'In-Reply-To' : WithProperties ("<%s@gdb-build>", + 'got_revision') }) c['status'].append (mn) @@ -415,6 +427,7 @@ c['db'] = { ## does. class CloneOrUpdateGDBMasterRepo (Git): + """This build step updates the so-called "master" git repository. For each buildslave, we have one master GDB git repository, which is then used to create each builder's GDB git repository. In other words, @@ -560,6 +573,12 @@ class CleanupBreakageLockfile (ShellCommand): return SUCCESS +def scheduler_is_racy (step): + return step.build.properties.getProperty ('scheduler') == 'racy' + +def scheduler_is_not_racy (step): + return step.build.properties.getProperty ('scheduler') != 'racy' + ####################### #### Build Factory #### ####################### @@ -701,16 +720,39 @@ The parameters of the class are: self.extra_make_check_flags.append (r'FORCE_PARALLEL=1') self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags, - self.test_env)) + self.test_env, + doStepIf = scheduler_is_not_racy, + hideStepIf = scheduler_is_racy)) self.addStep (GdbCatSumfileCommand (workdir = WithProperties (r'%s/build/gdb/testsuite', r'builddir'), - description = r'analyze test results')) + description = r'analyze test results', + doStepIf = scheduler_is_not_racy, + hideStepIf = scheduler_is_racy)) self.addStep (FileUpload (slavesrc = WithProperties (r"%s/build/gdb/testsuite/gdb.log", r'builddir'), masterdest = WithProperties (r"public_html/results/%s/gdb.log", - r'buildername'))) - self.addStep (SaveGDBResults ()) + r'buildername'), + doStepIf = scheduler_is_not_racy, + hideStepIf = scheduler_is_racy)) + self.addStep (SaveGDBResults (doStepIf = scheduler_is_not_racy, + hideStepIf = scheduler_is_racy)) + + ##################### Racy ###################### + + self.extra_make_check_flags.append ('RACY_ITER=5') + + self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags, + self.test_env, + doStepIf = scheduler_is_racy, + hideStepIf = scheduler_is_not_racy)) + + self.addStep (GDBAnalyzeRacyTests (workdir = WithProperties ('%s/build/gdb/testsuite', + 'builddir'), + description = 'analyzing racy tests', + descriptionDone = 'analyzed racy tests', + doStepIf = scheduler_is_racy, + hideStepIf = scheduler_is_not_racy)) ################## @@ -906,10 +948,18 @@ def load_config (c): c['schedulers'] = [] for s in config['schedulers']: + # Ugh :-/. There should be no special case here... + if s['type'] is not 'Nightly': + s['treeStableTimer'] = None + s['fileIsImportant'] = DefaultGDBfileIsImportant + else: + s['dayOfWeek'] = int (s['dayOfWeek']) + s['hour'] = int (s['hour']) + s['minute'] = int (s['minute']) + s['onlyIfChanged'] = True + s['branch'] = None if "change_filter" in s: s['change_filter'] = globals ()[s['change_filter']] - s['treeStableTimer'] = None - s['fileIsImportant'] = DefaultGDBfileIsImportant kls = globals ()[s.pop ('type')] s = dict (map (lambda key_value_pair : (str (key_value_pair[0]), key_value_pair[1]),