Implementing racy testcase detection
Several modifications were necessary. Mainly, the Nightly scheduler has been added. I've also created a new class in order to help detect and notify about the racy testcases found.
This commit is contained in:
parent
551fa187c3
commit
6c86aad971
4 changed files with 195 additions and 32 deletions
|
|
@ -318,6 +318,51 @@
|
|||
"FreeBSD-x86_64-m64",
|
||||
"FreeBSD-x86_64-cc-with-index" ]
|
||||
|
||||
},
|
||||
|
||||
{ "type" : "Nightly", "name" : "racy",
|
||||
"dayOfWeek" : "6", "hour" : "3", "minute" : "0",
|
||||
"builderNames" : [ "Fedora-x86_64-m64",
|
||||
"Fedora-x86_64-m32",
|
||||
"Fedora-x86_64-native-gdbserver-m64",
|
||||
"Fedora-x86_64-native-gdbserver-m32",
|
||||
"Fedora-x86_64-native-extended-gdbserver-m64",
|
||||
"Fedora-x86_64-native-extended-gdbserver-m32",
|
||||
"Fedora-x86_64-cc-with-index",
|
||||
"Fedora-i686",
|
||||
"Fedora-x86_64-cxx-build-m64",
|
||||
|
||||
"Debian-x86_64-m64",
|
||||
"Debian-x86_64-native-gdbserver-m64",
|
||||
"Debian-x86_64-native-extended-gdbserver-m64",
|
||||
"Debian-i686-native-gdbserver",
|
||||
"Debian-i686-native-extended-gdbserver",
|
||||
"Debian-i686",
|
||||
"Debian-s390x-m64",
|
||||
"Debian-s390x-native-gdbserver-m64",
|
||||
"Debian-s390x-native-extended-gdbserver-m64",
|
||||
|
||||
"Fedora-s390x-m64",
|
||||
|
||||
"Fedora-ppc64be-m64",
|
||||
"Fedora-ppc64be-native-gdbserver-m64",
|
||||
"Fedora-ppc64be-native-extended-gdbserver-m64",
|
||||
"Fedora-ppc64be-cc-with-index",
|
||||
|
||||
"Fedora-ppc64le-m64",
|
||||
"Fedora-ppc64le-native-gdbserver-m64",
|
||||
"Fedora-ppc64le-native-extended-gdbserver-m64",
|
||||
"Fedora-ppc64le-cc-with-index",
|
||||
|
||||
"AIX-POWER7-plain",
|
||||
|
||||
"RHEL-s390x-m64",
|
||||
|
||||
"NetBSD-x86_64-m64",
|
||||
|
||||
"FreeBSD-x86_64-m64",
|
||||
"FreeBSD-x86_64-cc-with-index" ]
|
||||
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
42
lib/racyanalyze.py
Normal file
42
lib/racyanalyze.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
from sumfiles import DejaResults
|
||||
import smtplib
|
||||
import socket
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
class GDBAnalyzeRacyTests (ShellCommand):
|
||||
"""Analyze the racy tests"""
|
||||
command = ['cat', 'racy.sum']
|
||||
|
||||
def __init__ (self, **kwargs):
|
||||
ShellCommand.__init__ (self, **kwargs)
|
||||
|
||||
def evaluateCommand (self, cmd):
|
||||
builder = self.getProperty('buildername')
|
||||
branch = self.getProperty('branch')
|
||||
|
||||
p = DejaResults ()
|
||||
|
||||
racy_tests = p.read_sum_text (self.getLog ('stdio').getText ())
|
||||
xfails = p.read_xfail (builder, branch)
|
||||
|
||||
unique_tests = racy_tests[1]['NONE'] - xfails[1]['FAIL']
|
||||
|
||||
msg = "============================\n"
|
||||
for t in unique_tests:
|
||||
msg += "FAIL: %s\n" % t
|
||||
msg += "============================\n"
|
||||
|
||||
mail = MIMEText (msg)
|
||||
mail['Subject'] = 'Failures on %s, branch %s' % (builder, branch)
|
||||
mail['From'] = 'GDB BuildBot Racy Detector <gdb-buildbot@sergiodj.net>'
|
||||
mail['To'] = 'gdb-buildbot@sergiodj.net'
|
||||
|
||||
s = smtplib.SMTP ('localhost')
|
||||
s.sendmail ('gdb-buildbot@sergiodj.net',
|
||||
[ 'GDB BuildBot Racy Detector <gdb-buildbot@sergiodj.net>' ],
|
||||
mail.as_string ())
|
||||
s.quit ()
|
||||
|
||||
return SUCCESS
|
||||
|
|
@ -8,7 +8,7 @@ from StringIO import StringIO
|
|||
from collections import OrderedDict
|
||||
|
||||
# Helper regex for parse_sum_line.
|
||||
sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$')
|
||||
sum_matcher = re.compile('^(.?(PASS|FAIL))?(: )?(.*)$')
|
||||
|
||||
# You must call set_web_base at startup to set this.
|
||||
gdb_web_base = None
|
||||
|
|
@ -34,11 +34,19 @@ class DejaResults(object):
|
|||
# If the line does not appear to be about a test, ignore it.
|
||||
def parse_sum_line(self, out_dict, line):
|
||||
global sum_matcher
|
||||
|
||||
line = line.rstrip()
|
||||
m = re.match(sum_matcher, line)
|
||||
if m:
|
||||
result = m.group(1)
|
||||
test_name = m.group(3)
|
||||
if not result:
|
||||
# On racy.sum files, there is no result to parse.
|
||||
result = 'NONE'
|
||||
test_name = m.group(4)
|
||||
# Remove tail parentheses
|
||||
test_name = re.sub ('(\s+)?\(.*$', '', test_name)
|
||||
if result not in out_dict[1].keys ():
|
||||
out_dict[1][result] = set ()
|
||||
if test_name in out_dict:
|
||||
i = 2
|
||||
while True:
|
||||
|
|
@ -47,7 +55,10 @@ class DejaResults(object):
|
|||
break
|
||||
i = i + 1
|
||||
test_name = nname
|
||||
out_dict[test_name] = result
|
||||
# Add the testname to the dictionary...
|
||||
out_dict[0][test_name] = result
|
||||
# and to the set.
|
||||
out_dict[1][result].add (test_name)
|
||||
|
||||
def _write_sum_file(self, sum_dict, subdir, rev_or_branch, filename,
|
||||
header = None):
|
||||
|
|
@ -59,7 +70,7 @@ class DejaResults(object):
|
|||
if not os.path.isdir (bdir):
|
||||
os.makedirs (bdir, 0755)
|
||||
fname = os.path.join (bdir, filename)
|
||||
keys = sum_dict.keys ()
|
||||
keys = sum_dict[0].keys ()
|
||||
mode = 'w'
|
||||
if header:
|
||||
with open (fname, 'w') as f:
|
||||
|
|
@ -67,7 +78,7 @@ class DejaResults(object):
|
|||
mode = 'a'
|
||||
with open (fname, mode) as f:
|
||||
for k in keys:
|
||||
f.write (sum_dict[k] + ': ' + k + '\n')
|
||||
f.write (sum_dict[0][k] + ': ' + k + '\n')
|
||||
|
||||
def write_sum_file(self, sum_dict, builder, branch):
|
||||
self._write_sum_file (sum_dict, builder, None, 'gdb.sum')
|
||||
|
|
@ -89,7 +100,12 @@ class DejaResults(object):
|
|||
else:
|
||||
fname = os.path.join (gdb_web_base, subdir, rev_or_branch, filename)
|
||||
if os.path.exists (fname):
|
||||
result = OrderedDict ()
|
||||
result = []
|
||||
# result[0] is the OrderedDict containing all the tests
|
||||
# and results.
|
||||
result.append (OrderedDict ())
|
||||
# result[1] is a dictionary containing sets of tests
|
||||
result.append (dict ())
|
||||
with open (fname, 'r') as f:
|
||||
for line in f:
|
||||
self.parse_sum_line (result, line)
|
||||
|
|
@ -114,33 +130,43 @@ class DejaResults(object):
|
|||
# dictionary.
|
||||
def read_sum_text (self, text):
|
||||
cur_file = StringIO (text)
|
||||
cur_results = OrderedDict ()
|
||||
cur_results = []
|
||||
cur_results.append (OrderedDict ())
|
||||
cur_results.append (dict ())
|
||||
for line in cur_file.readlines ():
|
||||
self.parse_sum_line (cur_results, line)
|
||||
return cur_results
|
||||
|
||||
# Parse some text as the racy.sum file and return the resulting
|
||||
# dictionary.
|
||||
def read_racy_sum_text (self, text):
|
||||
return self.read_sum_text (text)
|
||||
|
||||
# Compute regressions between RESULTS and BASELINE on BUILDER.
|
||||
# BASELINE will be modified if any new PASSes are seen.
|
||||
# Returns a regression report, as a string.
|
||||
def compute_regressions (self, builder, branch, results, baseline):
|
||||
our_keys = results.keys ()
|
||||
def compute_regressions (self, builder, branch, results, old_res):
|
||||
our_keys = results[0].keys ()
|
||||
result = ''
|
||||
xfails = self.read_xfail (builder, branch)
|
||||
if xfails is None:
|
||||
xfails = {}
|
||||
else:
|
||||
xfails = xfails[0]
|
||||
for key in our_keys:
|
||||
# An XFAIL entry means we have an unreliable test.
|
||||
if key in xfails:
|
||||
continue
|
||||
# A transition to PASS means we should update the baseline.
|
||||
if results[key] == 'PASS':
|
||||
if key not in baseline or baseline[key] != 'PASS':
|
||||
baseline[key] = 'PASS'
|
||||
# A regression is just a transition to FAIL.
|
||||
if results[key] != 'FAIL':
|
||||
if results[0][key] == 'PASS':
|
||||
if key not in old_res[0] or old_res[0][key] != 'PASS':
|
||||
old_res[0][key] = 'PASS'
|
||||
continue
|
||||
if key not in baseline:
|
||||
# A regression is just a transition to FAIL.
|
||||
if results[0][key] != 'FAIL':
|
||||
continue
|
||||
if key not in old_res[0]:
|
||||
result = result + 'new FAIL: ' + key + '\n'
|
||||
elif baseline[key] != 'FAIL':
|
||||
result = result + baseline[key] + ' -> FAIL: ' + key + '\n'
|
||||
elif old_res[0][key] != 'FAIL':
|
||||
result = result + old_res[0][key] + ' -> FAIL: ' + key + '\n'
|
||||
return result
|
||||
|
|
|
|||
80
master.cfg
80
master.cfg
|
|
@ -11,6 +11,7 @@
|
|||
|
||||
|
||||
from buildbot.schedulers.basic import SingleBranchScheduler, AnyBranchScheduler
|
||||
from buildbot.schedulers.timed import Nightly
|
||||
from buildbot.schedulers.forcesched import ForceScheduler
|
||||
from buildbot.process import factory
|
||||
from buildbot.process.properties import WithProperties
|
||||
|
|
@ -26,6 +27,7 @@ from buildbot.buildslave import BuildSlave
|
|||
from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION
|
||||
from gdbcommand import CopyOldGDBSumFile, GdbCatSumfileCommand
|
||||
from gdbgitdb import SaveGDBResults, get_builder_commit_id
|
||||
from racyanalyze import GDBAnalyzeRacyTests
|
||||
from urllib import quote
|
||||
|
||||
from sumfiles import DejaResults, set_web_base
|
||||
|
|
@ -384,15 +386,25 @@ send to the gdb-testers mailing list."""
|
|||
'subject' : subj }
|
||||
|
||||
from buildbot.status import mail
|
||||
mn = mail.MailNotifier(fromaddr = GDB_MAIL_FROM,
|
||||
sendToInterestedUsers = False,
|
||||
extraRecipients = [ GDB_MAIL_TO ],
|
||||
mode = ('failing'),
|
||||
messageFormatter = MessageGDBTesters,
|
||||
tags = [ "MAIL" ],
|
||||
extraHeaders = { 'X-GDB-Buildbot' : '1',
|
||||
'In-Reply-To' : WithProperties ("<%s@gdb-build>",
|
||||
'got_revision') })
|
||||
|
||||
class MyMailNotifier (mail.MailNotifier):
|
||||
"""Extend the regular MailNotifier class in order to filter e-mails by
|
||||
scheduler."""
|
||||
def isMailNeeded (self, build):
|
||||
if build.properties.getProperty ('scheduler') is not 'racy':
|
||||
return mail.MailNotifier.isMailNeeded (self, build)
|
||||
else:
|
||||
return False
|
||||
|
||||
mn = MyMailNotifier(fromaddr = GDB_MAIL_FROM,
|
||||
sendToInterestedUsers = False,
|
||||
extraRecipients = [ GDB_MAIL_TO ],
|
||||
mode = ('failing'),
|
||||
messageFormatter = MessageGDBTesters,
|
||||
tags = [ "MAIL" ],
|
||||
extraHeaders = { 'X-GDB-Buildbot' : '1',
|
||||
'In-Reply-To' : WithProperties ("<%s@gdb-build>",
|
||||
'got_revision') })
|
||||
|
||||
c['status'].append (mn)
|
||||
|
||||
|
|
@ -415,6 +427,7 @@ c['db'] = {
|
|||
## does.
|
||||
|
||||
class CloneOrUpdateGDBMasterRepo (Git):
|
||||
|
||||
"""This build step updates the so-called "master" git repository. For
|
||||
each buildslave, we have one master GDB git repository, which is then
|
||||
used to create each builder's GDB git repository. In other words,
|
||||
|
|
@ -560,6 +573,12 @@ class CleanupBreakageLockfile (ShellCommand):
|
|||
return SUCCESS
|
||||
|
||||
|
||||
def scheduler_is_racy (step):
|
||||
return step.build.properties.getProperty ('scheduler') == 'racy'
|
||||
|
||||
def scheduler_is_not_racy (step):
|
||||
return step.build.properties.getProperty ('scheduler') != 'racy'
|
||||
|
||||
#######################
|
||||
#### Build Factory ####
|
||||
#######################
|
||||
|
|
@ -701,16 +720,39 @@ The parameters of the class are:
|
|||
self.extra_make_check_flags.append (r'FORCE_PARALLEL=1')
|
||||
|
||||
self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags,
|
||||
self.test_env))
|
||||
self.test_env,
|
||||
doStepIf = scheduler_is_not_racy,
|
||||
hideStepIf = scheduler_is_racy))
|
||||
|
||||
self.addStep (GdbCatSumfileCommand (workdir = WithProperties (r'%s/build/gdb/testsuite',
|
||||
r'builddir'),
|
||||
description = r'analyze test results'))
|
||||
description = r'analyze test results',
|
||||
doStepIf = scheduler_is_not_racy,
|
||||
hideStepIf = scheduler_is_racy))
|
||||
self.addStep (FileUpload (slavesrc = WithProperties (r"%s/build/gdb/testsuite/gdb.log",
|
||||
r'builddir'),
|
||||
masterdest = WithProperties (r"public_html/results/%s/gdb.log",
|
||||
r'buildername')))
|
||||
self.addStep (SaveGDBResults ())
|
||||
r'buildername'),
|
||||
doStepIf = scheduler_is_not_racy,
|
||||
hideStepIf = scheduler_is_racy))
|
||||
self.addStep (SaveGDBResults (doStepIf = scheduler_is_not_racy,
|
||||
hideStepIf = scheduler_is_racy))
|
||||
|
||||
##################### Racy ######################
|
||||
|
||||
self.extra_make_check_flags.append ('RACY_ITER=5')
|
||||
|
||||
self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags,
|
||||
self.test_env,
|
||||
doStepIf = scheduler_is_racy,
|
||||
hideStepIf = scheduler_is_not_racy))
|
||||
|
||||
self.addStep (GDBAnalyzeRacyTests (workdir = WithProperties ('%s/build/gdb/testsuite',
|
||||
'builddir'),
|
||||
description = 'analyzing racy tests',
|
||||
descriptionDone = 'analyzed racy tests',
|
||||
doStepIf = scheduler_is_racy,
|
||||
hideStepIf = scheduler_is_not_racy))
|
||||
|
||||
|
||||
##################
|
||||
|
|
@ -906,10 +948,18 @@ def load_config (c):
|
|||
|
||||
c['schedulers'] = []
|
||||
for s in config['schedulers']:
|
||||
# Ugh :-/. There should be no special case here...
|
||||
if s['type'] is not 'Nightly':
|
||||
s['treeStableTimer'] = None
|
||||
s['fileIsImportant'] = DefaultGDBfileIsImportant
|
||||
else:
|
||||
s['dayOfWeek'] = int (s['dayOfWeek'])
|
||||
s['hour'] = int (s['hour'])
|
||||
s['minute'] = int (s['minute'])
|
||||
s['onlyIfChanged'] = True
|
||||
s['branch'] = None
|
||||
if "change_filter" in s:
|
||||
s['change_filter'] = globals ()[s['change_filter']]
|
||||
s['treeStableTimer'] = None
|
||||
s['fileIsImportant'] = DefaultGDBfileIsImportant
|
||||
kls = globals ()[s.pop ('type')]
|
||||
s = dict (map (lambda key_value_pair : (str (key_value_pair[0]),
|
||||
key_value_pair[1]),
|
||||
|
|
|
|||
Loading…
Reference in a new issue