From 3c736356e5fa78ac2cc9d8e96b135b7a3093b6b3 Mon Sep 17 00:00:00 2001 From: Sergio Durigan Junior Date: Sat, 5 Nov 2016 23:59:59 -0400 Subject: [PATCH] Initial attempt to change the way we store results This was imported (with modifications) from the 0.9-prep branch. --- lib/fsdb.py | 20 +++++ lib/gdbcommand.py | 109 ++++++++++++++--------- lib/sumfiles.py | 74 ++++++++-------- master.cfg | 192 ++++++++++++++++++----------------------- scripts/update-logs.sh | 124 ++++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 186 deletions(-) create mode 100644 lib/fsdb.py create mode 100755 scripts/update-logs.sh diff --git a/lib/fsdb.py b/lib/fsdb.py new file mode 100644 index 0000000..56b4624 --- /dev/null +++ b/lib/fsdb.py @@ -0,0 +1,20 @@ +# DB-like with filesystem + +from buildbot.plugins import util, steps +from sumfiles import get_web_base +import os + +class SaveGDBResults (steps.MasterShellCommand): + name = 'save build results' + description = 'saving build results' + descriptionDone = 'saved build results' + + def __init__ (self, **kwargs): + steps.MasterShellCommand.__init__ (self, **kwargs) + self.command = [ os.path.expanduser ("~/scripts/update-logs.sh"), + "--commit", util.Property ('got_revision'), + "--builder", util.Property ('buildername'), + "--base-directory", get_web_base (), + "--branch", util.Property ('branch'), + "--is-try-sched", util.Property ('isTrySched', + default = 'no') ] diff --git a/lib/gdbcommand.py b/lib/gdbcommand.py index b6d8fa5..9eddcc9 100644 --- a/lib/gdbcommand.py +++ b/lib/gdbcommand.py @@ -1,67 +1,92 @@ # GDB .sum-fetching command. -from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION -from buildbot.steps.shell import ShellCommand +from buildbot.process.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION +from buildbot.plugins import steps, util from sumfiles import DejaResults, get_web_base from gdbgitdb import switch_to_branch -from shutil import copyfile +import os +import sqlite3 -class CopyOldGDBSumFile (ShellCommand): +@util.renderer +def create_copy_command (props): + rev = props.getProperty ('got_revision') + builder = props.getProperty ('buildername') + istry = props.getProperty ('isTrySched') + branch = props.getProperty ('branch') + command = [ 'cp', '-a' ] + + db_file = os.path.join (get_web_base (), builder, builder + '.db') + if not os.path.exists (db_file): + # This is probably the first commit being tested. Don't do anything. + return [ 'true' ] + + con = sqlite3.connect (db_file) + c = con.cursor () + c.execute ('SELECT commitid WHERE branch = "%s" AND trysched = 0 FROM logs ORDER BY timestamp DESC LIMIT 1' % branch) + con.close () + + commit = c.fetchone ()[0] + + from_path = os.path.join (get_web_base (), commit[:2], commit, 'gdb.sum') + if istry and istry == 'yes': + to_path = os.path.join (get_web_base (), 'try', rev[:2], rev, 'previous_gdb.sum') + else: + to_path = os.path.join (get_web_base (), rev[:2], rev, 'previous_gdb.sum') + + command += [ from_path, to_path ] + + return command + +class CopyOldGDBSumFile (steps.MasterShellCommand): """Copy the current gdb.sum file into the old_gdb.sum file.""" name = "copy gdb.sum file" description = "copying previous gdb.sum file" descriptionDone = "copied previous gdb.sum file" - command = [ 'true' ] def __init__ (self, **kwargs): - ShellCommand.__init__ (self, **kwargs) + steps.MasterShellCommand.__init__ (self, command = create_copy_command, **kwargs) - def evaluateCommand (self, cmd): - rev = self.getProperty('got_revision') - builder = self.getProperty('buildername') - isrebuild = self.getProperty ('isRebuild') - branch = self.getProperty('branch') - wb = get_web_base () - if branch is None: - branch = 'master' - - if isrebuild and isrebuild == 'yes': - return SUCCESS - - # Switch to the right branch inside the BUILDER repo - switch_to_branch (builder, branch, force_switch = True) - - try: - copyfile ("%s/%s/gdb.sum" % (wb, builder), - "%s/%s/previous_gdb.sum" % (wb, builder)) - except IOError: - # If the dest file does not exist, ignore - pass - - return SUCCESS - -class GdbCatSumfileCommand(ShellCommand): +class GdbCatSumfileCommand(steps.ShellCommand): name = 'regressions' command = ['cat', 'gdb.sum'] def __init__(self, **kwargs): - ShellCommand.__init__(self, **kwargs) + steps.ShellCommand.__init__(self, **kwargs) def evaluateCommand(self, cmd): rev = self.getProperty('got_revision') builder = self.getProperty('buildername') istrysched = self.getProperty('isTrySched') branch = self.getProperty('branch') + db_file = os.path.join (get_web_base (), builder, builder + '.db') + parser = DejaResults() + cur_results = parser.read_sum_text(self.getLog('stdio').getText()) + baseline = None + if branch is None: branch = 'master' - # Switch to the right branch inside the BUILDER repo - switch_to_branch (builder, branch, force_switch = False) + if not os.path.exists (db_file): + # This takes care of our very first build. + parser.write_sum_file (cur_results, builder, branch, rev) + # If there was no previous baseline, then this run + # gets the honor. + if baseline is None: + baseline = cur_results + parser.write_baseline (baseline, builder, branch, rev) + return SUCCESS - parser = DejaResults() - cur_results = parser.read_sum_text(self.getLog('stdio').getText()) - baseline = parser.read_baseline (builder, branch) - old_sum = parser.read_sum_file (builder, branch) + con = sqlite3.connect (db_file) + c = con.cursor () + c.execute ('SELECT commitid WHERE branch = "%s" AND trysched = 0 FROM logs ORDER BY timestamp DESC LIMIT 1' % branch) + con.close () + prevcommit = c.fetchone ()[0] + + # Switch to the right branch inside the BUILDER repo +# switch_to_branch (builder, branch, force_switch = False) + + baseline = parser.read_baseline (builder, branch, prevcommit) + old_sum = parser.read_sum_file (builder, branch, prevcommit) result = SUCCESS if baseline is not None: @@ -78,14 +103,14 @@ class GdbCatSumfileCommand(ShellCommand): self.addCompleteLog ('regressions', report) result = FAILURE - if not istrysched or istrysched == 'no': - parser.write_sum_file (cur_results, builder, branch) + if istrysched and istrysched == 'yes': + parser.write_try_build_sum_file (cur_results, builder, branch, rev) + else: + parser.write_sum_file (cur_results, builder, branch, rev) # If there was no previous baseline, then this run # gets the honor. if baseline is None: baseline = cur_results parser.write_baseline (baseline, builder, branch, rev) - else: - parser.write_try_build_sum_file (cur_results, builder, branch) return result diff --git a/lib/sumfiles.py b/lib/sumfiles.py index 220bb16..68e097d 100644 --- a/lib/sumfiles.py +++ b/lib/sumfiles.py @@ -6,6 +6,7 @@ from StringIO import StringIO # Necessary for ordered dictionaries. We use them when the order or # the tests matters to us. from collections import OrderedDict +import lzma # Helper regex for parse_sum_line. sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$') @@ -68,18 +69,16 @@ class DejaResults(object): # and to the set. out_dict[1][result].add (test_name) - def _write_sum_file(self, sum_dict, subdir, rev_or_branch, filename, - header = None): + def _write_sum_file(self, sum_dict, builder, rev, filename, header = None): global gdb_web_base - if not rev_or_branch: - bdir = os.path.join (gdb_web_base, subdir) - else: - bdir = os.path.join (gdb_web_base, subdir, rev_or_branch) - if not os.path.isdir (bdir): - os.makedirs (bdir, 0755) + + bdir = os.path.join (gdb_web_base, builder, rev[:2], rev) + if not os.path.exists (bdir): + os.makedirs (bdir) fname = os.path.join (bdir, filename) keys = sum_dict[0].keys () mode = 'w' + old_umask = os.umask (0133) if header: with open (fname, 'w') as f: f.write (header) @@ -87,16 +86,17 @@ class DejaResults(object): with open (fname, mode) as f: for k in keys: f.write (sum_dict[0][k] + ': ' + k + '\n') + os.umask (old_umask) - def write_sum_file(self, sum_dict, builder, branch): - self._write_sum_file (sum_dict, builder, None, 'gdb.sum') + def write_sum_file(self, sum_dict, builder, branch, rev): + self._write_sum_file (sum_dict, builder, rev, 'gdb.sum') - def write_try_build_sum_file (self, sum_dict, builder, branch): - self._write_sum_file (sum_dict, builder, None, 'trybuild_gdb.sum', + def write_try_build_sum_file (self, sum_dict, builder, branch, rev): + self._write_sum_file (sum_dict, builder, rev, 'trybuild_gdb.sum', header = "### THIS SUM FILE WAS GENERATED BY A TRY BUILD ###\n\n") def write_baseline(self, sum_dict, builder, branch, rev): - self._write_sum_file(sum_dict, builder, None, 'baseline', + self._write_sum_file(sum_dict, builder, rev, 'baseline', header = "### THIS BASELINE WAS LAST UPDATED BY COMMIT %s ###\n\n" % rev) # Read a .sum file. @@ -105,40 +105,46 @@ class DejaResults(object): # revision; to read the baseline file for a branch, use `read_baseline'. # Returns a dictionary holding the .sum contents, or None if the # file did not exist. - def _read_sum_file(self, subdir, rev_or_branch, filename, - is_racy_file = False): + def _read_sum_file(self, builder, branch, rev, filename, + is_racy_file = False, is_xfail_file = False): global gdb_web_base - if not rev_or_branch: - fname = os.path.join (gdb_web_base, subdir, filename) + + if is_xfail_file: + fname = os.path.join (gdb_web_base, builder, 'xfails', branch, filename) else: - fname = os.path.join (gdb_web_base, subdir, rev_or_branch, filename) + fname = os.path.join (gdb_web_base, builder, rev[:2], rev, filename) + result = [] + # result[0] is the OrderedDict containing all the tests + # and results. + result.append (OrderedDict ()) + # result[1] is a dictionary containing sets of tests + result.append (dict ()) + if os.path.exists (fname): - result = [] - # result[0] is the OrderedDict containing all the tests - # and results. - result.append (OrderedDict ()) - # result[1] is a dictionary containing sets of tests - result.append (dict ()) with open (fname, 'r') as f: for line in f: self.parse_sum_line (result, line, is_racy_file = is_racy_file) + elif os.path.exists (fname + '.xz'): + with lzma.open (fname, 'r') as f: + for line in f: + self.parse_sum_line (result, line, + is_racy_file = is_racy_file) else: - result = None + return None return result - def read_sum_file (self, builder, branch): - return self._read_sum_file (builder, None, 'gdb.sum') + def read_sum_file (self, builder, branch, rev): + return self._read_sum_file (builder, branch, rev, 'gdb.sum') - def read_baseline(self, builder, branch): - return self._read_sum_file (builder, None, 'baseline') + def read_baseline(self, builder, branch, rev): + return self._read_sum_file (builder, branch, rev, 'baseline') - def read_xfail (self, builder, branch): - return self._read_sum_file (builder, os.path.join ('xfails', branch), - 'xfail') + def read_xfail (self, builder, branch, rev): + return self._read_sum_file (builder, branch, rev, 'xfail', is_xfail_file = True) - def read_old_sum_file (self, builder, branch): - return self._read_sum_file (builder, None, 'previous_gdb.sum') + def read_old_sum_file (self, builder, branch, rev): + return self._read_sum_file (builder, branch, rev, 'previous_gdb.sum') # Parse some text as a .sum file and return the resulting # dictionary. diff --git a/master.cfg b/master.cfg index 8dd25f6..6f03f0b 100644 --- a/master.cfg +++ b/master.cfg @@ -15,17 +15,10 @@ from buildbot.schedulers.timed import Nightly from buildbot.schedulers.trysched import Try_Jobdir from buildbot.schedulers.forcesched import ForceScheduler from buildbot.process import factory -from buildbot.process.properties import WithProperties, Property -from buildbot.steps.shell import Compile -from buildbot.steps.shell import Configure -from buildbot.steps.shell import ShellCommand -from buildbot.steps.shell import SetPropertyFromCommand -from buildbot.steps.transfer import FileUpload -from buildbot.steps.source.git import Git -from buildbot.steps.slave import RemoveDirectory +from buildbot.plugins import util, reporters, steps, worker +from buildbot.reporters import utils from buildbot.changes.filter import ChangeFilter -from buildbot.buildslave import BuildSlave -from buildbot.status.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION +from buildbot.process.results import SUCCESS, WARNINGS, FAILURE, EXCEPTION from buildbot.interfaces import IEmailLookup from zope.interface import implements from gdbcommand import CopyOldGDBSumFile, GdbCatSumfileCommand @@ -34,10 +27,12 @@ from racyanalyze import GDBAnalyzeRacyTests from urllib import quote from sumfiles import DejaResults, set_web_base -import os.path +import os import urllib from json import load import re +import jinja2 +import git #################################### #################################### @@ -55,9 +50,8 @@ c = BuildmasterConfig = {} # Base directory for the web server. This is needed in order to # compare the test results. -gdb_web_base = os.path.expanduser (os.path.join (basedir, - 'public_html', - 'results')) +gdb_web_base = os.path.expanduser ("~/results/") + set_web_base (gdb_web_base) GDB_MAIL_FROM = 'sergiodj+buildbot@sergiodj.net' @@ -81,12 +75,12 @@ def should_watch_branch (branch): return False from buildbot.changes.gitpoller import GitPoller +master_git_repo = os.path.expanduser (os.path.join ('~/', 'buildbot-master-binutils-gdb')) c['change_source'] = [] -c['change_source'].append(GitPoller( - repourl = 'git://git.libreplanetbr.org/gdb.git', - workdir = os.path.expanduser (os.path.join ('~/', 'buildbot-master-binutils-gdb')), - branches = should_watch_branch, - pollinterval = 60 * 3)) +c['change_source'].append (GitPoller (repourl = 'git://git.libreplanetbr.org/gdb.git', + workdir = master_git_repo, + branches = should_watch_branch, + pollinterval = 60 * 3)) # 'status' is a list of Status Targets. The results of each build will be # pushed to these targets. buildbot/status/*.py has a variety to choose from, @@ -567,8 +561,7 @@ mn = MyMailNotifier(fromaddr = GDB_MAIL_FROM, messageFormatter = MessageGDBTesters, tags = [ "MAIL" ], extraHeaders = { 'X-GDB-Buildbot' : '1', - 'In-Reply-To' : WithProperties ("<%s@gdb-build>", - 'got_revision') }) + 'In-Reply-To' : util.Interpolate ("<%(prop:got_revision)s@gdb-build>")}) class LookupEmailTryBuild (object): implements (IEmailLookup) @@ -607,7 +600,7 @@ c['db'] = { ## the documentation on each build step class to understand what it ## does. -class CloneOrUpdateGDBMasterRepo (Git): +class CloneOrUpdateGDBMasterRepo (steps.Git): """This build step updates the so-called "master" git repository. For each buildslave, we have one master GDB git repository, which is then @@ -629,18 +622,18 @@ from the principal repository.""" name = "update gdb master repo" description = r"fetching GDB master sources" descriptionDone = r"fetched GDB master sources" - def __init__ (self): - Git.__init__ (self, - repourl = 'git://git.libreplanetbr.org/gdb.git', - workdir = WithProperties (r"%s/../binutils-gdb-master/", - r'builddir'), - retryFetch = True, - mode = r'incremental', - progress = True) + def __init__ (self, **kwargs): + steps.Git.__init__ (self, + repourl = 'git://git.libreplanetbr.org/gdb.git', + workdir = util.Interpolate ("%(prop:builddir)s/../binutils-gdb-master/"), + retryFetch = True, + mode = 'incremental', + progress = True, + **kwargs) self.haltOnFailure = False self.flunkOnFailure = False -class CloneOrUpdateGDBRepo (Git): +class CloneOrUpdateGDBRepo (steps.Git): """This build step is used to clone the GDB git repository that will be used by an specific builder (inside a buildslave). The trick here is to use the "reference" parameter to initialize the class, which @@ -649,24 +642,24 @@ present at the reference repository (i.e., locally).""" name = "clone gdb repo" description = "fetching GDB sources" descriptionDone = "fetched GDB sources" - def __init__ (self): - Git.__init__ (self, - repourl = 'git://git.libreplanetbr.org/gdb.git', - workdir = WithProperties ('%s/binutils-gdb/', 'builddir'), - reference = WithProperties ("%s/../binutils-gdb-master/", - 'builddir'), - retryFetch = True, - progress = True) + def __init__ (self, **kwargs): + steps.Git.__init__ (self, + repourl = 'git://git.libreplanetbr.org/gdb.git', + workdir = util.Interpolate ('%(prop:builddir)s/binutils-gdb/'), + reference = util.Interpolate ("%(prop:builddir)s/../binutils-gdb-master/"), + retryFetch = True, + progress = True, + **kwargs) -class ConfigureGDB (Configure): +class ConfigureGDB (steps.Configure): """This build step runs the GDB "configure" command, providing extra flags for it if needed.""" name = "configure gdb" description = r"configure GDB" descriptionDone = r"configured GDB" def __init__ (self, extra_conf_flags, **kwargs): - Configure.__init__ (self, **kwargs) - self.workdir = WithProperties (r"%s", r'builddir') + steps.Configure.__init__ (self, **kwargs) + self.workdir = util.Interpolate ('%(prop:builddir)s/build/') self.command = ['../binutils-gdb/configure', '--disable-binutils', '--disable-ld', @@ -675,7 +668,7 @@ flags for it if needed.""" '--disable-sim', '--disable-gprof'] + extra_conf_flags -class CompileGDB (Compile): +class CompileGDB (steps.Compile): """This build step runs "make" to compile the GDB sources. It provides extra "make" flags to "make" if needed. It also uses the "jobs" properties to figure out how many parallel jobs we can use when @@ -687,27 +680,27 @@ buildslave.""" descriptionDone = r"compiled GDB" def __init__ (self, make_command = 'make', extra_make_flags = [], **kwargs): - Compile.__init__ (self, **kwargs) - self.workdir = WithProperties (r"%s", r'builddir') + steps.Compile.__init__ (self, **kwargs) + self.workdir = util.Interpolate ('%(prop:builddir)s/build/') self.command = ['%s' % make_command, - WithProperties (r"-j%s", r'jobs'), + util.Interpolate ("-j%(prop:jobs)s"), 'all'] + extra_make_flags -class MakeTAGSGDB (ShellCommand): +class MakeTAGSGDB (steps.ShellCommand): name = 'make tags' description = 'running make TAGS' descriptionDone = 'ran make TAGS' def __init__ (self, **kwargs): - ShellCommand.__init__ (self, make_command = 'make', - **kwargs) - self.workdir = WithProperties ("%s/build/gdb", 'builddir') + steps.ShellCommand.__init__ (self, make_command = 'make', + **kwargs) + self.workdir = util.Interpolate ("%(prop:builddir)s/build/gdb") self.command = [ '%s' % make_command, 'TAGS' ] # We do not want to stop testing when this command fails. self.haltOnFailure = False self.flunkOnFailure = False self.flunkOnWarnings = False -class TestGDB (ShellCommand): +class TestGDB (steps.ShellCommand): """This build step runs the full testsuite for GDB. It can run in parallel mode (see BuildAndTestGDBFactory below), and it will also provide any extra flags for "make" if needed. Unfortunately, because @@ -718,12 +711,12 @@ BuildBot halt on failure.""" descriptionDone = r"tested GDB" def __init__ (self, make_command = 'make', extra_make_check_flags = [], test_env = {}, **kwargs): - ShellCommand.__init__ (self, decodeRC = { 0 : SUCCESS, - 1 : SUCCESS, - 2 : SUCCESS }, - **kwargs) + steps.ShellCommand.__init__ (self, decodeRC = { 0 : SUCCESS, + 1 : SUCCESS, + 2 : SUCCESS }, + **kwargs) - self.workdir = WithProperties (r"%s/build/gdb/testsuite", r'builddir') + self.workdir = util.Interpolate ("%(prop:builddir)s/build/gdb/testsuite") self.command = ['%s' % make_command, '-k', 'check'] + extra_make_check_flags @@ -734,7 +727,7 @@ BuildBot halt on failure.""" self.flunkOnFailure = False self.flunkOnWarnings = False -class TestRacyGDB (ShellCommand): +class TestRacyGDB (steps.ShellCommand): """This build step runs the full testsuite for GDB for racy testcases. It can run in parallel mode (see BuildAndTestGDBFactory below), and it will also provide any extra flags for "make" if needed. @@ -745,12 +738,12 @@ command must not make BuildBot halt on failure.""" descriptionDone = r"tested GDB (racy)" def __init__ (self, make_command = 'make', extra_make_check_flags = [], test_env = {}, **kwargs): - ShellCommand.__init__ (self, decodeRC = { 0 : SUCCESS, - 1 : SUCCESS, - 2 : SUCCESS }, - **kwargs) + steps.ShellCommand.__init__ (self, decodeRC = { 0 : SUCCESS, + 1 : SUCCESS, + 2 : SUCCESS }, + **kwargs) - self.workdir = WithProperties (r"%s/build/gdb/testsuite", r'builddir') + self.workdir = util.Interpolate ("%(prop:builddir)s/build/gdb/testsuite") self.command = ['%s' % make_command, '-k', 'check', @@ -762,26 +755,6 @@ command must not make BuildBot halt on failure.""" self.flunkOnFailure = False self.flunkOnWarnings = False -class CleanupBreakageLockfile (ShellCommand): - """Clean up (i.e., remove) the breakage lockfile for a specific builder.""" - name = "cleanup breakage lockfile" - description = "cleaning up breakage lockfile" - descriptionDone = "cleaned up breakage lockfile" - command = [ 'true' ] - - def __init__ (self, **kwargs): - ShellCommand.__init__ (self, **kwargs) - - def evaluateCommand (self, cmd): - builder = self.getProperty ('buildername') - branch = self.getProperty ('branch') - lockfile = make_breakage_lockfile_name (builder, branch) - - if os.path.isfile (lockfile): - os.remove (lockfile) - - return SUCCESS - def scheduler_is_racy (step): return step.getProperty ('scheduler').startswith ('racy') @@ -900,29 +873,28 @@ The parameters of the class are: # mjw asked me to delay the build by X number of seconds. if initial_delay: - self.addStep (ShellCommand (command = ['sleep', '%d' % initial_delay], - description = "delaying start of build by %d seconds" % initial_delay, - descriptionDone = "delayed start of build by %d seconds" % initial_delay)) + self.addStep (steps.ShellCommand (command = ['sleep', '%d' % initial_delay], + description = "delaying start of build by %d seconds" % initial_delay, + descriptionDone = "delayed start of build by %d seconds" % initial_delay)) - self.addStep (RemoveDirectory (dir = WithProperties (r"%s/build", - r'builddir'), - description = r"removing old build dir", - descriptionDone = r"removed old build dir")) - self.addStep (CloneOrUpdateGDBMasterRepo ()) + self.addStep (steps.RemoveDirectory (dir = util.Interpolate ("%(prop:builddir)s/build"), + description = "removing old build dir", + descriptionDone = "removed old build dir")) + self.addStep (CloneOrUpdateGDBMasterRepo (hideStepIf = True)) self.addStep (CloneOrUpdateGDBRepo ()) if self.run_testsuite: self.addStep (CopyOldGDBSumFile (doStepIf = scheduler_is_not_racy_try_do, - hideStepIf = scheduler_is_racy_try_hide)) + hideStepIf = True)) if not self.extra_conf_flags: self.extra_conf_flags = [] if self.enable_targets_all: - self.extra_conf_flags.append (r'--enable-targets=all') + self.extra_conf_flags.append ('--enable-targets=all') if self.use_system_debuginfo: - self.extra_conf_flags.append (r'--with-separate-debug-dir=/usr/lib/debug') + self.extra_conf_flags.append ('--with-separate-debug-dir=/usr/lib/debug') self.addStep (self.ConfigureClass (self.extra_conf_flags + architecture_triplet, haltOnFailure = True)) @@ -935,7 +907,9 @@ The parameters of the class are: # This last will be executed when the build succeeds. It is # needed in order to cleanup the breakage lockfile, if it # exists. - self.addStep (CleanupBreakageLockfile (hideStepIf = True)) + self.addStep (steps.MasterShellCommand (command = [ 'rm', '-f', + util.Interpolate ('/tmp/gdb-buildbot-breakage-report-%(prop:branch)s-%(prop:buildername)s') ], + hideStepIf = True)) # Disabling this until we figure out how to properly run + test # self.addStep (MakeTAGSGDB ()) @@ -948,7 +922,7 @@ The parameters of the class are: self.test_env = {} if self.test_parallel: - self.extra_make_check_flags.append (WithProperties (r"-j%s", r'jobs')) + self.extra_make_check_flags.append (util.Interpolate ("-j%(prop:jobs)s")) self.extra_make_check_flags.append (r'FORCE_PARALLEL=1') self.addStep (self.TestClass (self.make_command, self.extra_make_check_flags, @@ -956,17 +930,16 @@ The parameters of the class are: doStepIf = scheduler_is_not_racy_do, hideStepIf = scheduler_is_racy_hide)) - self.addStep (GdbCatSumfileCommand (workdir = WithProperties (r'%s/build/gdb/testsuite', - r'builddir'), - description = r'analyze test results', + self.addStep (GdbCatSumfileCommand (workdir = util.Interpolate ('%(prop:builddir)s/build/gdb/testsuite'), + description = 'analyzing test results', + descriptionDone = 'analyzed test results', doStepIf = scheduler_is_not_racy_do, hideStepIf = scheduler_is_racy_hide)) - self.addStep (FileUpload (slavesrc = WithProperties (r"%s/build/gdb/testsuite/gdb.log", - r'builddir'), - masterdest = WithProperties (r"public_html/results/%s/gdb.log", - r'buildername'), - doStepIf = scheduler_is_not_racy_do, - hideStepIf = scheduler_is_racy_hide)) + self.addStep (steps.FileUpload (workersrc = util.Interpolate ("%(prop:builddir)s/build/gdb/testsuite/gdb.log"), + masterdest = util.Interpolate ("public_html/results/%(prop:buildername)s/tmp/$(prop:got_revision)s/gdb.log"), + mode = 0644, + doStepIf = scheduler_is_not_racy_do, + hideStepIf = True)) self.addStep (SaveGDBResults (doStepIf = scheduler_is_not_racy_do, hideStepIf = scheduler_is_racy_hide)) @@ -977,8 +950,7 @@ The parameters of the class are: doStepIf = scheduler_is_racy_do, hideStepIf = scheduler_is_not_racy_hide)) - self.addStep (GDBAnalyzeRacyTests (workdir = WithProperties ('%s/build/gdb/testsuite', - 'builddir'), + self.addStep (GDBAnalyzeRacyTests (workdir = util.Interpolate ('%(prop:builddir)s/build/gdb/testsuite'), description = 'analyzing racy tests', descriptionDone = 'analyzed racy tests', doStepIf = scheduler_is_racy_do, @@ -1061,16 +1033,16 @@ class RunTestGDBNativeExtendedGDBServer_c32t32 (BuildAndTestGDBFactory): class RunTestGDBIndexBuild (BuildAndTestGDBFactory): """Testing with the "cc-with-tweaks.sh" passing -i.""" def __init__ (self, **kwargs): - self.extra_make_check_flags = [ WithProperties (r'CC_FOR_TARGET=/bin/sh %s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i gcc', r'builddir'), - WithProperties (r'CXX_FOR_TARGET=/bin/sh %s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i g++', r'builddir') ] + self.extra_make_check_flags = [ util.Interpolate ('CC_FOR_TARGET=/bin/sh %(prop:builddir)s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i gcc'), + util.Interpolate ('CXX_FOR_TARGET=/bin/sh %(prop:builddir)s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i g++') ] BuildAndTestGDBFactory.__init__ (self, **kwargs) class RunTestGDBIndexBuild_c32t32 (BuildAndTestGDBFactory): """Testing with the "cc-with-tweaks.sh" passing -i. 32-bit version""" def __init__ (self, **kwargs): self.extra_conf_flags = [ 'CFLAGS=-m32', 'CXXFLAGS=-m32' ] - self.extra_make_check_flags = [ WithProperties (r'CC_FOR_TARGET=/bin/sh %s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i gcc', r'builddir'), - WithProperties (r'CXX_FOR_TARGET=/bin/sh %s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i g++', r'builddir'), + self.extra_make_check_flags = [ util.Interpolate ('CC_FOR_TARGET=/bin/sh %(prop:builddir)s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i gcc'), + util.Interpolate ('CXX_FOR_TARGET=/bin/sh %(prop:builddir)s/binutils-gdb/gdb/contrib/cc-with-tweaks.sh -i g++'), 'RUNTESTFLAGS=--target_board unix/-m32' ] BuildAndTestGDBFactory.__init__ (self, **kwargs) diff --git a/scripts/update-logs.sh b/scripts/update-logs.sh new file mode 100755 index 0000000..3b647b5 --- /dev/null +++ b/scripts/update-logs.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +set -e + +umask u=rw,g=r,o=r + +usage () +{ + cat > /dev/stderr < /dev/stderr + exit 1 +} + +msg () +{ + local msg=$1 + + echo ">>> INFO: $msg" +} + +while test "$1" != "" ; do + case "$1" in + "-c"|"--commit") + COMMIT=$2 + shift 2 + ;; + "-b"|"--builder") + BUILDER=$2 + shift 2 + ;; + "-d"|"--base-directory") + BASE_DIR=$2 + shift 2 + ;; + "-h"|"--help") + usage + exit 0 + ;; + "-t"|"--is-try-sched") + IS_TRY_SCHED=$2 + shift 2 + ;; + "--branch") + BRANCH=$2 + shift 2 + ;; + *) + usage + exit 1 + ;; + esac +done + +DIR=$BASE_DIR/$BUILDER/ + +if test ! -d $DIR ; then + msg "$DIR is not a valid directory. Creeating it..." + mkdir -p $DIR +fi + +cd $DIR + +DB_NAME=$DIR/${BUILDER}.db + +if test ! -f $DB_NAME ; then + msg "Database $DB_NAME does not exist. Creating it..." + sqlite3 $DB_NAME "CREATE TABLE logs(commitid TEXT, branch TEXT DEFAULT 'master', trysched BOOLEAN DEFAULT 0, timestamp TIMESTAMP PRIMARY KEY DEFAULT (strftime('%s', 'now')) NOT NULL)" +fi + +COMMIT_2_DIG=`echo $COMMIT | sed 's/^\(..\).*$/\1/'` + +CDIR=$COMMIT_2_DIG/$COMMIT/ +ISTRY=0 +if test "$IS_TRY_SCHED" = "yes" ; then + CDIR=try/${CDIR} + ISTRY=1 +fi + +if test -d $CDIR ; then + # If this is a try build, the user is doing a rebuild. + # If this is a normal build, someone triggered a rebuild. + # Either way, we need to delete the current log dir. + msg "Log dir $CDIR already exists. Deleting it so that we can update the logs..." + rm --verbose -rf $CDIR +fi + +msg "Creating directory structure $CDIR..." +mkdir --verbose -p $CDIR +cd $CDIR + +TMP_DIR=$DIR/tmp/$COMMIT/ + +msg "Moving log files to $PWD..." +mv --verbose $TMP_DIR/* . +rmdir $TMP_DIR +msg "Compressing log files..." +xz --verbose --compress * + +PREV_COMMIT=`sqlite3 $DB_NAME "SELECT commitid FROM logs WHERE branch = '$BRANCH' AND trysched = 0 ORDER BY timestamp DESC LIMIT 1"` + +msg "Update database..." +sqlite3 $DB_NAME "INSERT INTO logs(commitid, branch, trysched) VALUES('$COMMIT', '$BRANCH', $ISTRY)" + +msg "Creating README.txt..." +cat > README.txt <