From 2dc344d03830a51d1017882549ea94ff273dbe57 Mon Sep 17 00:00:00 2001 From: Sergio Durigan Junior Date: Tue, 5 Sep 2017 09:43:15 -0400 Subject: [PATCH] Revamping the try_count approach --- lib/fsdb.py | 4 +++- lib/gdbcommand.py | 13 +++++-------- lib/sumfiles.py | 19 ++++++------------- master.cfg | 37 +++++++++++++++++++++++++------------ scripts/update-logs.sh | 7 +++++-- 5 files changed, 44 insertions(+), 36 deletions(-) diff --git a/lib/fsdb.py b/lib/fsdb.py index 9bdd4d1..0bb50a5 100644 --- a/lib/fsdb.py +++ b/lib/fsdb.py @@ -17,4 +17,6 @@ class SaveGDBResults (steps.MasterShellCommand): "--base-directory", get_web_base (), "--branch", util.Property ('branch'), "--is-try-sched", util.Property ('isTrySched', - default = 'no') ] + default = 'no'), + "--try-count", util.Property ('try_count', + default = '0') ] diff --git a/lib/gdbcommand.py b/lib/gdbcommand.py index a623901..cf15dde 100644 --- a/lib/gdbcommand.py +++ b/lib/gdbcommand.py @@ -39,13 +39,8 @@ def create_copy_command (props): return [ 'true' ] if istry and istry == 'yes': - con = sqlite3.connect (db_file) - c = con.cursor () - c.execute ('SELECT COUNT(*) FROM logs WHERE commitid = "%s" AND branch = "%s" AND trysched = 1' % (rev, branch)) - count = int (c.fetchone ()[0]) - con.close () - - to_path = os.path.join (get_web_base (), builder, 'try', rev[:2], rev, count) + try_count = props.getProperty ('try_count') + to_path = os.path.join (get_web_base (), builder, 'try', rev[:2], rev, try_count) else: to_path = os.path.join (get_web_base (), builder, rev[:2], rev) @@ -137,7 +132,9 @@ class GdbCatSumfileCommand(steps.ShellCommand): result = FAILURE if istry: - parser.write_try_build_sum_file (cur_results, builder, branch, rev) + try_count = self.getProperty ('try_count') + parser.write_try_build_sum_file (cur_results, builder, branch, rev, + try_count) else: parser.write_sum_file (cur_results, builder, branch, rev, istry) # If there was no previous baseline, then this run diff --git a/lib/sumfiles.py b/lib/sumfiles.py index b94f2e7..945cc88 100644 --- a/lib/sumfiles.py +++ b/lib/sumfiles.py @@ -7,7 +7,6 @@ from StringIO import StringIO # the tests matters to us. from collections import OrderedDict import lzma -import sqlite3 # Helper regex for parse_sum_line. sum_matcher = re.compile('^(.?(PASS|FAIL)): (.*)$') @@ -70,19 +69,12 @@ class DejaResults(object): # and to the set. out_dict[1][result].add (test_name) - def _write_sum_file(self, sum_dict, builder, rev, filename, header = None, istry = False, branch = "master"): + def _write_sum_file(self, sum_dict, builder, rev, filename, header = None, + istry = False, branch = "master", try_count = 0): global gdb_web_base if istry: - db_file = os.path.join (get_web_base (), builder, builder + '.db') - - con = sqlite3.connect (db_file) - c = con.cursor () - c.execute ('SELECT COUNT(*) FROM logs WHERE commitid = "%s" AND branch = "%s" AND trysched = 1' % (rev, branch)) - count = int (c.fetchone ()[0]) - con.close () - - bdir = os.path.join (gdb_web_base, builder, 'try', rev[:2], rev, count) + bdir = os.path.join (gdb_web_base, builder, 'try', rev[:2], rev, try_count) else: bdir = os.path.join (gdb_web_base, builder, rev[:2], rev) @@ -110,11 +102,12 @@ class DejaResults(object): self._write_sum_file (sum_dict, builder, rev, 'gdb.sum', istry = False, branch = branch) - def write_try_build_sum_file (self, sum_dict, builder, branch, rev): + def write_try_build_sum_file (self, sum_dict, builder, branch, rev, try_count): self._write_sum_file (sum_dict, builder, rev, 'trybuild_gdb.sum', header = "### THIS SUM FILE WAS GENERATED BY A TRY BUILD ###\n\n", istry = True, - branch = branch) + branch = branch, + try_count = try_count) def write_baseline(self, sum_dict, builder, branch, rev, istry): if istry: diff --git a/master.cfg b/master.cfg index cb1b0ec..37266a7 100644 --- a/master.cfg +++ b/master.cfg @@ -440,23 +440,17 @@ send to the gdb-testers mailing list.""" cur_change = sourcestamp.patch[1] properties = build.getProperties () isrebuild = properties.getProperty ('isRebuild') + try_count = properties.getProperty ('try_count') db_file = os.path.join (get_web_base (), name, name + '.db') - con = sqlite3.connect (db_file) - c = con.cursor () - c.execute ('SELECT COUNT(*) FROM logs WHERE commitid = "%s" AND branch = "%s" AND trysched = 1' % (sourcestamp.revision, branch)) - # We subtract one here because the "try" has been already stored - # into the db. - count = int (c.fetchone ()[0]) - 1 - con.close () - try_to = build.getReason ().strip ("'try' job by user ") # Sending the root message to gdb-testers. SendRootMessageGDBTesters (branch, cur_change, properties.getProperty ('revision'), - istrysched = True, try_to = try_to, try_count = count) + istrysched = True, try_to = try_to, + try_count = try_count) # Subject - subj = "Try Build #%d on %s, branch %s" % (count, name, branch) + subj = "Try Build #%d on %s, branch %s" % (try_count, name, branch) # Body text = "" @@ -477,7 +471,7 @@ send to the gdb-testers mailing list.""" # URL to find more info about what went wrong. text += "\nTestsuite log (gdb.sum and gdb.log) URL(s):\n" text += "\t<%s/%s/try/%s/%s/%d/>\n\n" % (res_url, name, sourcestamp.revision[:2], - sourcestamp.revision, count) + sourcestamp.revision, try_count) # commit_id = get_builder_commit_id (name, sourcestamp.revision, # sourcestamp.branch) @@ -611,7 +605,7 @@ mn_try = MyMailNotifier(fromaddr = GDB_MAIL_FROM, lookup = LookupEmailTryBuild (), tags = [ "MAIL", "TRY" ], extraHeaders = { 'X-GDB-Buildbot' : '1', - 'In-Reply-To' : util.Interpolate ("<%(prop:got_revision)s-try@gdb-build>")}) + 'In-Reply-To' : util.Interpolate ("<%(prop:got_revision)s-try-$(prop:try_count)s@gdb-build>")}) c['status'].append (mn) c['status'].append (mn_try) @@ -814,6 +808,21 @@ def scheduler_is_not_racy_do (step): def scheduler_is_not_racy_try_do (step): return scheduler_is_not_racy (step) and scheduler_is_not_try (step) +@util.renderer +def compute_try_build_count (props): + istry = props.getProperty ('scheduler').startswith ('try') + + if not istry: + return { "try_count" : "0" } + + con = sqlite3.connect (db_file) + c = con.cursor () + c.execute ('SELECT COUNT(*) FROM logs WHERE commitid = "%s" AND branch = "%s" AND trysched = 1' % (sourcestamp.revision, branch)) + count = c.fetchone ()[0] + con.close () + + return { "try_count" : count } + ####################### #### Build Factory #### ####################### @@ -917,6 +926,10 @@ The parameters of the class are: self.addStep (CloneOrUpdateGDBMasterRepo (hideStepIf = True)) self.addStep (CloneOrUpdateGDBRepo ()) + # Set the count of the try build + self.addStep (steps.SetProperties (properties = compute_try_build_count), + hideStepIf = scheduler_is_not_try) + if self.run_testsuite: self.addStep (CopyOldGDBSumFile (doStepIf = scheduler_is_not_racy_try_do, hideStepIf = False)) diff --git a/scripts/update-logs.sh b/scripts/update-logs.sh index 0e764af..36565c2 100755 --- a/scripts/update-logs.sh +++ b/scripts/update-logs.sh @@ -50,6 +50,10 @@ while test "$1" != "" ; do IS_TRY_SCHED=$2 shift 2 ;; + "--try-count") + TRY_COUNT=$2 + shift 2 + ;; "--branch") BRANCH=$2 shift 2 @@ -82,8 +86,7 @@ COMMIT_2_DIG=`echo $COMMIT | sed 's/^\(..\).*$/\1/'` CDIR=$COMMIT_2_DIG/$COMMIT/ ISTRY=0 if test "$IS_TRY_SCHED" = "yes" ; then - COUNT=`sqlite3 $DB_NAME "SELECT COUNT(*) FROM logs WHERE commitid = '${COMMIT}' AND trysched = 1"` - CDIR=try/${CDIR}/${COUNT} + CDIR=try/${CDIR}/${TRY_COUNT} ISTRY=1 fi