From 87d9106718458c00dd6e03f782ab11e3fee163dd Mon Sep 17 00:00:00 2001 From: Buildbot system user Date: Sat, 7 Mar 2020 17:39:01 +0000 Subject: Update configuration for updated buildbot --- master.cfg | 62 ++++++++++++++++++++++------------------------------- modulesetparser.py | 63 ++++++++++++++++++++++++++++++------------------------ 2 files changed, 61 insertions(+), 64 deletions(-) diff --git a/master.cfg b/master.cfg index abf8d67..13d311d 100755 --- a/master.cfg +++ b/master.cfg @@ -3,36 +3,32 @@ import os -from buildbot.buildslave import BuildSlave +from buildbot.worker import Worker from buildbot.revlinks import RevlinkMatch, RevlinkMultiplexer -from buildbot.status import html, words, mail -from buildbot.status.web import authz, auth +from buildbot.reporters import mail, irc from passwords import SlavePassword, IRCPassword -from gitmaildirsource import GitMaildirSource from modulesetparser import ChangeSourceList, SchedulerList, BuilderList # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. c = BuildmasterConfig = {} -####### BUILDSLAVES +####### BUILDWORKERS -# The 'slaves' list defines the set of recognized buildslaves. Each element is -# a BuildSlave object, specifying a unique slave name and password. The same -# slave name and password must be configured on the slave. +# The 'workers' list defines the set of recognized buildworkers. Each element is +# a BuildWorker object, specifying a unique worker name and password. The same +# worker name and password must be configured on the worker. -c['slaves'] = [BuildSlave("allegra", SlavePassword("allegra"), max_builds=1)] - -c['slavePortnum'] = 9989 +c['workers'] = [Worker("allegra", SlavePassword("allegra"), max_builds=1)] +c['protocols'] = {'pb': {'port': 9989}} ####### CHANGESOURCES c['change_source'] = [] -c['change_source'].append(GitMaildirSource('/var/lib/buildbot/Maildir')) c['change_source'].extend(ChangeSourceList()) -c['mergeRequests'] = False +c['collapseRequests'] = False ####### REVLINKS @@ -62,34 +58,24 @@ c['schedulers'].extend(SchedulerList()) c['builders'] = [] c['builders'].extend(BuilderList(['allegra'])) -####### STATUS TARGETS +####### WEB SERVER -# 'status' is a list of Status Targets. The results of each build will be -# pushed to these targets. buildbot/status/*.py has a variety to choose from, -# including web pages, email senders, and IRC bots. +# a web server which shows the build status -c['status'] = [] +c['www'] = { + 'port': 8010, + 'plugins': {'console_view': True}, + 'allowed_origins': ['http://medusa:8010'], +} -authz_cfg=authz.Authz( - # change any of these to True to enable; see the manual for more - # options - gracefulShutdown = False, - forceBuild = True, - forceAllBuilds = False, - pingBuilder = False, - stopBuild = True, - stopAllBuilds = False, - cancelPendingBuild = False, -) +####### REPORTERS -# a web server which shows the build status -c['status'].append(html.WebStatus(http_port=8010, authz=authz_cfg, order_console_by_time=True, - change_hook_dialects={'poller' : True})) +c['services'] = [] # an irc bot which reports build status -irc = words.IRC("chat.freenode.org", "medusabot", - channels=["cygwin-bots"], +irc = irc.IRC("chat.freenode.org", "medusabot", + channels=["#cygwin-bots"], password=IRCPassword(), notify_events={ 'exception': 1, @@ -98,11 +84,11 @@ irc = words.IRC("chat.freenode.org", "medusabot", }, allowForce=True) -c['status'].append(irc) +c['services'].append(irc) # email notification of status change -c['status'].append(mail.MailNotifier(fromaddr='jon.turney@dronecode.org.uk', +c['services'].append(mail.MailNotifier(fromaddr='jon.turney@dronecode.org.uk', sendToInterestedUsers=False, extraRecipients=['jon.turney@dronecode.org.uk'], mode='change', @@ -132,3 +118,7 @@ c['db'] = { # this at its default for all but the largest installations. 'db_url' : "sqlite:///state.sqlite", } + +###### USAGE REPORTING + +c['buildbotNetUsageData'] = 'basic' diff --git a/modulesetparser.py b/modulesetparser.py index fffe796..e98c37c 100755 --- a/modulesetparser.py +++ b/modulesetparser.py @@ -23,7 +23,8 @@ from buildbot.steps.shell import Configure,Compile,ShellCommand from buildbot.steps.trigger import Trigger from buildbot.steps.transfer import FileDownload from buildbot.config import BuilderConfig -from buildbot.status.results import SUCCESS,SKIPPED +from buildbot.status.builder import SUCCESS,SKIPPED +from buildbot.plugins import util tree = None repos = {} @@ -140,7 +141,7 @@ def parse(): # stash this node in a dict indexed by name repos[r.get('name')] = r - + if r.get('default') == 'yes': if default_repo: print("error: multiple default repository elements") @@ -181,7 +182,7 @@ def ChangeSourceList(): result = [] interval = 6000 # 100 minutes - # process modules of interest + # process modules of interest for i in modules: m = modules[i] b = m.find('branch') @@ -190,14 +191,15 @@ def ChangeSourceList(): repo = get_repo(b) if repo.get('type') == 'git': - result.append(GitPoller(repo.get('href') + b.get('module'), branches=['master'], + result.append(GitPoller(repourl=repo.get('href') + b.get('module'), + branches=['master'], workdir='gitpoller-work/' + m.get('id'), pollInterval=random.randint(int(interval*0.9), int(interval*1.10)))) # XXX: anongit.f.o doesn't like it if we poll 100 git repos in rapid succession (go figure!) # really we want to stagger the poll times, but buildbot doesn't have that feature, so instead # add some randomness to the timers, so all the pollers aren't synchronized. elif repo.get('type') == 'svn': - result.append(SVNPoller(repo.get('href') + repo.get('trunk-template'), + result.append(SVNPoller(repourl=repo.get('href') + repo.get('trunk-template'), pollInterval=interval, split_file=split_file_branches)) else: @@ -219,7 +221,7 @@ def SchedulerList(): # high-activity repositories get a tree-stable timer if m.get('id') in [ 'mesa-mesa' ]: timer = 300 # 5 minutes - print("%s %d" % (m.get('id'), timer)) +# print("%s %d" % (m.get('id'), timer)) else: timer = None @@ -240,7 +242,13 @@ def SchedulerList(): print("error: repo type %s unhandled" % repo.get('type')) # a scheduler to force building the module + codebase = util.CodebaseParameter(codebase='', + project=util.FixedParameter(name='project', default=''), + repository=repo_url, + branch='master', + revision='') result.append(ForceScheduler(name='force-' + m.get('id'), + codebases=[codebase], builderNames=[m.get('id')])) # a scheduler to trigger building the module when a dependency is rebuilt @@ -249,34 +257,33 @@ def SchedulerList(): return result -def BuilderList(slaves): +def BuilderList(workers): parse() - + result = [] for i in modules: m = modules[i] factory = BuildFactory() - # determine a category for the builder - # N.B. in buildbot 0.8.12 and later, builders can be given (potentially multiple) 'tags' rather than a single 'category' + # determine tags for the builder name = m.get('id') if name.startswith('app-'): - category = 'app' + tags = ['app'] elif name.startswith('data-'): - category = 'data' + tags = ['data'] elif name.startswith('font-'): - category = 'font' + tags = ['font'] elif name.endswith('proto'): - category = 'proto' + tags = ['proto'] elif name.startswith('lib'): - category = 'lib' + tags = ['lib'] elif name.startswith('mesa-'): - category = 'mesa' + tags = ['mesa'] elif name.startswith('xcb'): - category = 'xcb' + tags = ['xcb'] else: - category = 'misc' + tags = ['misc'] if name in [ 'mesa-mesa', 'xserver', 'xserver-meson' ]: incremental = True @@ -293,12 +300,12 @@ def BuilderList(slaves): # construct a build factory which knows how to build this repo - # prefix: the location on the buildslave where we stash the results of other builders + # prefix: the location on the buildworker where we stash the results of other builders # XXX: this is very wrong, as it means a builder is not self contained - prefix = '%(prop:workdir)s/../install' + prefix = '%(prop:builddir)s/../install' # construct env for build steps (maybe this should be sourced from a file or script?) - # XXX: need to run test on buildslave to determine if ccache is available, and if so, use it + # XXX: need to run test on buildworker to determine if ccache is available, and if so, use it env = {} env['CC'] = 'ccache gcc' env['CXX'] = 'ccache g++' @@ -311,7 +318,7 @@ def BuilderList(slaves): env['CXXFLAGS'] = '-g -O0 -fdiagnostics-show-option' env['AUTOMAKE'] = 'automake --force-missing --copy' #env['V'] = '1' - env['MAKEFLAGS'] = '-j4' # should depend on 'nproc' run on buildslave? + env['MAKEFLAGS'] = '-j4' # should depend on 'nproc' run on buildworker? env['PATH'] = Interpolate(os.path.join(prefix, 'bin') + ':${PATH}') env['PREFIX'] = Interpolate(prefix) @@ -336,11 +343,11 @@ def BuilderList(slaves): extra = [] if name in autogenargs: extra = autogenargs[name].split() - extra = map(Interpolate, extra) + extra = list(map(Interpolate, extra)) confcmd.extend(extra) # fontrootdir doesn't default to $prefix/share/fonts/ (bug?), so set it explicitly for font pacakages - if category == 'font': + if 'font' in tags: confcmd.append(Interpolate('--with-fontrootdir=' + os.path.join(prefix, 'share', 'fonts'))) if repo.get('type') == 'git': @@ -362,7 +369,7 @@ def BuilderList(slaves): patches = sorted(glob.glob('patches/' + name + '/*.patch')) for fn in patches: p = os.path.basename(fn) - factory.addStep(FileDownload(mastersrc=fn, slavedest=p, hideStepIf=lambda r,s: r == SUCCESS)) + factory.addStep(FileDownload(mastersrc=fn, workerdest=p, hideStepIf=lambda r,s: r == SUCCESS)) factory.addStep(ShellCommand(command=['sh', '-c', 'patch -p1 <' + p], name='patch', description='patching', descriptionDone='patch', logEnviron=logEnviron, haltOnFailure=False, flunkOnFailure=False)) factory.addStep(ShellCommand(command=['sh', '-c', 'rm ' + p], name='tidy', description='tidying', descriptionDone='tidy', hideStepIf=lambda r,s: r == SUCCESS, haltOnFailure=False, flunkOnFailure=False)) @@ -408,11 +415,11 @@ def BuilderList(slaves): result.append(BuilderConfig(name=name, builddir='builder-work/' + name, - slavenames=slaves, + workernames=workers, factory=factory, - category=category)) + tags=tags)) - return result + return result if __name__ == "__main__": print("%d changesources " % len(ChangeSourceList())) -- cgit v1.2.3