path: root/modulesetparser.py

#!/usr/bin/env python
#
# this reads a jhbuild moduleset file called 'xorg.modules' and generates a buildbot configuration
#

import glob
import os
import random

from collections import defaultdict
from lxml import etree
from buildbot.changes.gitpoller import GitPoller
from buildbot.changes.svnpoller import SVNPoller,split_file_branches
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.triggerable import Triggerable
from buildbot.changes.filter import ChangeFilter
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate
from buildbot.steps.source.git import Git
from buildbot.steps.source.svn import SVN
from buildbot.steps.shell import Configure,Compile,ShellCommand
from buildbot.steps.trigger import Trigger
from buildbot.steps.transfer import FileDownload
from buildbot.config import BuilderConfig
from buildbot.status.builder import SUCCESS,SKIPPED
from buildbot.plugins import util

tree = None
repos = {}
default_repo = None
rdep = defaultdict(list)
modules = {}

# XXX: needs to be arch-dependent
# skip building stuff which isn't appropriate for Windows target
skip = [ 'applewmproto',
         'libAppleWM',
         'xf86dgaproto',
         'xf86driproto',
         'xf86vidmodeproto',
         'dri2proto',
         'dri3proto',
         'libXv',
         'libXvMC',
         'libXxf86dga',
         'libXxf86dga',
         'libXxf86vm',
         'libxshmfence',
         'libxkbui',
         'libevdev',
         'libpciaccess',
         'mesa-drm',
         'mtdev',
         'app-xdriinfo',
         'app-xgamma',
         'app-xfs',
         'app-xvidtune',
         'app-xvinfo' ]

# skip building all drivers apart from xf86-video-dummy and xf86-video-nested
skip += [
         'glamor',
         'xf86-input-acecad',
         'xf86-input-aiptek',
         'xf86-input-evdev',
         'xf86-input-joystick',
         'xf86-input-keyboard',
         'xf86-input-libinput',
         'xf86-input-mouse',
         'xf86-input-synaptics',
         'xf86-input-vmmouse',
         'xf86-input-void',
         'xf86-video-apm',
         'xf86-video-ark',
         'xf86-video-ast',
         'xf86-video-ati',
         'xf86-video-chips',
         'xf86-video-cirrus',
         'xf86-video-fbdev',
         'xf86-video-geode',
         'xf86-video-glint',
         'xf86-video-i128',
         'xf86-video-i740',
         'xf86-video-intel',
         'xf86-video-mach64',
         'xf86-video-mga',
         'xf86-video-modesetting',
         'xf86-video-neomagic',
         'xf86-video-nouveau',
         'xf86-video-nv',
         'xf86-video-openchrome',
         'xf86-video-r128',
         'xf86-video-rendition',
         'xf86-video-s3',
         'xf86-video-s3virge',
         'xf86-video-savage',
         'xf86-video-siliconmotion',
         'xf86-video-sis',
         'xf86-video-sisusb',
         'xf86-video-suncg14',
         'xf86-video-suncg3',
         'xf86-video-suncg6',
         'xf86-video-sunffb',
         'xf86-video-sunleo',
         'xf86-video-suntcx',
         'xf86-video-tdfx',
         'xf86-video-tga',
         'xf86-video-trident',
         'xf86-video-tseng',
         'xf86-video-v4l',
         'xf86-video-vesa',
         'xf86-video-vmware',
         'xf86-video-wsfb',
         'xf86-video-voodoo',
         'xf86-video-xgixp' ]

def parse():
  global tree
  global default_repo

  # already parsed
  if tree:
    return

  # parse the XML moduleset
  tree = etree.parse('xorg.modules')

  # process any 'include' elements
  for r in tree.iter('include'):
    href = r.get('href')
    r[:] = etree.parse(href).getroot()

  # process 'repository' elements
  for r in tree.iter('repository'):
    t = r.get('type')
    n = r.get('name')
    d = r.get('default')
    h = r.get('href')
#    print("%s %s %s %s" % (t, n ,d , h))

    # stash this node in a dict indexed by name
    repos[r.get('name')] = r

    if r.get('default') == 'yes':
      if default_repo:
        print("error: multiple default repository elements")
      else:
        default_repo = r

  # process module elements
  for m in tree.iter('autotools', 'meson', 'metamodule'):
    #print("%s %s" % (m.get('id'), m))

    # ignore if on skip list
    if m.get('id') in skip:
      continue

    # stash module
    modules[m.get('id')] = m

    # store depedencies
    for d in m.iter('dependencies'):
      for p in m.iter('dep'):
        rdep[p.get('package')].append(m.get('id'))

  return

def get_repo(b):
#    print("%s - %s" % (b.get('module'), b.get('checkoutdir')))
  r = b.get('repo')
  if not r:
    repo = default_repo
  else:
    repo = repos[r]

  return repo

def is_incremental(name):
  return name in [ 'mesa-mesa', 'xserver', 'xserver-meson' ]

def ChangeSourceList():
  parse()

  result = []
  interval = 6000 # 100 minutes

  # process modules of interest
  for i in modules:
    m = modules[i]
    b = m.find('branch')
    if b is not None:
      # if a specific commit is specified, we don't need to poll
      if b.get('tag', None):
         continue

      #print("%s %s %s" % (m.get('id'), m, b))
      repo = get_repo(b)

      if repo.get('type') == 'git':
        result.append(GitPoller(repourl=repo.get('href') + b.get('module'),
                                branches=['master'],
                                workdir='gitpoller-work/' + m.get('id'),
                                pollInterval=random.randint(int(interval*0.9), int(interval*1.10))))
        # XXX: anongit.f.o doesn't like it if we poll 100 git repos in rapid succession (go figure!)
        # really we want to stagger the poll times, but buildbot doesn't have that feature, so instead
        # add some randomness to the timers, so all the pollers aren't synchronized.
      elif repo.get('type') == 'svn':
        result.append(SVNPoller(repourl=repo.get('href') + repo.get('trunk-template'),
                                pollInterval=interval,
                                split_file=split_file_branches))
      else:
        print("error: repo type %s unhandled" % repo.get('type'))

  return result

def SchedulerList():
  parse()

  result = []

  for i in modules:
    m = modules[i]
    b = m.find('branch')
    if b is not None:
      repo = get_repo(b)

      # high-activity repositories get a tree-stable timer
      if m.get('id') in [ 'mesa-mesa' ]:
        timer = 300 # 5 minutes
#        print("%s %d" % (m.get('id'), timer))
      else:
        timer = None

      # a scheduler which builds the repo when changes occur in that repo
      if repo.get('type') == 'git':
        repo_url = repo.get('href') + b.get('module')
        filter = ChangeFilter(repository=repo_url, branch='master')
#        print("filter on repo %s" % repo_url)
        result.append(SingleBranchScheduler(
                              name=m.get('id'),
                              change_filter=filter,
                              treeStableTimer=timer,
                              builderNames=[m.get('id')]))
      elif repo.get('type') == 'svn':
        repo_url = repo.get('href') + repo.get('trunk-template')
        filter = ChangeFilter(repository=repo_url, branch='trunk')
      else:
        print("error: repo type %s unhandled" % repo.get('type'))

      # a scheduler to force building the module
      codebase = util.CodebaseParameter(codebase='',
                                        project=util.FixedParameter(name='project', default=''),
                                        repository=repo_url,
                                        branch='master',
                                        revision='')

      custom_properties = []
      if is_incremental(m.get('id')):
        custom_properties += [util.BooleanParameter(name="configure",
                                                    label="run configuration step",
                                                    default=False)]

      result.append(ForceScheduler(name='force-' + m.get('id'),
                                 codebases=[codebase],
                                 builderNames=[m.get('id')],
                                 properties=custom_properties))

    # a scheduler to trigger building the module when a dependency is rebuilt
#    result.append(Triggerable(name='triggerable-' + m.get('id'),
#                              builderNames=[m.get('id')]))

  return result

def BuilderList(workers):
  parse()

  result = []

  for i in modules:
    m = modules[i]
    factory = BuildFactory()

    # determine tags for the builder
    name = m.get('id')
    if name.startswith('app-'):
      tags = ['app']
    elif name.startswith('data-'):
      tags = ['data']
    elif name.startswith('font-'):
      tags = ['font']
    elif name.endswith('proto'):
      tags = ['proto']
    elif name.startswith('lib'):
      tags = ['lib']
    elif name.startswith('mesa-'):
      tags = ['mesa']
    elif name.startswith('xcb'):
      tags = ['xcb']
    else:
      tags = ['misc']

    incremental = is_incremental(name)

    # log environment if the logEnviron property is present ?
#    logEnviron = Interpolate('%(prop:logEnviron:-False)s')
    logEnviron = False

    b = m.find('branch')
    if b is not None:
      repo = get_repo(b)

      # construct a build factory which knows how to build this repo

      # prefix: the location on the buildworker where we stash the results of other builders
      # XXX: this is very wrong, as it means a builder is not self contained
      prefix = '%(prop:builddir)s/../install'

      # construct env for build steps (maybe this should be sourced from a file or script?)
      # XXX: need to run test on buildworker to determine if ccache is available, and if so, use it
      env = {}
      env['CC'] = 'ccache gcc'
      env['CXX'] = 'ccache g++'
      env['CC_FOR_BUILD'] = env['CC']
      env['ACLOCAL'] = Interpolate('aclocal -I ' + os.path.join(prefix, 'share', 'aclocal'))
      #env['PKG_CONFIG_DEBUG_SPEW'] = '1'
      env['PKG_CONFIG_PATH'] = Interpolate(os.path.join(prefix, 'lib', 'pkgconfig') + ':' + os.path.join(prefix, 'share', 'pkgconfig') + ':${PKG_CONFIG_PATH}')
      env['INSTALL']  = '/usr/local/bin/install-check'
      env['CFLAGS'] = '-g -O0 -fdiagnostics-show-option'
      env['CXXFLAGS'] = '-g -O0 -fdiagnostics-show-option'
      env['AUTOMAKE'] = 'automake --force-missing --copy'
      #env['V'] = '1'
      env['MAKEFLAGS'] = '-j4' # should depend on 'nproc' run on buildworker?
      env['PATH'] = Interpolate(os.path.join(prefix, 'bin') + ':${PATH}')
      env['PREFIX'] = Interpolate(prefix)
      env['PYTHONUNBUFFERED'] = '1'  # make meson output appear while it's running

      if name == 'libpixman':
        # pixman tests run slower when OpenMP is allowed to use multiple threads. I'm assuming this is a toolchain bug...
        os.environ['OMP_NUM_THREADS'] = '1'

      # construct the configuration command
      # some modules require additional configuration options
      confcmd = ['./configure', '--config-cache', '--enable-silent-rules', Interpolate('--prefix='+prefix)]

      autogenargs = {
                      'xserver'          : '--enable-xvfb --enable-xnest --enable-dmx --enable-kdrive --enable-xephyr --enable-xfake --disable-xfbdev --enable-xorg --enable-xwin --enable-xf86bigfont --enable-debug --disable-glamor --disable-xf86-input-inputtest',
                      'mesa-demos'       : '--with-glut='+prefix,
                      'xf86-video-dummy' : '--disable-dga', # config check needs fixing to default to auto, not yes
                      'xts'              : '--disable-shared',
                      'fontconfig'       : '--with-default-fonts=' + os.path.join(prefix, 'share', 'fonts'),
                      'xserver-meson'    : '-Dudev=false -Dudev_kms=false -Dpciaccess=false -Dint10=false -Dsystemd_logind=false -Ddmx=true -Dxephyr=true -Dglamor=false -Ddefault_font_path=catalogue:/etc/X11/fontpath.d,built-ins -Dxkb_default_rules=base -Dxf86-input-inputtest=false',
                      'mesa-mesa'  : '-Degl=false -Dbuild-tests=true',
                    }

      extra = []
      if name in autogenargs:
         extra = autogenargs[name].split()
         extra = list(map(Interpolate, extra))
         confcmd.extend(extra)

      # fontrootdir doesn't default to $prefix/share/fonts/ (bug?), so set it explicitly for font pacakages
      if 'font' in tags:
         confcmd.append(Interpolate('--with-fontrootdir=' + os.path.join(prefix, 'share', 'fonts')))

      if repo.get('type') == 'git':
        repo_url = repo.get('href') + b.get('module')
        if incremental:
           factory.addStep(Git(repourl=repo_url, logEnviron=logEnviron, submodules=True, mode='incremental'))
        else:
           factory.addStep(Git(repourl=repo_url, logEnviron=logEnviron, submodules=True, mode='full', method='fresh'))
      elif repo.get('type') == 'svn':
        repo_url = repo.get('href') + repo.get('trunk-template')
        if incremental:
           factory.addStep(SVN(repourl=repo_url, logEnviron=logEnviron, mode='incremental'))
        else:
           factory.addStep(SVN(repourl=repo_url, logEnviron=logEnviron, mode='full', method='fresh'))
      else:
        print("error: repo type %s unhandled" % repo.get('type'))

      # some repos need patches (which are not yet upstreamable) applied
      patches = sorted(glob.glob('patches/' + name + '/*.patch'))
      for fn in patches:
        p = os.path.basename(fn)
        factory.addStep(FileDownload(mastersrc=fn, workerdest=p, hideStepIf=lambda r,s: r == SUCCESS))
        factory.addStep(ShellCommand(command=['sh', '-c', 'patch -p1 <' + p], name='patch', description='patching', descriptionDone='patch', logEnviron=logEnviron, haltOnFailure=False, flunkOnFailure=False))
        factory.addStep(ShellCommand(command=['sh', '-c', 'rm ' + p], name='tidy', description='tidying', descriptionDone='tidy', hideStepIf=lambda r,s: r == SUCCESS, haltOnFailure=False, flunkOnFailure=False))

      # incremental builds only configure if property 'configure' is present and True
      def force_configure(step):
        return step.getProperty("configure", False)

      if m.tag == 'autotools':

        if (not incremental) and (m.get('skip-autogen') != 'true'):
          factory.addStep(Configure(command=['sh', '-c', 'NOCONFIGURE=1 ./autogen.sh'], name='autoreconf', description='autoreconfing', descriptionDone='autoreconf', logEnviron=logEnviron, env=env))
          factory.addStep(Configure(command=confcmd, logEnviron=logEnviron, env=env))
        elif incremental:
          factory.addStep(Configure(command=['sh', '-c', 'NOCONFIGURE=1 ./autogen.sh'], name='autoreconf', description='autoreconfing', descriptionDone='autoreconf', logEnviron=logEnviron, env=env, doStepIf=force_configure, hideStepIf=lambda results, s: results==SKIPPED))
          factory.addStep(Configure(command=confcmd, logEnviron=logEnviron, env=env, logfiles={'log' : 'config.log'}, doStepIf=force_configure, hideStepIf=lambda results, s: results==SKIPPED))

        factory.addStep(Compile(logEnviron=logEnviron, env=env))

        if not (m.get('check-target') == 'false'):
          factory.addStep(Compile(command=['make', 'check'], name='test', description='testing', descriptionDone='test', logEnviron=logEnviron, env=env, timeout=3600))

        factory.addStep(Compile(command=['make', 'install'], name='install', description='installing', descriptionDone='install', logEnviron=logEnviron, env=env))

      elif m.tag == 'meson':

        if not incremental:
            factory.addStep(Configure(command=['meson', '--prefix', Interpolate(prefix), 'build'] + extra, name='meson', description='meson', descriptionDone='meson', logEnviron=logEnviron, env=env))
        else:
            factory.addStep(Configure(command=['rm', '-rf', 'build'], name='wipe', description='wipe', descriptionDone='wipe', logEnviron=logEnviron, env=env, doStepIf=force_configure, hideStepIf=lambda results, s: results==SKIPPED))
            factory.addStep(Configure(command=['meson', '--prefix', Interpolate(prefix), 'build'] + extra, name='meson', description='meson', descriptionDone='meson', logEnviron=logEnviron, env=env, doStepIf=force_configure, hideStepIf=lambda results, s: results==SKIPPED))

        factory.addStep(Compile(command=['ninja', '-C', 'build'], logEnviron=logEnviron, env=env))
        factory.addStep(Compile(command=['meson', 'test', '-C', 'build', '--timeout-multiplier=4'], name='test', description='testing', descriptionDone='test', logEnviron=logEnviron, env=env, timeout=3600))
        factory.addStep(Compile(command=['ninja', '-C', 'build', 'install'], name='install', description='installing', descriptionDone='install', logEnviron=logEnviron, env=env))

    # add build step to trigger dependent modules (if any)
#    print("%s rdeps are %s" % (e.get('id'), rdep[e.get('id')]))
#    if len(rdep[name]):
#      factory.addStep(Trigger(schedulerNames=['triggerable-' + i for i in rdep[name] ],
#                              alwaysUseLatest=True))

      result.append(BuilderConfig(name=name,
                                builddir='builder-work/' + name,
                                workernames=workers,
                                factory=factory,
                                tags=tags))

  return result

if __name__ == "__main__":
    print("%d changesources " % len(ChangeSourceList()))
    print("%d schedulers " % len(SchedulerList()))
    print("%d builders" % len(BuilderList('')))