Initial commitHEADmaster

As a start, this gives us:
  - generation of base per-platform images (i.e. with toolchain)
  - separate images to add build dependencies
  - jobs using these separate images to build projects in containers

Initial implementations for CentOS 7, Fedora 24, Debian Jessie (8),
Ubuntu 14.04 (Trusty), and Ubuntu 16.04 (Xenial).

Fedora 24 is currently disabled in the 'all' jobs, as the container
unexpectedly takes a dive when running autoreconf.

Signed-off-by: Daniel Stone <daniels@collabora.com>

25 files changed, 378 insertions, 0 deletions

diff --git a/base-images/base-image-axes.yaml b/base-images/base-image-axes.yaml
new file mode 100644
index 0000000..8f1effb
--- /dev/null
+++ b/base-images/base-image-axes.yaml
@@ -0,0 +1,13 @@
+name: base-image-axes
+platform:
+  - centos-7
+  - fedora-24
+  - debian-jessie
+  - ubuntu-14.04
+  - ubuntu-16.04
+arch:
+  - x86-64
+exclude-from-all:
+  - exclusion:
+      platform: fedora-24
+      arch: x86-64
diff --git a/base-images/base-images-all.jpl b/base-images/base-images-all.jpl
new file mode 100644
index 0000000..5cc3914
--- /dev/null
+++ b/base-images/base-images-all.jpl
@@ -0,0 +1,39 @@
+def expanded = [:]
+def excluded = []
+
+my_str = readFile(file: "base-images/base-image-axes.yaml")
+
+yaml = new org.yaml.snakeyaml.Yaml()
+axis_config = yaml.load(my_str)
+yaml = null
+
+/* Sadly we can't use the map.each {} iterator in pipeline scripts yet ... */
+exclusions = axis_config["exclude-from-all"]
+for (int i = 0; exclusions && i < exclusions.size(); i++) {
+  exclusion = exclusions[i]["exclusion"]
+  platform = exclusion["platform"]
+  arch = exclusion["arch"]
+  excluded << "${platform}-${arch}"
+  println "excluding ${platform}-${arch}"
+}
+
+platform_list = axis_config["platform"]
+arch_list = axis_config["arch"]
+
+for (int i = 0; i < platform_list.size(); i++) {
+  def platform = platform_list[i]
+
+  for (int j = 0; j < arch_list.size(); j++) {
+    def arch = arch_list[j]
+    def name = "${platform}-${arch}"
+    if (excluded.contains(name)) {
+      continue;
+    }
+
+    expanded[name] = {
+      build(job: "docker-images/base-images/${name}", propagate: true, wait: true)
+    }
+  }
+}
+
+parallel expanded
diff --git a/base-images/base-images.jjb b/base-images/base-images.jjb
new file mode 100644
index 0000000..265e644
--- /dev/null
+++ b/base-images/base-images.jjb
@@ -0,0 +1,54 @@
+- job-template:
+    name: docker-images/base-images/all
+    defaults: global
+    project-type: pipeline
+    properties:
+      - inject:
+          properties-content: |
+            JOB_DEFINITION_GIT={job-definition-git}
+            JOB_DEFINITION_GIT_BRANCH={job-definition-git-branch}
+    pipeline:
+      script: |
+        node('docker') {{
+          git(url: env.JOB_DEFINITION_GIT, branch: env.JOB_DEFINITION_GIT_BRANCH, poll: false)
+          load "base-images/base-images-all.jpl"
+        }}
+      sandbox: false
+
+- job-template:
+    name: docker-images/base-images/{platform}-{arch}
+    defaults: global
+    project-type: pipeline
+    properties:
+      - inject:
+          properties-content: |
+            PLATFORM={platform}
+            ARCH={arch}
+            DOCKER_REGISTRY_URI_SCHEME={docker-registry-uri-scheme}
+            DOCKER_REGISTRY={docker-registry}
+            DOCKER_REGISTRY_CREDENTIALS={docker-registry-credentials}
+            JOB_DEFINITION_GIT={job-definition-git}
+            JOB_DEFINITION_GIT_BRANCH={job-definition-git-branch}
+    pipeline:
+      # If we run a pipeline definition directly from SCM, it's always
+      # sandboxed, which means it can do ~nothing. Work around this by
+      # loading the script manually.
+      script: |
+        node('docker') {{
+          git(url: env.JOB_DEFINITION_GIT, branch: env.JOB_DEFINITION_GIT_BRANCH, poll: false)
+          load "base-images/build-image.jpl"
+        }}
+      sandbox: false
+
+- project:
+    name: docker-images/base-images
+    defaults: base-image-axes
+    jobs:
+      - docker-images/base-images/{platform}-{arch}
+      - docker-images/base-images/all
+
+- defaults:
+    !include: base-images/base-image-axes.yaml
+
+- defaults:
+    !include: defaults.yaml
diff --git a/base-images/build-image.jpl b/base-images/build-image.jpl
new file mode 100644
index 0000000..a715b66
--- /dev/null
+++ b/base-images/build-image.jpl
@@ -0,0 +1,30 @@
+def stamp = sh(script: "date '+%Y%m%d%H%M'", returnStdout: true).trim()
+def img_name = "${env.PLATFORM}-base-${env.ARCH}"
+def dir_name = "${img_name}-${stamp}"
+def registry = "${env.DOCKER_REGISTRY_URI_SCHEME}://${env.DOCKER_REGISTRY}"
+
+stage("Generating Dockerfile") {
+  sh(script: "./base-images/gen-dockerfile.sh ${env.PLATFORM} ${env.ARCH} ${dir_name}")
+}
+
+def img_base
+
+stage("Building base image") {
+  docker.withRegistry(registry, env.DOCKER_REGISTRY_CREDENTIALS) {
+    /*
+     * We use --no-cache, as Docker sees steps like 'apt-get update' as being
+     * idempotent, rather than temporally-bound.
+     */
+    img_base = docker.build("${img_name}:${stamp}", "--no-cache ${dir_name}")
+  }
+}
+
+stage("Pushing base image") {
+  docker.withRegistry(registry, env.DOCKER_REGISTRY_CREDENTIALS) {
+    img_base.push()
+  }
+}
+
+stage("Building Wayland image") {
+  def img_wl = build(job: "docker-images/wayland-images/${env.PLATFORM}-${env.ARCH}",   parameters: [string(name: "STAMP", value: stamp)], propagate: true, wait: true)
+}
diff --git a/base-images/centos-7/Dockerfile.template b/base-images/centos-7/Dockerfile.template
new file mode 100644
index 0000000..63182d1
--- /dev/null
+++ b/base-images/centos-7/Dockerfile.template
@@ -0,0 +1,26 @@
+FROM @@BASE@@
+MAINTAINER Daniel Stone <daniels@collabora.com>
+
+# Pull the latest updates, if any
+RUN yum clean expire-cache
+RUN yum -y update
+
+# Get a basic toolchain together
+RUN yum install -y gcc glibc-devel autoconf automake libtool intltool git make
+
+# Add a user for CI
+RUN mkdir /scratch
+RUN adduser -p '!!' -m --home /scratch/jenkins --shell /bin/bash --comment 'Jenkins build user' jenkins-build
+VOLUME "/scratch/jenkins"
+
+# git-phab
+RUN yum install python
+
+# install pip, argcomplete, GitPython, python-phabricator
+
+WORKDIR /tmp
+RUN git clone git://anongit.freedesktop.org/git/git-phab
+RUN cp git-phab/git-phab /usr/bin/
+
+# Clean up after ourselves
+RUN yum clean packages
diff --git a/base-images/debian-jessie/Dockerfile.template b/base-images/debian-jessie/Dockerfile.template
new file mode 100644
index 0000000..acb913a
--- /dev/null
+++ b/base-images/debian-jessie/Dockerfile.template
@@ -0,0 +1,29 @@
+FROM @@BASE@@
+MAINTAINER Daniel Stone <daniels@collabora.com>
+
+# Don't bother us
+ENV DEBIAN_FRONTEND noninteractive
+COPY apt-dpkg-config-updates.conf /etc/apt/apt.conf.d/90-dpkg-config-updates.conf
+COPY apt-force-yes.conf /etc/apt/apt.conf.d/90-force-yes.conf
+COPY apt-no-recommends.conf /etc/apt/apt.conf.d/90-no-recommends.conf
+COPY apt-retry-lots.conf /etc/apt/apt.conf.d/90-retry-lots.conf
+
+# Pull the latest updates, if any
+COPY apt-sources.list /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get dist-upgrade
+
+# Get a basic toolchain together
+RUN apt-get install build-essential autoconf automake libtool intltool git-core
+
+# Add a user for CI
+RUN adduser --disabled-password --home /scratch/jenkins --shell /bin/bash --gecos 'Jenkins build user' jenkins-build
+VOLUME "/scratch/jenkins"
+
+# git-phab
+RUN apt-get install python3 python3-argcomplete python3-appdirs python3-pip python3-git/jessie-backports
+RUN pip3 install phabricator
+
+WORKDIR /tmp
+RUN git clone git://anongit.freedesktop.org/git/git-phab
+RUN cp git-phab/git-phab /usr/bin/
diff --git a/base-images/debian-jessie/apt-dpkg-config-updates.conf b/base-images/debian-jessie/apt-dpkg-config-updates.conf
new file mode 100644
index 0000000..a8a34ba
--- /dev/null
+++ b/base-images/debian-jessie/apt-dpkg-config-updates.conf
@@ -0,0 +1,2 @@
+Dpkg::Options::="--force-confdef";
+Dpkg::Options::="--force-confnew";
diff --git a/base-images/debian-jessie/apt-force-yes.conf b/base-images/debian-jessie/apt-force-yes.conf
new file mode 100644
index 0000000..1ab128d
--- /dev/null
+++ b/base-images/debian-jessie/apt-force-yes.conf
@@ -0,0 +1,2 @@
+APT::Get::Assume-Yes "true";
+APT::Get::force-yes "true";
diff --git a/base-images/debian-jessie/apt-no-recommends.conf b/base-images/debian-jessie/apt-no-recommends.conf
new file mode 100644
index 0000000..d12154b
--- /dev/null
+++ b/base-images/debian-jessie/apt-no-recommends.conf
@@ -0,0 +1,2 @@
+APT::Get::Install-Recommends "false";
+APT::Get::Install-Suggests "false";
diff --git a/base-images/debian-jessie/apt-retry-lots.conf b/base-images/debian-jessie/apt-retry-lots.conf
new file mode 100644
index 0000000..86f8298
--- /dev/null
+++ b/base-images/debian-jessie/apt-retry-lots.conf
@@ -0,0 +1 @@
+Acquire::Retries "20";
diff --git a/base-images/debian-jessie/apt-sources.list b/base-images/debian-jessie/apt-sources.list
new file mode 100644
index 0000000..3364a39
--- /dev/null
+++ b/base-images/debian-jessie/apt-sources.list
@@ -0,0 +1,11 @@
+deb http://httpredir.debian.org/debian jessie main
+deb-src http://httpredir.debian.org/debian jessie main
+
+deb http://httpredir.debian.org/debian jessie-updates main
+deb-src http://httpredir.debian.org/debian jessie-updates main
+
+deb http://httpredir.debian.org/debian jessie-backports main
+deb-src http://httpredir.debian.org/debian jessie-backports main
+
+deb http://security.debian.org/ jessie/updates main
+deb-src http://security.debian.org/ jessie/updates main
diff --git a/base-images/fedora-24/Dockerfile.template b/base-images/fedora-24/Dockerfile.template
new file mode 100644
index 0000000..2ad22cd
--- /dev/null
+++ b/base-images/fedora-24/Dockerfile.template
@@ -0,0 +1,24 @@
+FROM @@BASE@@
+MAINTAINER Daniel Stone <daniels@collabora.com>
+
+# Pull the latest updates, if any
+RUN dnf --refresh -y update
+
+# Get a basic toolchain together
+RUN dnf install -y gcc glibc-devel autoconf automake libtool intltool git make
+
+# Add a user for CI
+RUN mkdir /scratch
+RUN adduser -p '!!' -m --home /scratch/jenkins --shell /bin/bash --comment 'Jenkins build user' jenkins-build
+VOLUME "/scratch/jenkins"
+
+# git-phab
+RUN dnf install python3 python3-argcomplete python3-appdirs python3-pip
+RUN pip install GitPython
+RUN pip install phabricator
+WORKDIR /tmp
+RUN git clone git://anongit.freedesktop.org/git/git-phab
+RUN cp git-phab/git-phab /usr/bin/
+
+# Clean up after ourselves
+RUN dnf clean packages
diff --git a/base-images/gen-dockerfile.sh b/base-images/gen-dockerfile.sh
new file mode 100755
index 0000000..8ab4b3c
--- /dev/null
+++ b/base-images/gen-dockerfile.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+
+set -e
+
+BASEDIR=$(dirname $0)
+
+PLATFORM=$1
+shift
+ARCH=$1
+shift
+DEST=$1
+shift
+
+case "${PLATFORM}:${ARCH}" in
+  centos-7:armv7)
+    exit 1
+    ;;
+  centos-7:x86-64)
+    BASE="centos:7"
+    ;;
+  debian-jessie:armv7)
+    BASE="armhf/debian:jessie"
+    ;;
+  debian-jessie:x86-64)
+    BASE="debian:jessie"
+    ;;
+  fedora-24:armv7)
+    exit 1
+    ;;
+  fedora-24:x86-64)
+    BASE="fedora:24"
+    ;;
+  ubuntu-14.04:armv7)
+    exit 1
+    ;;
+  ubuntu-14.04:x86-64)
+    BASE="ubuntu:14.04"
+    ;;
+  ubuntu-16.04:armv7)
+    BASE="armhf/ubuntu:16.04"
+    ;;
+  ubuntu-16.04:x86-64)
+    BASE="ubuntu:16.04"
+    ;;
+  *)
+    exit 1
+    ;;
+esac
+
+mkdir ${DEST}
+cp ${BASEDIR}/${PLATFORM}/* ${DEST}
+sed -e "s#@@BASE@@#${BASE}#;" < ${DEST}/Dockerfile.template > ${DEST}/Dockerfile
+
+echo "Generated ${DEST}/Dockerfile from ${BASE}"
diff --git a/base-images/ubuntu-14.04/Dockerfile.template b/base-images/ubuntu-14.04/Dockerfile.template
new file mode 100644
index 0000000..94b1806
--- /dev/null
+++ b/base-images/ubuntu-14.04/Dockerfile.template
@@ -0,0 +1,31 @@
+FROM @@BASE@@
+MAINTAINER Daniel Stone <daniels@collabora.com>
+
+# Don't bother us
+ENV DEBIAN_FRONTEND noninteractive
+COPY apt-dpkg-config-updates.conf /etc/apt/apt.conf.d/90-dpkg-config-updates.conf
+COPY apt-force-yes.conf /etc/apt/apt.conf.d/90-force-yes.conf
+COPY apt-no-recommends.conf /etc/apt/apt.conf.d/90-no-recommends.conf
+COPY apt-retry-lots.conf /etc/apt/apt.conf.d/90-retry-lots.conf
+
+# Pull the latest updates, if any
+COPY apt-sources.list /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get dist-upgrade
+
+# Get a basic toolchain together
+RUN apt-get install build-essential autoconf automake libtool intltool git-core
+
+# Add a user for CI
+RUN adduser --disabled-password --home /scratch/jenkins --shell /bin/bash --gecos 'Jenkins build user' jenkins-build
+VOLUME "/scratch/jenkins"
+
+# git-phab
+RUN apt-get install python3 python3-appdirs python3-pip
+RUN pip3 install argcomplete
+RUN pip3 install GitPython
+RUN pip3 install phabricator
+
+WORKDIR /tmp
+RUN git clone git://anongit.freedesktop.org/git/git-phab
+RUN cp git-phab/git-phab /usr/bin/
diff --git a/base-images/ubuntu-14.04/apt-dpkg-config-updates.conf b/base-images/ubuntu-14.04/apt-dpkg-config-updates.conf
new file mode 100644
index 0000000..a8a34ba
--- /dev/null
+++ b/base-images/ubuntu-14.04/apt-dpkg-config-updates.conf
@@ -0,0 +1,2 @@
+Dpkg::Options::="--force-confdef";
+Dpkg::Options::="--force-confnew";
diff --git a/base-images/ubuntu-14.04/apt-force-yes.conf b/base-images/ubuntu-14.04/apt-force-yes.conf
new file mode 100644
index 0000000..1ab128d
--- /dev/null
+++ b/base-images/ubuntu-14.04/apt-force-yes.conf
@@ -0,0 +1,2 @@
+APT::Get::Assume-Yes "true";
+APT::Get::force-yes "true";
diff --git a/base-images/ubuntu-14.04/apt-no-recommends.conf b/base-images/ubuntu-14.04/apt-no-recommends.conf
new file mode 100644
index 0000000..d12154b
--- /dev/null
+++ b/base-images/ubuntu-14.04/apt-no-recommends.conf
@@ -0,0 +1,2 @@
+APT::Get::Install-Recommends "false";
+APT::Get::Install-Suggests "false";
diff --git a/base-images/ubuntu-14.04/apt-retry-lots.conf b/base-images/ubuntu-14.04/apt-retry-lots.conf
new file mode 100644
index 0000000..86f8298
--- /dev/null
+++ b/base-images/ubuntu-14.04/apt-retry-lots.conf
@@ -0,0 +1 @@
+Acquire::Retries "20";
diff --git a/base-images/ubuntu-14.04/apt-sources.list b/base-images/ubuntu-14.04/apt-sources.list
new file mode 100644
index 0000000..77caae4
--- /dev/null
+++ b/base-images/ubuntu-14.04/apt-sources.list
@@ -0,0 +1,8 @@
+deb http://archive.ubuntu.com/ubuntu trusty main universe
+deb-src http://archive.ubuntu.com/ubuntu trusty main universe
+
+deb http://archive.ubuntu.com/ubuntu trusty-updates main universe
+deb-src http://archive.ubuntu.com/ubuntu trusty-updates main universe
+
+deb http://archive.ubuntu.com/ubuntu trusty-security main universe
+deb-src http://archive.ubuntu.com/ubuntu trusty-security main universe
diff --git a/base-images/ubuntu-16.04/Dockerfile.template b/base-images/ubuntu-16.04/Dockerfile.template
new file mode 100644
index 0000000..de686b3
--- /dev/null
+++ b/base-images/ubuntu-16.04/Dockerfile.template
@@ -0,0 +1,30 @@
+FROM @@BASE@@
+MAINTAINER Daniel Stone <daniels@collabora.com>
+
+# Don't bother us
+ENV DEBIAN_FRONTEND noninteractive
+COPY apt-dpkg-config-updates.conf /etc/apt/apt.conf.d/90-dpkg-config-updates.conf
+COPY apt-force-yes.conf /etc/apt/apt.conf.d/90-force-yes.conf
+COPY apt-no-recommends.conf /etc/apt/apt.conf.d/90-no-recommends.conf
+COPY apt-retry-lots.conf /etc/apt/apt.conf.d/90-retry-lots.conf
+
+# Pull the latest updates, if any
+COPY apt-sources.list /etc/apt/sources.list
+RUN apt-get update
+RUN apt-get dist-upgrade
+
+# Get a basic toolchain together
+RUN apt-get install build-essential autoconf automake libtool intltool git-core
+
+# Add a user for CI
+RUN adduser --disabled-password --home /scratch/jenkins --shell /bin/bash --gecos 'Jenkins build user' jenkins-build
+VOLUME "/scratch/jenkins"
+
+# git-phab
+RUN apt-get install python3 python3-argcomplete python3-appdirs python3-pip
+RUN pip3 install GitPython # we need a newer version than Xenial provides
+RUN pip3 install phabricator
+
+WORKDIR /tmp
+RUN git clone git://anongit.freedesktop.org/git/git-phab
+RUN cp git-phab/git-phab /usr/bin/
diff --git a/base-images/ubuntu-16.04/apt-dpkg-config-updates.conf b/base-images/ubuntu-16.04/apt-dpkg-config-updates.conf
new file mode 100644
index 0000000..a8a34ba
--- /dev/null
+++ b/base-images/ubuntu-16.04/apt-dpkg-config-updates.conf
@@ -0,0 +1,2 @@
+Dpkg::Options::="--force-confdef";
+Dpkg::Options::="--force-confnew";
diff --git a/base-images/ubuntu-16.04/apt-force-yes.conf b/base-images/ubuntu-16.04/apt-force-yes.conf
new file mode 100644
index 0000000..1ab128d
--- /dev/null
+++ b/base-images/ubuntu-16.04/apt-force-yes.conf
@@ -0,0 +1,2 @@
+APT::Get::Assume-Yes "true";
+APT::Get::force-yes "true";
diff --git a/base-images/ubuntu-16.04/apt-no-recommends.conf b/base-images/ubuntu-16.04/apt-no-recommends.conf
new file mode 100644
index 0000000..d12154b
--- /dev/null
+++ b/base-images/ubuntu-16.04/apt-no-recommends.conf
@@ -0,0 +1,2 @@
+APT::Get::Install-Recommends "false";
+APT::Get::Install-Suggests "false";
diff --git a/base-images/ubuntu-16.04/apt-retry-lots.conf b/base-images/ubuntu-16.04/apt-retry-lots.conf
new file mode 100644
index 0000000..86f8298
--- /dev/null
+++ b/base-images/ubuntu-16.04/apt-retry-lots.conf
@@ -0,0 +1 @@
+Acquire::Retries "20";
diff --git a/base-images/ubuntu-16.04/apt-sources.list b/base-images/ubuntu-16.04/apt-sources.list
new file mode 100644
index 0000000..d6f76e0
--- /dev/null
+++ b/base-images/ubuntu-16.04/apt-sources.list
@@ -0,0 +1,8 @@
+deb http://archive.ubuntu.com/ubuntu xenial main universe
+deb-src http://archive.ubuntu.com/ubuntu xenial main universe
+
+deb http://archive.ubuntu.com/ubuntu xenial-updates main universe
+deb-src http://archive.ubuntu.com/ubuntu xenial-updates main universe
+
+deb http://archive.ubuntu.com/ubuntu xenial-security main universe
+deb-src http://archive.ubuntu.com/ubuntu xenial-security main universe