ci: Move needed scripts from gst-ci to cerbero

Part-of: <https://gitlab.freedesktop.org/gstreamer/cerbero/-/merge_requests/740>

3 files changed, 237 insertions, 0 deletions

diff --git a/ci/cerbero_setup.sh b/ci/cerbero_setup.sh
new file mode 100755
index 00000000..8f855376
--- /dev/null
+++ b/ci/cerbero_setup.sh
@@ -0,0 +1,164 @@
+#!/bin/bash
+
+set -ex
+
+show_ccache_sum() {
+    if [[ -n ${HAVE_CCACHE} ]]; then
+        ccache -s
+    fi
+}
+
+# XXX: This is copied and modified from the cerbero-uninstalled script
+# Use `mount` to get a list of MSYS mount points that the MSYS shell uses.
+# That's our reference point for translating from MSYS paths to Win32 paths.
+# We assume that the MSYS mount point directories are only in the filesystem
+# root. This will break if people add their own custom mount points beyond what
+# MSYS automatically creates, which is highly unlikely.
+#
+# /d -> d:/
+# /c -> c:/
+# /d/projects/cerbero -> d:/projects/cerbero/
+# /home/USERNAME/cerbero -> C:\\MinGW\\msys\\1.0/home/USERNAME/
+# /mingw -> C:\\MinGW/
+# /mingw/bin/foobar -> C:\\MinGW\\bin/foobar/
+# /tmp/baz -> C:\\Users\\USERNAME\\AppData\\Local\\Temp/baz/
+msys_dir_to_win32() {
+    set -e
+    local msys_path stripped_path mount_point path mounted_path
+    # If the path is already a native path, just return that
+    if [[ $1 == ?:/* ]] || [[ $1 == ?:\\* ]]; then
+      echo $1
+      return
+    fi
+    # Convert /c or /mingw etc to /c/ or /mingw/ etc; gives us a necessary
+    # anchor to split the path into components
+    msys_path="$1/"
+    # Strip leading slash
+    stripped_path="${msys_path#/}"
+    # Get the first path component, which may be a mount point
+    mount_point="/${stripped_path%%/*}"
+    # Get the path inside the mountp oint
+    path="/${stripped_path#*/}"
+    mounted_path="$(mount | sed -n "s|\(.*\) on $mount_point type.*|\1|p")"
+    # If it's not a mounted path (like /c or /tmp or /mingw), then it's in the
+    # general MSYS root mount
+    if [[ -z $mounted_path ]]; then
+        mounted_path="$(mount | sed -n "s|\(.*\) on / type.*|\1|p")"
+        path="$1"
+    fi
+    echo ${mounted_path}${path%/}
+}
+
+# Print the working directory in the native OS path format, but with forward
+# slashes
+pwd_native() {
+    if [[ -n "$MSYSTEM" ]]; then
+        msys_dir_to_win32 "$(pwd)"
+    else
+        pwd
+    fi
+}
+
+fix_build_tools() {
+    if [[ $(uname) == Darwin ]]; then
+        # Bison needs these env vars set for the build-tools prefix to be
+        # relocatable, and we only build it on macOS. On Linux we install it
+        # using the package manager, and on Windows we use the MSYS Bison.
+        export M4="$(pwd)/${CERBERO_HOME}/build-tools/bin/m4"
+        export BISON_PKGDATADIR="$(pwd)/${CERBERO_HOME}/build-tools/share/bison"
+    fi
+}
+
+# Produces runtime and devel tarball packages for linux/android or .pkg for macos
+cerbero_package_and_check() {
+    $CERBERO $CERBERO_ARGS package --offline ${CERBERO_PACKAGE_ARGS} -o "$(pwd_native)" gstreamer-1.0
+
+    # Run gst-inspect-1.0 for some basic checks. Can't do this for cross-(android|ios)-universal, of course.
+    if [[ $CONFIG != *universal* ]]; then
+        $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX --version
+        $CERBERO $CERBERO_ARGS run $CERBERO_RUN_WRAPPER gst-inspect-1.0$CERBERO_RUN_SUFFIX
+    fi
+
+    show_ccache_sum
+}
+
+cerbero_before_script() {
+    pwd
+    ls -lha
+
+    # If there's no cerbero-sources directory in the runner cache, copy it from
+    # the image cache
+    if ! [[ -d ${CERBERO_SOURCES} ]]; then
+        time cp -a "${CERBERO_HOST_DIR}/${CERBERO_SOURCES}" .
+    fi
+    du -sch "${CERBERO_SOURCES}" || true
+
+    echo "home_dir = \"$(pwd_native)/${CERBERO_HOME}\"" > localconf.cbc
+    echo "local_sources = \"$(pwd_native)/${CERBERO_SOURCES}\"" >> localconf.cbc
+    if [[ $CONFIG == win??.cbc ]]; then
+        # Visual Studio 2017 build tools install path
+        echo 'vs_install_path = "C:/BuildTools"' >> localconf.cbc
+        echo 'vs_install_version = "vs15"' >> localconf.cbc
+    fi
+    echo "recipes_commits = {'gstreamer-1.0': 'ci/${CI_GSTREAMER_REF_NAME}'}" >> localconf.cbc
+    echo "recipes_remotes = {'gstreamer-1.0': {'ci': '${CI_GSTREAMER_URL}'}}" >> localconf.cbc
+    cat localconf.cbc
+
+    # GitLab runner does not always wipe the image after each job, so do that
+    # to ensure we don't have any leftover data from a previous job such as
+    # a dirty builddir, or tarballs/pkg files, leftover files from an old
+    # cerbero commit, etc. Skip the things we actually need to keep.
+    time git clean -xdf -e localconf.cbc -e "${CERBERO_SOURCES}"
+}
+
+cerbero_script() {
+    show_ccache_sum
+
+    $CERBERO $CERBERO_ARGS show-config
+    $CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4
+    $CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0
+    du -sch "${CERBERO_SOURCES}" || true
+
+    $CERBERO $CERBERO_ARGS fetch-cache --branch "${GST_UPSTREAM_BRANCH}"
+
+    if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} && -d "${CERBERO_HOME}/dist/${ARCH}" ]]; then
+        mkdir -p "${CERBERO_OVERRIDDEN_DIST_DIR}"
+        time rsync -aH "${CERBERO_HOME}/dist/${ARCH}/" "${CERBERO_OVERRIDDEN_DIST_DIR}"
+    fi
+
+    $CERBERO $CERBERO_ARGS bootstrap --offline --system=no
+    fix_build_tools
+
+    cerbero_package_and_check
+}
+
+cerbero_deps_script() {
+    show_ccache_sum
+
+    $CERBERO $CERBERO_ARGS show-config
+    $CERBERO $CERBERO_ARGS fetch-bootstrap --jobs=4
+    $CERBERO $CERBERO_ARGS fetch-package --jobs=4 --deps gstreamer-1.0
+    $CERBERO $CERBERO_ARGS bootstrap --offline --system=no
+    $CERBERO $CERBERO_ARGS build-deps --offline \
+        gstreamer-1.0 gst-plugins-base-1.0 gst-plugins-good-1.0 \
+        gst-plugins-bad-1.0 gst-plugins-ugly-1.0 gst-rtsp-server-1.0 \
+        gst-libav-1.0 gst-devtools-1.0 gst-editing-services-1.0 libnice
+
+    if [[ -n ${CERBERO_OVERRIDDEN_DIST_DIR} ]]; then
+        mkdir -p "${CERBERO_HOME}/dist/${ARCH}"
+        time rsync -aH "${CERBERO_OVERRIDDEN_DIST_DIR}/" "${CERBERO_HOME}/dist/${ARCH}"
+    fi
+
+    # Check that the env var is set. Don't expand this protected variable by
+    # doing something silly like [[ -n ${CERBERO_...} ]] because it will get
+    # printed in the CI logs due to set -x
+    if env | grep -q -e CERBERO_PRIVATE_SSH_KEY; then
+        time $CERBERO $CERBERO_ARGS gen-cache --branch "${GST_UPSTREAM_BRANCH}"
+        time $CERBERO $CERBERO_ARGS upload-cache --branch "${GST_UPSTREAM_BRANCH}"
+    fi
+
+    cerbero_package_and_check
+}
+
+# Run whichever function is asked of us
+eval "$1"
diff --git a/ci/docker_android_setup.sh b/ci/docker_android_setup.sh
new file mode 100755
index 00000000..dba648a7
--- /dev/null
+++ b/ci/docker_android_setup.sh
@@ -0,0 +1,50 @@
+set -eux
+
+export ANDROID_HOME=$1
+export ANDROID_NDK_HOME=$2
+DEFAULT_BRANCH=$3
+
+mkdir -p /android/sources
+
+curl -o /android/sources/android-ndk.zip https://dl.google.com/android/repository/android-ndk-r21-linux-x86_64.zip
+unzip /android/sources/android-ndk.zip -d ${ANDROID_NDK_HOME}/
+# remove the intermediate versioned directory
+mv ${ANDROID_NDK_HOME}/*/* ${ANDROID_NDK_HOME}/
+
+curl -o /android/sources/android-sdk-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-6200805_latest.zip
+unzip /android/sources/android-sdk-tools.zip -d ${ANDROID_HOME}/
+mkdir -p ${ANDROID_HOME}/licenses
+
+# Accept licenses. Values taken from:
+# $ANDROID_HOME/tools/bin/sdkmanager --sdk_root=$ANDROID_HOME --licenses
+# cd $ANDROID_HOME
+# for f in licenses/*; do echo "echo \"$(cat $f | tr -d '\n')\" > \${ANDROID_HOME}/$f"; done
+echo "601085b94cd77f0b54ff86406957099ebe79c4d6" > ${ANDROID_HOME}/licenses/android-googletv-license
+echo "859f317696f67ef3d7f30a50a5560e7834b43903" > ${ANDROID_HOME}/licenses/android-sdk-arm-dbt-license
+echo "24333f8a63b6825ea9c5514f83c2829b004d1fee" > ${ANDROID_HOME}/licenses/android-sdk-license
+echo "84831b9409646a918e30573bab4c9c91346d8abd" > ${ANDROID_HOME}/licenses/android-sdk-preview-license
+echo "33b6a2b64607f11b759f320ef9dff4ae5c47d97a" > ${ANDROID_HOME}/licenses/google-gdk-license
+echo "e9acab5b5fbb560a72cfaecce8946896ff6aab9d" > ${ANDROID_HOME}/licenses/mips-android-sysimage-license
+
+# pre-cache deps
+export GSTREAMER_ROOT_ANDROID=/android/sources/gstreamer-android
+curl -o /android/sources/gstreamer-android.tar.xz https://gstreamer.freedesktop.org/data/pkg/android/1.16.2/gstreamer-1.0-android-universal-1.16.2.tar.xz
+mkdir $GSTREAMER_ROOT_ANDROID
+tar -xvf /android/sources/gstreamer-android.tar.xz -C $GSTREAMER_ROOT_ANDROID
+ls $GSTREAMER_ROOT_ANDROID
+
+git clone -b ${DEFAULT_BRANCH} https://gitlab.freedesktop.org/gstreamer/gstreamer.git /android/sources/gstreamer
+export GSTREAMER_ROOT=/android/sources/gstreamer/subprojects
+
+chmod +x $GSTREAMER_ROOT/gst-examples/playback/player/android/gradlew
+$GSTREAMER_ROOT/gst-examples/playback/player/android/gradlew --no-daemon --project-dir $GSTREAMER_ROOT/gst-examples/playback/player/android dependencies --refresh-dependencies
+
+chmod +x $GSTREAMER_ROOT/gst-examples/vulkan/android/gradlew
+$GSTREAMER_ROOT/gst-examples/vulkan/android/gradlew --no-daemon --project-dir $GSTREAMER_ROOT/gst-examples/vulkan/android dependencies --refresh-dependencies
+
+chmod +x $GSTREAMER_ROOT/gst-docs/examples/tutorials/android/gradlew
+$GSTREAMER_ROOT/gst-docs/examples/tutorials/android/gradlew --no-daemon --project-dir $GSTREAMER_ROOT/gst-docs/examples/tutorials/android dependencies --refresh-dependencies
+
+unset GSTREAMER_ROOT_ANDROID
+
+rm -rf /android/sources
diff --git a/ci/docker_setup.sh b/ci/docker_setup.sh
new file mode 100755
index 00000000..fae3329c
--- /dev/null
+++ b/ci/docker_setup.sh
@@ -0,0 +1,23 @@
+#! /bin/bash
+
+set -e
+
+git config --global user.email "cerbero@gstreamer.freedesktop.org"
+git config --global user.name  "Cerbero Build System"
+
+mkdir $HOME/.cerbero
+echo "allow_parallel_build=True" > $HOME/.cerbero/cerbero.cbc
+echo "use_ccache=True" >> $HOME/.cerbero/cerbero.cbc
+echo "local_sources=\"/cerbero-sources\"" >> localconf.cbc
+echo "home_dir=\"/cerbero-build\"" >> localconf.cbc
+./cerbero-uninstalled -t -c localconf.cbc fetch-bootstrap --jobs=4
+./cerbero-uninstalled -t -c localconf.cbc fetch-package --jobs=4 gstreamer-1.0
+./cerbero-uninstalled -t -c localconf.cbc bootstrap -y --build-tools=no --toolchains=no
+./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-bootstrap --jobs=4
+./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win32.cbc fetch-package --jobs=4 gstreamer-1.0
+./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-bootstrap --jobs=4
+./cerbero-uninstalled -t -c localconf.cbc -c config/cross-win64.cbc fetch-package --jobs=4 gstreamer-1.0
+./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-bootstrap --jobs=4
+./cerbero-uninstalled -t -c localconf.cbc -c config/cross-android-universal.cbc fetch-package --jobs=4 gstreamer-1.0
+rm -rf /cerbero-build/{dist,logs,sources}
+rm -f /cerbero-build/{linux,windows,android}*.cache