runner.sh: introduce a new leaner and more modular runner

v2 (reported by Olivier Berthier):
 - make sure to write the deployed version to the journal when we start
   testing. This will prevent the DEPLOYMENT_ERROR from being reported.
 - write the compilation error code to the compilation log
 - update the readme to mention runner.sh and deprecate core.sh

v3:
 - only set the timings on a full run, not on resumes
 - runner.py: make the command return a RunnerCmdResult
 - document functions

1 files changed, 794 insertions, 0 deletions

diff --git a/runner.sh b/runner.sh
new file mode 100755
index 0000000..65872b8
--- /dev/null
+++ b/runner.sh
@@ -0,0 +1,794 @@
+#!/bin/bash
+
+# Copyright (c) 2015, Intel Corporation
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#     * Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#     * Neither the name of Intel Corporation nor the names of its contributors
+#       may be used to endorse or promote products derived from this software
+#       without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# The script is known to work with recent versions of bash.
+# Authors:
+# - Martin Peres <martin.peres@intel.com>
+# - Chris Wilson <chris@chris-wilson.co.uk>
+
+# Uncomment the following to track all the executed commands
+#set -o xtrace
+
+# Start by defining all the errors
+typeset -A errors
+errors[0]="No Error"
+errors[1]="Unknown error"
+errors[5]="An instance of runner.sh is already running"
+errors[6]="The report is already locked"
+
+# Commands parsing
+errors[10]="Invalid command"
+errors[11]="Parameter already set"
+errors[12]="The profile does not exist"
+errors[13]="The profile is not specified"
+errors[14]="The report cannot be opened/created"
+errors[15]="The report is missing"
+errors[16]="Testing has not been started"
+errors[17]="Test execution type unsupported"
+errors[18]="Test execution type requires a valid result file"
+errors[19]="Result already complete"
+
+# OS
+errors[30]="The shell does not support globstat"
+errors[31]="Cannot create the log folder"
+errors[32]="Cannot move to the repo directory"
+
+# SCM
+errors[50]="Invalid version ID"
+
+# Environment
+errors[60]="Failed to deploy the environment"
+
+# Compilation and deployment
+errors[70]="Compilation or deployment failed"
+errors[71]="Compilation failed"
+errors[72]="Deployment failed"
+errors[73]="The deployed version does not match the wanted version"
+errors[74]="A reboot is necessary"
+
+# Running
+errors[80]="Early exit requested"
+
+# Test
+errors[100]="The test does not exist"
+
+function lock {
+    # Inspired from http://www.kfirlavi.com/blog/2012/11/06/elegant-locking-of-bash-program/
+    local lock_file="$1"
+    local fd="$2"
+
+    eval "exec $fd>$lock_file"
+
+    flock -w 0.1 -x $fd
+    return $?
+}
+
+function unlock {
+    local fd="$1"
+    flock -u $fd
+    return $?
+}
+
+function use_report {
+    local name=$1
+
+    [ -n "$reportName" ] && return 11
+
+    # Keep the stupid name for the rapl_check profile
+    local folder="$ezBenchDir/logs/${name:-$(date +"%Y-%m-%d-%T")}"
+
+    # Create the folder
+    [ -d $folder ] || mkdir -p $folder || return 30
+    echo "report,$name" >> $folder/runner.log
+    exec > >(tee -a $folder/runner.log)
+    exec 2>&1
+
+    reportName=$name
+    logsFolder=$folder
+
+    # generate the early-exit filename
+    abortFile="$logsFolder/requestExit"
+
+    return 0
+}
+
+function use_profile {
+    local prof=$1
+
+    [ -n "$profile" ] && return 11
+
+    # Check if the profile exists
+    profileDir="$ezBenchDir/profiles.d/$prof"
+    if [ ! -d "$profileDir" ]; then
+        return 12
+    fi
+
+    # The profile name is valid, set some environment variables
+    PROFILE_TMP_BUILD_DIR="$DEPLOY_BASE_DIR/$prof/tmp"
+    PROFILE_DEPLOY_BASE_DIR="$DEPLOY_BASE_DIR/$prof/builds"
+    PROFILE_DEPLOY_DIR="$DEPLOY_BASE_DIR/$prof/cur"
+
+    # Default user options
+    for conf in $profileDir/conf.d/**/*.conf; do
+        [ "$conf" = "$profileDir/conf.d/**/*.conf" ] && continue
+        source "$conf"
+    done
+    source "$profileDir/profile"
+
+    profile=$prof
+
+    return 0
+}
+
+function parse_tests_profiles  {
+    [ $available_tests_parsed -eq 1 ] && return
+
+    # Parse the list of tests
+    for test_dir in ${testsDir:-$ezBenchDir/tests.d}; do
+        for test_file in $test_dir/**/*.test; do
+            unset test_name
+            unset test_unit
+            unset test_type
+            unset test_invert
+            unset test_exec_time
+
+            source "$test_file" || continue
+
+            # Sanity checks on the file
+            [ -z "$test_name" ] && continue
+            [ -z "$test_exec_time" ] && continue
+
+            # Set the default unit to FPS
+            [ -z "$test_unit" ] && test_unit="FPS"
+
+            # Set the default type to bench
+            [ -z "$test_type" ] && test_type="bench"
+
+            # Set the default type to bench
+            [ -z "$test_invert" ] && test_invert="0"
+
+            # Set the default error handling capabiliy
+            [ -z "$test_has_exit_code" ] && test_has_exit_code="0"
+
+            for test in $test_name; do
+                # TODO: Check that the run function exists
+
+                idx=${#availTestNames[@]}
+                availTestNames[$idx]=$test
+                availTestUnits[$idx]=$test_unit
+                availTestTypes[$idx]=$test_type
+                availTestIsInvert[$idx]=$test_invert
+                availTestExecTime[$idx]=$test_exec_time
+                availTestHasExitCode[$idx]=$test_has_exit_code
+            done
+        done
+    done
+    unset test_name
+    unset test_unit
+    unset test_type
+    unset test_invert
+    unset test_exec_time
+
+    available_tests_parsed=1
+}
+
+function done_testing {
+    [ "$testing_ready" -eq 1 ] || return
+
+    # Execute the user-defined post hook
+    callIfDefined ezbench_post_hook
+
+    # Delete the abort file
+    if [ -n "$abortFile" ]; then
+        rm $abortFile 2> /dev/null
+    fi
+
+    unlock 201
+    unlock 200
+
+    # Clear the traps before calling the finish hook
+    trap - EXIT
+    trap - INT # Needed for zsh
+
+    testing_ready=0
+}
+
+function start_testing {
+    [ "$testing_ready" -eq 0 ] || return
+    [ -n "$profile" ] || return 13
+    [ -n "$reportName" ] || return 15
+
+    # Lock against concurent testing with the runner
+    lock "$ezBenchDir/lock" 200 || return 5
+
+    # Lock the report, as it allows knowing which report is being worked on
+    if ! lock "$logsFolder/lock" 201; then
+        unlock 200
+        return 6
+    fi
+
+    # Execute the user-defined environment deployment hook
+    callIfDefined ezbench_env_deploy_hook
+
+    trap done_testing EXIT
+    trap done_testing INT # Needed for zsh
+
+    testing_ready=1
+
+    # Write in the journal what current version is deployed
+    write_to_journal deployed $(profile_repo_deployed_version)
+
+    return 0
+}
+
+function compile_and_deploy {
+    # Accessible variables
+    # $version      [RO]: SHA1 id of the current version
+    # $versionName  [RO]: Name of the version
+
+    profile_repo_get_patch $version > "$logsFolder/$1.patch"
+
+    # Select the version of interest
+    local versionListLog="$logsFolder/commit_list"
+    if [ -z "$(grep ^"$version" "$versionListLog" 2> /dev/null)" ]
+    then
+        local title=$(profile_repo_version_title "$version")
+        echo "$version $title" >> "$versionListLog"
+    fi
+
+    # early exit if the deployed version is the wanted version
+    deployed_version=$(profile_repo_deployed_version)
+    are_same_versions "$version" "$deployed_version" && return 0
+
+    local compile_logs=$logsFolder/${version}_compile_log
+
+    # Compile the version and check for failure. If it failed, go to the next version.
+    export REPO_COMPILE_AND_DEPLOY_VERSION=$version
+    eval "$makeAndDeployCmd" >> "$compile_logs" 2>&1
+    local exit_code=$?
+    unset REPO_COMPILE_AND_DEPLOY_VERSION
+
+    # Write the compilation status to the compile log.
+    # The exit code 74 actually means everything is fine but we need to reboot
+    if [ $exit_code -eq 74 ]
+    then
+        printf "Exiting with error code 0\n" >> "$compile_logs"
+    else
+        printf "Exiting with error code $exit_code\n" >> "$compile_logs"
+    fi
+
+    # Add to the log that we are now in the deployment phase
+    write_to_journal deploy $version
+
+    # Check for compilation errors
+    if [ "$exit_code" -ne '0' ]; then
+        if [[ $exit_code -lt 71 && $exit_code -gt 74 ]]; then
+            return 70
+        else
+            return $exit_code
+        fi
+    fi
+
+    # Check that the deployed image is the right one
+    are_same_versions "$version" "$(profile_repo_deployed_version)" || return 73
+
+    # Write in the journal that the version is deployed
+    write_to_journal deployed $version
+
+    return 0
+}
+
+function find_test {
+    # Accessible variables
+    # $availTestNames    [RO]: Array containing the names of available tests
+    # $availTestUnits    [RO]: Array containing the unit of available tests
+    # $availTestTypes    [RO]: Array containing the result type of available tests
+    # $availTestIsInvert [RO]: Array containing the execution time of available tests
+    # $availTestExecTime [RO]: Array containing the execution time of available tests
+
+    local test=$1
+    local basetest=$(echo "$test" | cut -d [ -f 1)
+    local subtests=$(echo $test | cut -s -d '[' -f 2- | rev | cut -d ']' -f 2- | rev)
+
+    # Try to find the test in the list of available tests
+    for (( a=0; a<${#availTestNames[@]}; a++ )); do
+        if [[ ${availTestNames[$a]} == $basetest ]]; then
+            # We do not accept running subtests on non-complete matches of names
+            [[ $basetest != ${availTestNames[$a]} && -n $subtests ]] && continue
+
+            testName="${availTestNames[$a]}"
+            testSubTests="$subtests"
+            testUnit="${availTestUnits[$a]}"
+            testType="${availTestTypes[$a]}"
+            testInvert="${availTestIsInvert[$a]}"
+            testExecTime="${availTestExecTime[$a]}"
+            testHasExitCode="${availTestHasExitCode[$a]}"
+
+            return 0
+        fi
+    done
+
+    return 1
+}
+
+function execute_test {
+    local testExecutionType=$1
+    local versionName=$2
+    local testName=$3
+    local verboseOutput=$4
+    local run_log_file_name=$5
+
+    [ -n "$versionName" ] || return 10
+    [ -n "$testName" ] || return 10
+    [ -n "$verboseOutput" ] || return 10
+    [ -n "$profile" ] || return 13
+    [ -n "$reportName" ] || return 15
+    [ "$testing_ready" -eq 1 ] || return 16
+
+    # Verify that, if specified, the run_log_file_name exits
+    [[ -n "$run_log_file_name" && ! -f "$logsFolder/$run_log_file_name" ]] && return 18
+
+    # Find the test
+    parse_tests_profiles
+    find_test "$testName" || return 100
+
+    # Get the full version name
+    local version=$(profile_repo_version_from_human $versionName) || return 50
+
+    # verify that the type of execution is available
+    local execFuncName=${testName}_${testExecutionType}
+    if ! function_available "$execFuncName"; then
+        # If the type was resume, mark the test as tested because there is
+        # nothing else we can do for this run
+        if [ "$testExecutionType" == "resume" ]; then
+            write_to_journal tested "$version" "${testName}" "$run_log_file_name"
+        fi
+        return 17
+    fi
+
+    # compile and deploy the version
+    compile_and_deploy $version || return $?
+
+    # Exit if asked to
+    [ -e "$abortFile" ] && return 80
+
+    # Generate the logs file names
+    local reportFileName="${version}_${testType}_${testName}"
+    local reportFile="$logsFolder/$reportFileName"
+
+    # Only generate the run_log_file_name if it is unspecified
+    if [ -z "$run_log_file_name" ]; then
+        # Find the first run id available
+        if [ -f "$reportFile" ] || [ ${testType} == "unified" ]; then
+            # The logs file exist, look for the number of runs
+            local run=0
+            while [ -f "${reportFile}#${run}" ]
+            do
+                local run=$((run+1))
+            done
+        else
+            if [ -z "${testInvert}" ]; then
+                direction="more is better"
+            else
+                direction="less is better"
+            fi
+            echo "# ${testUnit} ($direction) of '${testName}' using version ${version}" > "$reportFile"
+            local run=0
+        fi
+        local run_log_file_name="${reportFileName}#$run"
+    fi
+
+    # compute the different hook names
+    local preHookFuncName=${testName}_run_pre_hook
+    local postHookFuncName=${testName}_run_post_hook
+
+    local run_log_file="$logsFolder/$run_log_file_name"
+    IFS='|' read -a run_sub_tests <<< "$testSubTests"
+
+    echo "$run_log_file_name"
+
+    callIfDefined "$preHookFuncName"
+    callIfDefined benchmark_run_pre_hook
+
+    # This function will return multiple fps readings
+    write_to_journal test "$version" "${testName}" "$run_log_file_name"
+    if [ "$verboseOutput" -eq 1 ]; then
+        "$execFuncName" > >(tee "$run_log_file")
+    else
+        "$execFuncName" > "$run_log_file" 2> /dev/null
+    fi
+    local exit_code=$?
+    write_to_journal tested "$version" "${testName}" "$run_log_file_name"
+
+    callIfDefined benchmark_run_post_hook
+    callIfDefined "$postHookFuncName"
+
+    if [ ${testType} != "unified" ]; then
+        if [ -s "$run_log_file" ]; then
+            if [ ${testType} == "bench" ]; then
+                # Add the reported values before adding the result to the average values for
+                # the run.
+                run_avg=$(awk '{sum=sum+$1} END {print sum/NR}' $run_log_file)
+            elif [ ${testType} == "unit" ]; then
+                run_avg=$(head -n 1 $run_log_file)
+            elif [ ${testType} == "imgval" ]; then
+                run_avg=0.0
+            fi
+            echo "$run_avg" >> "$reportFile"
+
+        else
+            echo "0" >> "$run_log_file"
+            echo "0" >> "$reportFile"
+        fi
+    fi
+
+    # If the test returns exit codes, then use it
+    if [ "$testHasExitCode" -eq 1 ]; then
+        return $exit_code
+    else
+        return 0
+    fi
+}
+
+# set the default run_bench function which can be overriden by the profiles:
+# Bash variables: $run_log_file : filename of the log report of the current run
+#                 $testName : Name of the test
+#                 $testSubTests : List of subtests
+# Arguments: $1 : timeout (set to 0 for infinite wait)
+#            $2+: command line executing the test AND NOTHING ELSE!
+function run_bench {
+    timeout=$1
+    shift
+    cmd="LIBGL_DEBUG=verbose vblank_mode=0 stdbuf -oL timeout $timeout"
+    bench_binary=$(echo "$1" | rev | cut -d '/' -f 1 | rev)
+
+    env_dump_path="$ezBenchDir/utils/env_dump"
+    env_dump_lib="$env_dump_path/env_dump.so"
+    env_dump_launch="$env_dump_path/env_dump.sh"
+    env_dump_extend="$env_dump_path/env_dump_extend.sh"
+
+    if [ -f "$env_dump_lib" ]; then
+        run_log_file_env_dump="$run_log_file"
+        cmd="$cmd $env_dump_launch "$run_log_file" $@"
+    else
+        cmd="$cmd $@"
+    fi
+
+    run_log_file_stdout="$run_log_file.stdout"
+    run_log_file_stderr="$run_log_file.stderr"
+
+    callIfDefined run_bench_pre_hook
+    local time_before=$(date +%s.%N)
+
+    eval $cmd > "$run_log_file_stdout" 2> "$run_log_file_stderr"
+    local exit_code=$?
+
+    if [ -f "$env_dump_lib" ]; then
+        $env_dump_extend "$SHA1_DB" "$run_log_file.env_dump"
+    fi
+
+    local time_after=$(date +%s.%N)
+    test_exec_time=$(echo "$time_after - $time_before" | bc -l)
+    callIfDefined run_bench_post_hook
+
+    # If the test does not have subtests, then store the execution time
+    if [ -z "$testSubTests" && "$testExecutionType" != "resume" ]; then
+        "$ezBenchDir/timing_DB/timing.py" -n test -k "$testName" -a $test_exec_time
+    fi
+
+    # delete the log files if they are empty
+    if [ ! -s "$run_log_file_stdout" ] ; then
+        rm "$run_log_file_stdout"
+    else
+        cat "$run_log_file_stdout"
+    fi
+    if [ ! -s "$run_log_file_stderr" ] ; then
+        rm "$run_log_file_stderr"
+    else
+        cat "$run_log_file_stderr" >&2
+    fi
+
+    return $exit_code
+}
+
+function function_available() {
+    if [ "$(type -t "$1")" == 'function' ]; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function callIfDefined() {
+    if [ "$(type -t "$1")" == 'function' ]; then
+        local funcName=$1
+        shift
+        $funcName "$@"
+        return 0
+    else
+        return 1
+    fi
+}
+
+function write_to_journal {
+    local journal="$logsFolder/journal"
+    local operation=$1
+    local key=$2
+    shift 2
+
+    [ -d "$logsFolder" ] || return
+
+    local time=$(date +"%s.%6N")
+
+    # if the operation is "deployed", check how long it took to deploy
+    if [[ "$operation" == "deployed" ]]; then
+        local deploy_time=$(tail -n 1 "$logsFolder/journal" 2> /dev/null | awk -F ',' "{    \
+            if (\$2 == \"deploy\" && \$3 == \"$key\") {                          \
+                print $time - \$1                                              \
+            }                                                                  \
+        }")
+
+        if [ -n "$deploy_time" ]; then
+            "$ezBenchDir/timing_DB/timing.py" -n deploy -k "$profile" -a "$deploy_time"
+        fi
+    fi
+
+    echo -n "$time,$operation,$key" >> "$journal" 2> /dev/null
+
+    while [ "${1+defined}" ]; do
+        echo -n ",$1" >> "$journal" 2> /dev/null
+        shift
+    done
+    echo "" >> "$journal" 2> /dev/null
+
+    sync
+}
+
+function show_help {
+    echo "This tool is meant to be run by machines, but if you really want to deal with this"
+        echo "manually, here is the list of commands that can be used:"
+        echo ""
+        echo "  - side-effect-free commands:"
+        echo "    - conf_script:"
+        echo "          return the list of configuration scripts set"
+        echo ""
+        echo "    - echo:"
+        echo "          does nothing, useful as a fence mechanism"
+        echo ""
+        echo "    - help:"
+        echo "          displays this help message"
+        echo ""
+        echo "    - list_cached_versions:"
+        echo "          list all the compiled versions for the profile"
+        echo ""
+        echo "    - list_tests:"
+        echo "          list the set of available tests"
+        echo ""
+        echo "    - profile:"
+        echo "          return the current profile"
+        echo ""
+        echo "    - repo:"
+        echo "          return information about the profile's repository"
+        echo ""
+        echo "    - report:"
+        echo "          return the current profile"
+        echo ""
+        echo "    - version:"
+        echo "          return the implemented version of the protocol"
+        echo ""
+        echo ""
+        echo "  - state-changing commands:"
+        echo "    - conf_script,<path>:"
+        echo "          set the profile for the current session"
+        echo ""
+        echo "    - done:"
+        echo "          tear down the test environment and exit"
+        echo ""
+        echo "    - profile,<profile>:"
+        echo "          set the profile for the current session"
+        echo ""
+        echo "    - reboot:"
+        echo "          reboot the machine"
+        echo ""
+        echo "    - report,<report_name>:"
+        echo "          set the report for the current session"
+        echo ""
+        echo "    - resume,<version>,<test>,<result_file>,<verbose>:"
+        echo "          Run a test, verbose=1 for reading the output"
+        echo ""
+        echo "    - run,<version>,<test>,<verbose>:"
+        echo "          Run a test, verbose=1 for reading the output"
+        echo ""
+        echo "    - start_testing:"
+        echo "          start the test environment"
+        echo ""
+        echo "That's all, folks!"
+}
+
+shopt -s globstar || {
+    echo "ERROR: ezbench requires bash 4.0+ or zsh with globstat support."
+    exit 30
+}
+
+# Printf complains about floating point numbers having , as a delimiter otherwise
+LC_NUMERIC="C"
+
+# Get the current folder
+ezBenchDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+# initial cleanup
+mkdir "$ezBenchDir/logs" 2> /dev/null
+
+# Read the user parameters
+source "$ezBenchDir/user_parameters.sh"
+
+# Initialise the default state
+typeset -A availTestNames
+typeset -A availTestUnits
+typeset -A availTestTypes
+typeset -A availTestIsInvert
+typeset -A availTestExecTime
+typeset -A availTestHasExitCode
+typeset -A confs
+protocol_version=1
+reportName=""
+profile=""
+available_tests_parsed=0
+testing_ready=0
+dry_run=0
+
+# Now start the main loop
+while read line
+do
+    start_time=$(date +%s.%N)
+    break=0
+    error=1
+
+    echo "-->,$line"
+    cmd=$(echo $line | cut -d ',' -f 1)
+
+    if [ "$cmd" == "help" ]; then
+        show_help
+        error=0
+
+    elif [ "$cmd" == "report" ]; then
+        arg=$(echo "$line" | cut -d ',' -s -f 2)
+        if [ -n "$arg" ]; then
+            use_report "$arg"
+            error=$?
+        else
+            echo "$reportName"
+            error=0
+        fi
+
+        reportName=$arg
+
+    elif [ "$cmd" == "profile" ]; then
+        arg=$(echo "$line" | cut -d ',' -s -f 2)
+        if [ -n "$arg" ]; then
+            use_profile "$arg"
+            error=$?
+        else
+            echo "$profile"
+            error=0
+        fi
+
+    elif [ "$cmd" == "repo" ]; then
+        if [ -n "$profile" ]; then
+            echo "type=$(profile_repo_type)"
+            echo "path=$repoDir"
+            echo "head=$(profile_repo_version)"
+            echo "deployed_version=$(profile_repo_deployed_version)"
+            error=$?
+        else
+            error=13
+        fi
+
+    elif [ "$cmd" == "conf_script" ]; then
+        arg=$(echo "$line" | cut -d ',' -s -f 2)
+        if [ -n "$arg" ]; then
+            source "$arg"
+            idx=${#confs[@]}
+            confs[$idx]="$arg"
+            error=0
+        else
+            for (( c=0; c<${#confs[@]}; c++ )); do
+                echo "${confs[$c]}"
+            done
+            error=0
+        fi
+
+    elif [ "$cmd" == "list_tests" ]; then
+        parse_tests_profiles
+
+        for (( a=0; a<${#availTestNames[@]}; a++ )); do
+            echo "${availTestNames[$a]},${availTestTypes[$a]},${availTestUnits[$a]},${availTestIsInvert[$a]},${availTestExecTime[$a]}"
+        done
+        error=0
+
+    elif [ "$cmd" == "list_cached_versions" ]; then
+        if [ -n "$profile" ]; then
+            for v in $(profile_get_built_versions); do
+                echo $v
+            done
+            error=0
+        else
+            error=13
+        fi
+
+    elif [ "$cmd" == "start_testing" ]; then
+        start_testing
+        error=$?
+
+    elif [[ "$cmd" == "done" || "$cmd" == "reboot" ]]; then
+        break=1
+        error=0
+
+    elif [ "$cmd" == "resume" ]; then
+        version=$(echo "$line" | cut -d ',' -s -f 2)
+        test=$(echo "$line" | cut -d ',' -s -f 3)
+        result_file=$(echo "$line" | cut -d ',' -s -f 4)
+        verbose=$(echo "$line" | cut -d ',' -s -f 5)
+
+        execute_test resume "$version" "$test" "$verbose" "$result_file"
+        error=$?
+
+    elif [ "$cmd" == "run" ]; then
+        version=$(echo "$line" | cut -d ',' -s -f 2)
+        test=$(echo "$line" | cut -d ',' -s -f 3)
+        verbose=$(echo "$line" | cut -d ',' -s -f 4)
+
+        execute_test run "$version" "$test" "$verbose"
+        error=$?
+
+    elif [ "$cmd" == "echo" ]; then
+        error=0
+
+    elif [ "$cmd" == "version" ]; then
+        echo "$protocol_version"
+        error=0
+
+    else
+        error=10
+    fi
+
+    t=$(echo "($(date +%s.%N) - $start_time) * 1000 / 1" | bc)
+    echo "<--,$error,${errors[$error]},$t ms"
+
+    # Exit, if requested
+    [ "$break" == 1 ] && break
+done < "${1:-/dev/stdin}"
+
+# Make sure we tear-down everything we started
+done_testing
+
+# If the last command received was reboot, do it!
+if [ "$cmd" == "reboot" ]; then
+    sudo reboot
+fi
+
+exit 0