ci: Drop the git dependency in tracie

Instead of using git, use python and the Gitlab API
to fetch traces. This helps us slim down our ramdisks
in preparation for integrating trace replay on LAVA
devices.

Signed-off-by: Rohan Garg <rohan.garg@collabora.com>
Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Alexandros Frantzis <alexandros.frantzis@collabora.com>
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4000>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/4000>

8 files changed, 202 insertions, 225 deletions

diff --git a/.gitlab-ci/container/x86_build.sh b/.gitlab-ci/container/x86_build.sh
index 3ee0dad6463..641ef07f2dc 100644
--- a/.gitlab-ci/container/x86_build.sh
+++ b/.gitlab-ci/container/x86_build.sh
@@ -82,6 +82,8 @@ apt-get install -y --no-remove \
       pkg-config \
       python-mako \
       python3-mako \
+      python3-pil \
+      python3-requests \
       qemu-user \
       scons \
       x11proto-dri2-dev \
diff --git a/.gitlab-ci/container/x86_test-gl.sh b/.gitlab-ci/container/x86_test-gl.sh
index 2b94a2ef5c2..747e0b21166 100644
--- a/.gitlab-ci/container/x86_test-gl.sh
+++ b/.gitlab-ci/container/x86_test-gl.sh
@@ -34,7 +34,6 @@ apt-get install -y --no-remove \
       g++ \
       gcc \
       git \
-      git-lfs \
       libexpat1 \
       libgbm-dev \
       libgles2-mesa-dev \
@@ -64,6 +63,7 @@ apt-get install -y --no-remove \
       python3-mako \
       python3-numpy \
       python3-pil \
+      python3-requests \
       python3-six \
       python3-yaml \
       python3.7 \
diff --git a/.gitlab-ci/tracie-runner-gl.sh b/.gitlab-ci/tracie-runner-gl.sh
index 28331e83cd3..cb7113cfcbd 100755
--- a/.gitlab-ci/tracie-runner-gl.sh
+++ b/.gitlab-ci/tracie-runner-gl.sh
@@ -5,7 +5,7 @@ set -ex
 ARTIFACTS="$(pwd)/artifacts"
 
 # Set up the driver environment.
-export LD_LIBRARY_PATH="$(pwd)/install/lib/"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$(pwd)/install/lib/"
 
 # Set environment for renderdoc libraries.
 export PYTHONPATH="$PYTHONPATH:/renderdoc/build/lib"
@@ -25,10 +25,4 @@ export WAFFLE_PLATFORM=surfaceless_egl
 # Perform a self-test to ensure tracie is working properly.
 "$ARTIFACTS/tracie/tests/test.sh"
 
-ret=0
-
-"$ARTIFACTS/tracie/tracie.sh" "$ARTIFACTS/traces.yml" renderdoc || ret=1
-
-"$ARTIFACTS/tracie/tracie.sh" "$ARTIFACTS/traces.yml" apitrace || ret=1
-
-exit $ret
+python3 $ARTIFACTS/tracie/tracie.py --file $ARTIFACTS/traces.yml --device-name $DEVICE_NAME
diff --git a/.gitlab-ci/tracie-runner-vk.sh b/.gitlab-ci/tracie-runner-vk.sh
index 81ac001578f..e3f8c5d5319 100755
--- a/.gitlab-ci/tracie-runner-vk.sh
+++ b/.gitlab-ci/tracie-runner-vk.sh
@@ -22,7 +22,6 @@ ret=0
 # file:
 # https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section
 PATH="/gfxreconstruct/build/bin:$PATH" \
-    "$ARTIFACTS/tracie/tracie.sh"  "$ARTIFACTS/traces.yml" gfxreconstruct \
-    || ret=1
+    python3 $ARTIFACTS/tracie/tracie.py --file $ARTIFACTS/traces.yml --device-name $DEVICE_NAME
 
 exit $ret
diff --git a/.gitlab-ci/tracie/renderdoc_dump_images.py b/.gitlab-ci/tracie/renderdoc_dump_images.py
index 689ee2d0a87..93e24b9ca25 100755
--- a/.gitlab-ci/tracie/renderdoc_dump_images.py
+++ b/.gitlab-ci/tracie/renderdoc_dump_images.py
@@ -22,9 +22,22 @@
 #
 # SPDX-License-Identifier: MIT
 
+import atexit
+import os
+import shutil
 import sys
+import tempfile
 from pathlib import Path
 
+def cleanup(dirpath):
+        shutil.rmtree(dirpath)
+
+dirpath = tempfile.mkdtemp()
+atexit.register(cleanup, dirpath)
+RENDERDOC_DEBUG_FILE = dirpath + "/renderdoc.log"
+
+# Needs to be in the environment before importing the module
+os.environ['RENDERDOC_DEBUG_LOG_FILE'] = RENDERDOC_DEBUG_FILE
 import renderdoc as rd
 
 def findDrawWithEventId(controller, eventId):
@@ -75,11 +88,16 @@ def loadCapture(filename):
     if not cap.LocalReplaySupport():
         raise RuntimeError("Capture cannot be replayed")
 
-    status,controller = cap.OpenCapture(rd.ReplayOptions(), None)
+    status, controller = cap.OpenCapture(rd.ReplayOptions(), None)
 
     if status != rd.ReplayStatus.Succeeded:
+        if os.path.exists(RENDERDOC_DEBUG_FILE):
+            print(open(RENDERDOC_DEBUG_FILE, "r").read())
         raise RuntimeError("Couldn't initialise replay: " + str(status))
 
+    if os.path.exists(RENDERDOC_DEBUG_FILE):
+        open(RENDERDOC_DEBUG_FILE, "w").write("")
+
     return (cap, controller)
 
 def renderdoc_dump_images(filename, eventIds, outputDir):
diff --git a/.gitlab-ci/tracie/tests/test.sh b/.gitlab-ci/tracie/tests/test.sh
index 4d52578c47e..cd0f08e4efd 100755
--- a/.gitlab-ci/tracie/tests/test.sh
+++ b/.gitlab-ci/tracie/tests/test.sh
@@ -4,25 +4,6 @@ TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.."
 TEST_DIR=""
 TEST_EXIT=0
 
-create_repo() {
-    repo="$(mktemp -d $TEST_DIR/repo.XXXXXXXXXX)"
-    cp -R "$TEST_DIR"/tests/test-data/* "$repo"
-    (
-     cd "$repo";
-     git init -q .;
-     git config user.email "me@example.com"
-     git config user.name "Me me"
-     git lfs track '*.testtrace' > /dev/null;
-     git add .;
-     git commit -q -a -m 'initial';
-    )
-    echo $repo
-}
-
-destroy_repo() {
-    [ -d "$1"/.git ] && rm -rf "$1"
-}
-
 assert() {
     if ! $1; then
         echo "Assertion failed:  \"$1\""
@@ -32,22 +13,22 @@ assert() {
 
 run_tracie() {
     # Run tests for the .testtrace types, using the "gl-test-device" and "vk-test-device" device names.
-    DEVICE_NAME=gl-test-device CI_PROJECT_DIR="$TEST_DIR" \
-        "$TEST_DIR/tracie.sh" "$TEST_DIR/tests/traces.yml" testtrace && \
-    DEVICE_NAME=vk-test-device CI_PROJECT_DIR="$TEST_DIR" \
-        "$TEST_DIR/tracie.sh" "$TEST_DIR/tests/traces.yml" testtrace
+    python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name gl-test-device && \
+    python3 $TEST_DIR/tracie.py --file $TEST_DIR/tests/traces.yml --device-name vk-test-device
 }
 
 cleanup() {
-    rm -rf "$TEST_DIR"
+    [ "$TEST_DIR" = "/tmp/*" ] && rm -rf "$TEST_DIR"
 }
 
 prepare_for_run() {
     TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)"
-    # Copy all the tracie scripts to the test dir and later make that the
-    # CI_PROJECT_DIR for the run-tests.sh script. This avoids polluting the
-    # normal working dir with test result artifacts.
+    # Copy all the tracie scripts to the test dir for the run-tests.sh script.
+    # This avoids polluting the normal working dir with test result artifacts.
     cp -R "$TRACIE_DIR"/. "$TEST_DIR"
+    cd "$TEST_DIR"
+    mkdir traces-db
+    mv tests/test-data/* traces-db/.
     trap cleanup EXIT
     # Ensure we have a clean environment.
     unset TRACIE_STORE_IMAGES
@@ -76,89 +57,41 @@ run_test() {
 }
 
 tracie_succeeds_if_all_images_match() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     run_tracie
     assert "[ $? = 0 ]"
-
-    destroy_repo "$repo"
 }
 
 tracie_fails_on_image_mismatch() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
         "$TEST_DIR/tests/traces.yml"
 
     run_tracie
     assert "[ $? != 0 ]"
-
-    destroy_repo "$repo"
-}
-
-tracie_ignores_unspecified_trace_types() {
-    repo="$(create_repo)"
-    cd "$repo"
-
-    echo "  - path: trace1/empty.trace" >> "$TEST_DIR/tests/traces.yml"
-    echo "    expectations:" >> "$TEST_DIR/tests/traces.yml"
-    echo "    - device: gl-test-device" >> "$TEST_DIR/tests/traces.yml"
-    echo "      checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml"
-    # For the tests we only scan for the .testtrace type,
-    # so the .trace file added below should be ignored.
-    echo "empty" > trace1/empty.trace
-    git lfs track '*.trace'
-    git add trace1
-    git commit -a -m 'break'
-
-    run_tracie
-    assert "[ $? = 0 ]"
-
-    destroy_repo "$repo"
 }
 
 tracie_skips_traces_without_checksum() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     echo "  - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml"
     echo "    expectations:" >> "$TEST_DIR/tests/traces.yml"
     echo "    - device: bla" >> "$TEST_DIR/tests/traces.yml"
     echo "      checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml"
     # red.testtrace should be skipped, since it doesn't
     # have any checksums for our device
-    echo "ff0000ff" > trace1/red.testtrace
-    git add trace1
-    git commit -a -m 'red'
+    echo "ff0000ff" > traces-db/trace1/red.testtrace
 
     run_tracie
     assert "[ $? = 0 ]"
-
-    destroy_repo "$repo"
 }
 
 tracie_fails_on_dump_image_error() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     # "invalid" should fail to parse as rgba and
     # cause an error
-    echo "invalid" > trace1/magenta.testtrace
-    git add trace1
-    git commit -a -m 'invalid'
+    echo "invalid" > traces-db/trace1/magenta.testtrace
 
     run_tracie
     assert "[ $? != 0 ]"
-
-    destroy_repo "$repo"
 }
 
 tracie_stores_only_logs_on_checksum_match() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     run_tracie
     assert "[ $? = 0 ]"
 
@@ -169,14 +102,9 @@ tracie_stores_only_logs_on_checksum_match() {
     assert "[ ! -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
 
     ls -lR "$TEST_DIR"
-
-    destroy_repo "$repo"
 }
 
 tracie_stores_images_on_checksum_mismatch() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
         "$TEST_DIR/tests/traces.yml"
 
@@ -185,14 +113,9 @@ tracie_stores_images_on_checksum_mismatch() {
 
     assert "[ ! -f "$TEST_DIR/results/trace1/test/gl-test-device/magenta.testtrace-0.png" ]"
     assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
-
-    destroy_repo "$repo"
 }
 
 tracie_stores_images_on_request() {
-    repo="$(create_repo)"
-    cd "$repo"
-
     (export TRACIE_STORE_IMAGES=1; run_tracie)
     assert "[ $? = 0 ]"
 
@@ -200,13 +123,10 @@ tracie_stores_images_on_request() {
     assert "[ -f "$TEST_DIR/results/trace2/test/vk-test-device/olive.testtrace-0.png" ]"
 
     ls -lR "$TEST_DIR"
-
-    destroy_repo "$repo"
 }
 
 run_test tracie_succeeds_if_all_images_match
 run_test tracie_fails_on_image_mismatch
-run_test tracie_ignores_unspecified_trace_types
 run_test tracie_skips_traces_without_checksum
 run_test tracie_fails_on_dump_image_error
 run_test tracie_stores_only_logs_on_checksum_match
diff --git a/.gitlab-ci/tracie/tracie.py b/.gitlab-ci/tracie/tracie.py
new file mode 100644
index 00000000000..445f566f200
--- /dev/null
+++ b/.gitlab-ci/tracie/tracie.py
@@ -0,0 +1,167 @@
+import argparse
+import enum
+import glob
+import hashlib
+import os
+import requests
+import sys
+import tempfile
+import time
+import yaml
+
+from pathlib import Path
+from PIL import Image
+from urllib import parse
+
+import dump_trace_images
+
+TRACES_DB_PATH = os.getcwd() + "/traces-db/"
+RESULTS_PATH = os.getcwd() + "/results/"
+
+def replay(trace_path, device_name):
+    success = dump_trace_images.dump_from_trace(trace_path, [], device_name)
+
+    if not success:
+        print("[check_image] Trace %s couldn't be replayed. See above logs for more information." % (str(trace_path)))
+        return None, None, None
+    else:
+        base_path = trace_path.parent
+        file_name = trace_path.name
+        files = glob.glob(str(base_path / "test" / device_name / (file_name + "-*" + ".png")))
+        assert(files)
+        image_file = files[0]
+        files = glob.glob(str(base_path / "test" / device_name / (file_name + ".log")))
+        assert(files)
+        log_file = files[0]
+        return hashlib.md5(Image.open(image_file).tobytes()).hexdigest(), image_file, log_file
+
+def download_metadata(repo_url, repo_commit, trace_path):
+    # The GitLab API doesn't want the .git postfix
+    url = repo_url
+    if url.endswith(".git"):
+        url = url[:-4]
+    url = parse.urlparse(url)
+
+    url_path = url.path
+    if url_path.startswith("/"):
+        url_path = url_path[1:]
+
+    gitlab_api_url = url.scheme + "://" + url.netloc + "/api/v4/projects/" + parse.quote_plus(url_path)
+
+    r = requests.get(gitlab_api_url + "/repository/files/%s/raw?ref=%s" % (parse.quote_plus(trace_path), repo_commit))
+    metadata_raw = r.text.strip().split('\n')
+    metadata = dict(line.split(' ', 1) for line in metadata_raw[1:])
+    oid = metadata["oid"][7:] if metadata["oid"].startswith('sha256:') else metadata["oid"]
+    size = int(metadata['size'])
+
+    return oid, size
+
+def download_trace(repo_url, repo_commit, trace_path, oid, size):
+    headers = {
+        "Accept": "application/vnd.git-lfs+json",
+        "Content-Type": "application/vnd.git-lfs+json"
+    }
+    json = {
+        "operation": "download",
+        "transfers": [ "basic" ],
+        "ref": { "name": "refs/heads/%s" % repo_commit },
+        "objects": [
+            {
+                "oid": oid,
+                "size": size
+            }
+        ]
+    }
+
+    # The LFS API really wants the .git postfix...
+    if not repo_url.endswith(".git"):
+        repo_url += ".git"
+
+    r = requests.post(repo_url + "/info/lfs/objects/batch", headers=headers, json=json)
+    url = r.json()["objects"][0]["actions"]["download"]["href"]
+    open(TRACES_DB_PATH + trace_path, "wb").write(requests.get(url).content)
+
+def checksum(filename, hash_factory=hashlib.sha256, chunk_num_blocks=128):
+    h = hash_factory()
+    with open(filename,'rb') as f:
+        for chunk in iter(lambda: f.read(chunk_num_blocks*h.block_size), b''):
+            h.update(chunk)
+    return h.digest()
+
+def ensure_trace(repo_url, repo_commit, trace):
+    trace_path = TRACES_DB_PATH + trace['path']
+    if repo_url is None:
+        assert(repo_commit is None)
+        assert(os.path.exists(trace_path))
+        return
+
+    os.makedirs(os.path.dirname(trace_path), exist_ok=True)
+
+    if os.path.exists(trace_path):
+        local_oid = checksum(trace_path)
+
+    remote_oid, size = download_metadata(repo_url, repo_commit, trace['path'])
+
+    if not os.path.exists(trace_path) or local_oid != remote_oid:
+        print("[check_image] Downloading trace %s" % (trace['path']), end=" ", flush=True)
+        download_time = time.time()
+        download_trace(repo_url, repo_commit, trace['path'], remote_oid, size)
+        print("took %ds." % (time.time() - download_time), flush=True)
+
+def check_trace(repo_url, repo_commit, device_name, trace, expectation):
+    ensure_trace(repo_url, repo_commit, trace)
+
+    trace_path = Path(TRACES_DB_PATH + trace['path'])
+    checksum, image_file, log_file = replay(trace_path, device_name)
+    if checksum is None:
+            return False
+    elif checksum == expectation['checksum']:
+            print("[check_image] Images match for %s" % (trace['path']))
+            ok = True
+    else:
+            print("[check_image] Images differ for %s (expected: %s, actual: %s)" %
+                  (trace['path'], expectation['checksum'], checksum))
+            print("[check_image] For more information see "
+                  "https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md")
+            ok = False
+
+    trace_dir = os.path.split(trace['path'])[0]
+    results_path = os.path.join(RESULTS_PATH, trace_dir, "test", device_name)
+    os.makedirs(results_path, exist_ok=True)
+    os.rename(log_file, os.path.join(results_path, os.path.split(log_file)[1]))
+    if not ok or os.environ.get('TRACIE_STORE_IMAGES', '0') == '1':
+            os.rename(image_file, os.path.join(results_path, os.path.split(image_file)[1]))
+
+    return ok
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--file', required=True,
+                        help='the name of the traces.yml file listing traces and their checksums for each device')
+    parser.add_argument('--device-name', required=True,
+                        help="the name of the graphics device used to replay traces")
+
+    args = parser.parse_args()
+
+    with open(args.file, 'r') as f:
+        y = yaml.safe_load(f)
+
+    if "traces-db" in y:
+        repo = y["traces-db"]["repo"]
+        commit_id = y["traces-db"]["commit"]
+    else:
+        repo = None
+        commit_id = None
+
+    traces = y['traces']
+    all_ok = True
+    for trace in traces:
+        for expectation in trace['expectations']:
+                if expectation['device'] == args.device_name:
+                        ok = check_trace(repo, commit_id, args.device_name, trace, expectation)
+                        all_ok = all_ok and ok
+
+    sys.exit(0 if all_ok else 1)
+
+if __name__ == "__main__":
+    main()
diff --git a/.gitlab-ci/tracie/tracie.sh b/.gitlab-ci/tracie/tracie.sh
deleted file mode 100755
index afae5be3365..00000000000
--- a/.gitlab-ci/tracie/tracie.sh
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-
-TRACIE_SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
-TRACES_YAML="$(readlink -f "$1")"
-TRACE_TYPE="$2"
-
-# Clone the traces-db repo without a checkout. Since we are dealing with
-# git-lfs repositories, such clones are very lightweight. We check out
-# individual files as needed at a later stage (see fetch_trace).
-clone_traces_db_no_checkout()
-{
-    local repo="$1"
-    local commit="$2"
-    rm -rf traces-db
-    git clone --no-checkout -c lfs.storage="$CI_PROJECT_DIR/.git-lfs-storage" "$repo" traces-db
-    (cd traces-db; git reset "$commit" || git reset "origin/$commit")
-}
-
-query_traces_yaml()
-{
-    python3 "$TRACIE_SCRIPT_DIR/query_traces_yaml.py" \
-        --file "$TRACES_YAML" "$@"
-}
-
-create_clean_git()
-{
-    rm -rf .clean_git
-    cp -R .git .clean_git
-}
-
-restore_clean_git()
-{
-    rm -rf .git
-    cp -R .clean_git .git
-}
-
-fetch_trace()
-{
-    local trace="${1//,/?}"
-    echo -n "[fetch_trace] Fetching $1... "
-    local output=$(git lfs pull -I "$trace" 2>&1)
-    local ret=0
-    if [[ $? -ne 0 || ! -f "$1" ]]; then
-        echo "ERROR"
-        echo "$output"
-        ret=1
-    else
-        echo "OK"
-    fi
-    # Restore a clean .git directory, effectively removing any downloaded
-    # git-lfs objects, in order to limit required storage. Note that the
-    # checked out trace file is still present at this point. We remove it
-    # when we are done with the trace replay at a later stage.
-    restore_clean_git
-    return $ret
-}
-
-get_dumped_file()
-{
-    local trace="$1"
-    local tracedir="$(dirname "$trace")"
-    local tracename="$(basename "$trace")"
-
-    find "$tracedir/test/$DEVICE_NAME" -name "$tracename*.$2"
-}
-
-check_image()
-{
-    local trace="$1"
-    local image="$2"
-
-    checksum=$(python3 "$TRACIE_SCRIPT_DIR/image_checksum.py" "$image")
-    expected=$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")
-    if [[ "$checksum" = "$expected" ]]; then
-        echo "[check_image] Images match for $trace"
-        return 0
-    else
-        echo "[check_image] Images differ for $trace (expected: $expected, actual: $checksum)"
-        echo "[check_image] For more information see https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md"
-        return 1
-    fi
-}
-
-archive_artifact()
-{
-    mkdir -p "$CI_PROJECT_DIR/results"
-    cp --parents "$1" "$CI_PROJECT_DIR/results"
-}
-
-if [[ -n "$(query_traces_yaml traces_db_repo)" ]]; then
-    clone_traces_db_no_checkout "$(query_traces_yaml traces_db_repo)" \
-                                "$(query_traces_yaml traces_db_commit)"
-    cd traces-db
-else
-    echo "Warning: No traces-db entry in $TRACES_YAML, assuming traces-db is current directory"
-fi
-
-# During git operations various git objects get created which
-# may take up significant space. Store a clean .git instance,
-# which we restore after various git operations to keep our
-# storage consumption low.
-create_clean_git
-
-ret=0
-
-for trace in $(query_traces_yaml traces --device-name "$DEVICE_NAME" --trace-types "$TRACE_TYPE")
-do
-    [[ -n "$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")" ]] ||
-        { echo "[fetch_trace] Skipping $trace since it has no checksums for $DEVICE_NAME"; continue; }
-    fetch_trace "$trace" || exit $?
-    python3 "$TRACIE_SCRIPT_DIR/dump_trace_images.py" --device-name "$DEVICE_NAME" "$trace" || exit $?
-    image="$(get_dumped_file "$trace" png)"
-    check_image "$trace" "$image" && check_succeeded=true || { ret=1; check_succeeded=false; }
-    if [[ "$check_succeeded" = false || "$TRACIE_STORE_IMAGES" = "1" ]]; then
-        archive_artifact "$image"
-    fi
-    archive_artifact "$(get_dumped_file "$trace" log)"
-    # Remove the downloaded trace file to reduce the total amount of storage
-    # that is required.
-    rm "$trace"
-done
-
-exit $ret