cerbero: Allow bootstrappers to dynamically fetch URLs

The Rust bootstrapper (added in this commit series) needs to download
the Rust package manifest, parse it to get the package URLs
+ checksums, and then download those too.

This all needs to happen in the fetch phase, so we add a "fetch urls
function" that can yield more URLs to fetch after the static
fetch_urls list is fetched.

Part-of: <https://gitlab.freedesktop.org/gstreamer/cerbero/-/merge_requests/915>

1 files changed, 11 insertions, 3 deletions

diff --git a/cerbero/bootstrap/__init__.py b/cerbero/bootstrap/__init__.py
index 5d7f06a5..43e6ddf9 100644
--- a/cerbero/bootstrap/__init__.py
+++ b/cerbero/bootstrap/__init__.py
@@ -41,6 +41,8 @@ class BootstrapTarball(BaseTarball, Source):
 class BootstrapperBase (object):
     # List of URLs to be fetched
     fetch_urls = None
+    # A function that returns more URLs to fetch
+    fetch_urls_func = None
     # List of extract steps to be performed
     extract_steps = None
 
@@ -54,14 +56,20 @@ class BootstrapperBase (object):
     def start(self):
         raise NotImplemented("'start' must be implemented by subclasses")
 
-    async def fetch(self):
-        'Fetch bootstrap binaries'
-        for (url, name, checksum) in self.fetch_urls:
+    async def fetch_urls_impl(self, urls):
+        for (url, name, checksum) in urls:
             source = BootstrapTarball(self.config, self.offline, url, checksum,
                                       self.config.local_sources, tarball_name=name)
             self.sources[url] = source
             await source.fetch()
 
+    async def fetch(self):
+        'Fetch bootstrap binaries'
+        await self.fetch_urls_impl(self.fetch_urls)
+        if self.fetch_urls_func:
+            more_urls = self.fetch_urls_func()
+            await self.fetch_urls_impl(more_urls)
+
     def fetch_recipes(self, jobs):
         'Fetch build-tools recipes; only called by fetch-bootstrap'
         pass