Improve support for mirrors

Use a default list of mirrors and add the option
to extend that list in the config.

3 files changed, 39 insertions, 15 deletions

diff --git a/cerbero/build/source.py b/cerbero/build/source.py
index c59cf99a..03f9db6a 100644
--- a/cerbero/build/source.py
+++ b/cerbero/build/source.py
@@ -22,14 +22,11 @@ import tarfile
 import urllib.request, urllib.parse, urllib.error
 from hashlib import sha256
 
-from cerbero.config import Platform
+from cerbero.config import Platform, DEFAULT_MIRRORS
 from cerbero.utils import git, svn, shell, _
 from cerbero.errors import FatalError, InvalidRecipeError
 import cerbero.utils.messages as m
 
-# Must end in a / for urlparse.urljoin to work correctly
-TARBALL_MIRROR = 'https://gstreamer.freedesktop.org/src/mirror/'
-
 URL_TEMPLATES = {
     'gnome': ('https://download.gnome.org/sources/', '%(name)s/%(maj_ver)s/%(name)s-%(version)s', '.tar.xz'),
     'gnu': ('https://ftpmirror.gnu.org/', '%(name)s/%(name)s-%(version)s', '.tar.xz'),
@@ -167,7 +164,6 @@ class BaseTarball(object):
         split = list(urllib.parse.urlsplit(self.url))
         split[2] = urllib.parse.quote(split[2])
         self.url = urllib.parse.urlunsplit(split)
-        self.mirror_url = urllib.parse.urljoin(TARBALL_MIRROR, self.tarball_name)
         o = urllib.parse.urlparse(self.url)
         if o.scheme in ('http', 'ftp'):
             raise FatalError('Download URL {!r} must use HTTPS'.format(self.url))
@@ -187,13 +183,9 @@ class BaseTarball(object):
         # Enable certificate checking only on Linux for now
         # FIXME: Add more platforms here after testing
         cc = self.config.platform == Platform.LINUX
-        try:
-            shell.download(self.url, self.download_path, check_cert=cc,
-                           overwrite=redownload, logfile=get_logfile(self))
-        except (FatalError, urllib.error.URLError):
-            # Try our mirror
-            shell.download(self.mirror_url, self.download_path, check_cert=cc,
-                           overwrite=redownload, logfile=get_logfile(self))
+        shell.download(self.url, self.download_path, check_cert=cc,
+            overwrite=redownload, logfile=get_logfile(self),
+            mirrors= self.config.extra_mirrors + DEFAULT_MIRRORS)
         self.verify()
 
     @staticmethod
diff --git a/cerbero/config.py b/cerbero/config.py
index 6b574100..927c7c16 100644
--- a/cerbero/config.py
+++ b/cerbero/config.py
@@ -41,6 +41,7 @@ DEFAULT_ALLOW_PARALLEL_BUILD = True
 DEFAULT_PACKAGER = "Default <default@change.me>"
 CERBERO_UNINSTALLED = 'CERBERO_UNINSTALLED'
 CERBERO_PREFIX = 'CERBERO_PREFIX'
+DEFAULT_MIRRORS = ['https://gstreamer.freedesktop.org/src/mirror/']
 
 
 Platform = enums.Platform
@@ -114,7 +115,7 @@ class Config (object):
                    'msvc_version', 'msvc_toolchain_env', 'mingw_toolchain_env',
                    'meson_cross_properties', 'manifest', 'extra_properties',
                    'qt5_qmake_path', 'qt5_pkgconfigdir', 'for_shell',
-                   'package_tarball_compression']
+                   'package_tarball_compression', 'extra_mirrors']
 
     cookbook = None
 
@@ -411,6 +412,7 @@ class Config (object):
         self.set_property('meson_cross_properties', {})
         self.set_property('manifest', None)
         self.set_property('extra_properties', {})
+        self.set_property('extra_mirrors', [])
 
     def set_property(self, name, value, force=False):
         if name not in self._properties:
diff --git a/cerbero/utils/shell.py b/cerbero/utils/shell.py
index 9aa68f7b..b09dbf02 100644
--- a/cerbero/utils/shell.py
+++ b/cerbero/utils/shell.py
@@ -397,7 +397,23 @@ def download_curl(url, destination=None, check_cert=True, overwrite=False):
             os.remove(destination)
         raise e
 
-def download(url, destination=None, check_cert=True, overwrite=False, logfile=None):
+def download(url, destination=None, check_cert=True, overwrite=False, logfile=None, mirrors=None):
+    '''
+    Downloads a file
+
+    @param url: url to download
+    @type: str
+    @param destination: destination where the file will be saved
+    @type destination: str
+    @param check_cert: whether to check certificates or not
+    @type check_cert: bool
+    @param overwrite: whether to overwrite the destination or not
+    @type check_cert: bool
+    @param logfile: path to the file to log instead of stdout
+    @type logfile: str
+    @param mirrors: list of mirrors to use as fallback
+    @type logfile: list
+    '''
     if not overwrite and os.path.exists(destination):
         if logfile is None:
             logging.info("File %s already downloaded." % destination)
@@ -407,6 +423,13 @@ def download(url, destination=None, check_cert=True, overwrite=False, logfile=No
             os.makedirs(os.path.dirname(destination))
         log("Downloading {}".format(url), logfile)
 
+    urls = [url]
+    if mirrors is not None:
+        filename = os.path.basename(url)
+        # Add a traling '/' the url so that urljoin joins correctly urls
+        # in case users provided it without the trailing '/'
+        urls += [urllib.parse.urljoin(u + '/', filename) for u in mirrors]
+
     # wget shipped with msys fails with an SSL error on github URLs
     # https://githubengineering.com/crypto-removal-notice/
     # curl on Windows (if provided externally) is often badly-configured and fails
@@ -420,7 +443,14 @@ def download(url, destination=None, check_cert=True, overwrite=False, logfile=No
     else:
         # Fallback. TODO: make this the default and remove curl/wget dependency
         download_func = download_urllib2
-    return download_func(url, destination, check_cert, overwrite)
+
+    errors = []
+    for murl in urls:
+        try:
+            return download_func(murl, destination, check_cert, overwrite)
+        except Exception as ex:
+            errors.append(ex)
+    raise Exception (errors)
 
 
 def _splitter(string, base_url):