From 758b7ca7affed045c82fd5caef36c19c0ce82b36 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 17 Aug 2021 15:58:41 +0530 Subject: cerbero: Add a dotted progress bar for urllib downloads Also add a 20s timeout (same as wget) so that we timeout on download instead of timing out the entire job on CI. The global socket timeout with CPython is apparently -1, aka never timeout. Part-of: --- cerbero/utils/shell.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/cerbero/utils/shell.py b/cerbero/utils/shell.py index fda95832..f3be6c1b 100644 --- a/cerbero/utils/shell.py +++ b/cerbero/utils/shell.py @@ -377,6 +377,8 @@ async def download_urllib2(url, destination=None, check_cert=True, overwrite=Fal @param destination: destination where the file will be saved @type destination: str ''' + # 1MiB chunk size (same as dot:giga for wget) + chunk_size = 1024 * 1024 ctx = None if not check_cert: import ssl @@ -387,12 +389,25 @@ async def download_urllib2(url, destination=None, check_cert=True, overwrite=Fal if not destination: destination = os.path.basename(url) + from datetime import datetime + start_time = datetime.now() + try: with open(destination, 'wb') as d: req = urllib.request.Request(url) req.add_header('User-Agent', USER_AGENT) - f = urllib.request.urlopen(req, context=ctx) - d.write(f.read()) + f = urllib.request.urlopen(req, timeout=20, context=ctx) + while True: + chunk = f.read(chunk_size) + if not chunk: + break + d.write(chunk) + print('.', end='', file=logfile, flush=True) + total_time = (datetime.now() - start_time).total_seconds() + size = d.tell() / 1024 + speed = size / total_time + print('\nDownloaded {:2,.2f} KiB in {:2,.2f} seconds at {:2,.2f} KiB/s' + .format(size, total_time, speed), file=logfile, flush=True) except urllib.error.HTTPError as e: if os.path.exists(destination): os.remove(destination) -- cgit v1.2.3