cerbero: Implement --offline for package and build

This forces Cerbero to not attempt to update anything over the network,
especially git repositories. Very useful for reproducible builds.

The remaining bit is implementing it for bootstrap, which is much more
complicated.

https://bugzilla.gnome.org/show_bug.cgi?id=797048

6 files changed, 48 insertions, 18 deletions

diff --git a/cerbero/build/cookbook.py b/cerbero/build/cookbook.py
index d01a0498..597bb572 100644
--- a/cerbero/build/cookbook.py
+++ b/cerbero/build/cookbook.py
@@ -86,7 +86,8 @@ class CookBook (object):
 
     RECIPE_EXT = '.recipe'
 
-    def __init__(self, config, load=True):
+    def __init__(self, config, load=True, offline=False):
+        self.offline = offline
         self.set_config(config)
         self.recipes = {}  # recipe_name -> recipe
         self._invalid_recipes = {} # recipe -> error
@@ -343,6 +344,9 @@ class CookBook (object):
 
         # Check for updates in the recipe file to reset the status
         for recipe in list(self.recipes.values()):
+            # Set the offline property, used by the recipe while performing the
+            # fetch build step
+            recipe.offline = self.offline
             if recipe.name not in self.status:
                 continue
             st = self.status[recipe.name]
diff --git a/cerbero/build/source.py b/cerbero/build/source.py
index 6b03de37..f05d2b58 100644
--- a/cerbero/build/source.py
+++ b/cerbero/build/source.py
@@ -45,6 +45,7 @@ class Source (object):
 
     patches = []
     strip = 1
+    offline = False
 
     def fetch(self):
         '''
@@ -122,6 +123,12 @@ class Tarball (Source):
                      (cached_file, self.download_path, self.url))
             shutil.copy(cached_file, self.download_path)
             return
+        if self.offline:
+            if not os.path.isfile(self.download_path):
+                msg = 'Offline mode: tarball {!r} not found in cached sources ({}) or local sources ({})'
+                raise FatalError(msg.format(self.tarball_name, self.config.cached_sources, self.repo_dir))
+            m.action(_('Found tarball for %s at %s') % (self.url, self.download_path))
+            return
         m.action(_('Fetching tarball %s to %s') %
                  (self.url, self.download_path))
         # Enable certificate checking only on Linux for now
@@ -192,11 +199,15 @@ class GitCache (Source):
 
     def fetch(self, checkout=True):
         self._git_env_setup()
+        # First try to get the sources from the cached dir if there is one
+        cached_dir = os.path.join(self.config.cached_sources,  self.name)
+
         if not os.path.exists(self.repo_dir):
+            if not cached_dir and offline:
+                msg = 'Offline mode: git repo for {!r} not found in cached sources ({}) or local sources ({})'
+                raise FatalError(msg.format(self.name, self.config.cached_sources, self.repo_dir))
             git.init(self.repo_dir)
 
-        # First try to get the sources from the cached dir if there is one
-        cached_dir = os.path.join(self.config.cached_sources,  self.name)
         if os.path.isdir(os.path.join(cached_dir, ".git")):
             for remote, url in self.remotes.items():
                 git.add_remote(self.repo_dir, remote, "file://" + cached_dir)
@@ -212,11 +223,12 @@ class GitCache (Source):
             for remote, url in self.config.recipe_remotes(self.name).items():
                 git.add_remote(self.repo_dir, remote, url)
             # fetch remote branches
-            git.fetch(self.repo_dir, fail=False)
+            if not self.offline:
+                git.fetch(self.repo_dir, fail=False)
         if checkout:
             commit = self.config.recipe_commit(self.name) or self.commit
             git.checkout(self.repo_dir, commit)
-            git.submodules_update(self.repo_dir, cached_dir, fail=False)
+            git.submodules_update(self.repo_dir, cached_dir, fail=False, offline=self.offline)
         self._git_env_restore()
 
 
@@ -280,6 +292,7 @@ class Git (GitCache):
     def __init__(self):
         GitCache.__init__(self)
         if self.commit is None:
+            # Used by recipes in recipes/toolchain/
             self.commit = 'origin/sdk-%s' % self.version
         # For forced commits in the config
         self.commit = self.config.recipe_commit(self.name) or self.commit
@@ -378,6 +391,9 @@ class Svn(Source):
             return
 
         os.makedirs(self.repo_dir)
+        if self.offline:
+            raise FatalError('Offline mode: no cached svn repos found for {} at {!r}'
+                             ''.format(self.package_name, self.config.cached_sources))
         svn.checkout(self.url, self.repo_dir)
         svn.update(self.repo_dir, self.revision)
 
diff --git a/cerbero/commands/build.py b/cerbero/commands/build.py
index a0c62f93..17fde044 100644
--- a/cerbero/commands/build.py
+++ b/cerbero/commands/build.py
@@ -38,7 +38,10 @@ class Build(Command):
                            'listed in the recipe')),
                 ArgparseArgument('--dry-run', action='store_true',
                     default=False,
-                    help=_('only print commands instead of running them '))]
+                    help=_('only print commands instead of running them ')),
+                ArgparseArgument('--offline', action='store_true',
+                    default=False, help=_('Use only the source cache, no network')),
+                ]
             if force is None:
                 args.append(
                     ArgparseArgument('--force', action='store_true',
@@ -61,12 +64,12 @@ class Build(Command):
         if self.no_deps is None:
             self.no_deps = args.no_deps
         self.runargs(config, args.recipe, args.missing_files, self.force,
-                     self.no_deps, dry_run=args.dry_run)
+                     self.no_deps, dry_run=args.dry_run, offline=args.offline)
 
     def runargs(self, config, recipes, missing_files=False, force=False,
-                no_deps=False, cookbook=None, dry_run=False):
+                no_deps=False, cookbook=None, dry_run=False, offline=False):
         if cookbook is None:
-            cookbook = CookBook(config)
+            cookbook = CookBook(config, offline=offline)
 
         oven = Oven(recipes, cookbook, force=self.force,
                     no_deps=self.no_deps, missing_files=missing_files,
diff --git a/cerbero/commands/package.py b/cerbero/commands/package.py
index 1068b60a..d1d49391 100644
--- a/cerbero/commands/package.py
+++ b/cerbero/commands/package.py
@@ -58,10 +58,12 @@ class Package(Command):
                     'create this package (conflicts with --skip-deps-build)')),
             ArgparseArgument('-k', '--keep-temp', action='store_true',
                 default=False, help=_('Keep temporary files for debug')),
+            ArgparseArgument('--offline', action='store_true',
+                default=False, help=_('Use only the source cache, no network')),
             ])
 
     def run(self, config, args):
-        self.store = PackagesStore(config)
+        self.store = PackagesStore(config, offline=args.offline)
         p = self.store.get_package(args.package[0])
 
         if args.skip_deps_build and args.only_build_deps:
@@ -69,7 +71,7 @@ class Package(Command):
                     "--only-build-deps"))
 
         if not args.skip_deps_build:
-            self._build_deps(config, p, args.no_devel)
+            self._build_deps(config, p, args.no_devel, args.offline)
 
         if args.only_build_deps:
             return
@@ -93,10 +95,10 @@ class Package(Command):
         m.action(_("Package successfully created in %s") %
                  ' '.join([os.path.abspath(x) for x in paths]))
 
-    def _build_deps(self, config, package, has_devel):
+    def _build_deps(self, config, package, has_devel, offline):
         build_command = build.Build()
         build_command.runargs(config, package.recipes_dependencies(has_devel),
-            cookbook=self.store.cookbook)
+            cookbook=self.store.cookbook, offline=offline)
 
 
 register_command(Package)
diff --git a/cerbero/packages/packagesstore.py b/cerbero/packages/packagesstore.py
index e089cc16..a0cb1e42 100644
--- a/cerbero/packages/packagesstore.py
+++ b/cerbero/packages/packagesstore.py
@@ -36,12 +36,12 @@ class PackagesStore (object):
 
     PKG_EXT = '.package'
 
-    def __init__(self, config, load=True):
+    def __init__(self, config, load=True, offline=False):
         self._config = config
 
         self._packages = {}  # package_name -> package
 
-        self.cookbook = CookBook(config, load)
+        self.cookbook = CookBook(config, load=load, offline=offline)
         # used in tests to skip loading a dir with packages definitions
         if not load:
             return
diff --git a/cerbero/utils/git.py b/cerbero/utils/git.py
index d3b4f00a..852dc0d6 100644
--- a/cerbero/utils/git.py
+++ b/cerbero/utils/git.py
@@ -127,7 +127,7 @@ def fetch(git_dir, fail=True):
     '''
     return shell.call('%s fetch --all' % GIT, git_dir, fail=fail)
 
-def submodules_update(git_dir, src_dir=None, fail=True):
+def submodules_update(git_dir, src_dir=None, fail=True, offline=False):
     '''
     Update somdules from local directory
 
@@ -137,6 +137,8 @@ def submodules_update(git_dir, src_dir=None, fail=True):
     @type src_dir: src
     @param fail: raise an error if the command failed
     @type fail: false
+    @param offline: don't use the network
+    @type offline: false
     '''
     if src_dir:
         config = shell.check_call('%s config --file=.gitmodules --list' % GIT,
@@ -149,8 +151,11 @@ def submodules_update(git_dir, src_dir=None, fail=True):
                            (GIT, submodule, os.path.join(src_dir, c[1])),
                            git_dir)
     shell.call("%s submodule init" % GIT, git_dir)
-    shell.call("%s submodule sync" % GIT, git_dir)
-    shell.call("%s submodule update" % GIT, git_dir, fail=fail)
+    if src_dir or not offline:
+        shell.call("%s submodule sync" % GIT, git_dir)
+        shell.call("%s submodule update" % GIT, git_dir, fail=fail)
+    else:
+        shell.call("%s submodule update --no-fetch" % GIT, git_dir, fail=fail)
     if src_dir:
         for c in config_array:
             if c[0].startswith('submodule.') and c[0].endswith('.url'):