allow to put files listed in file into installation

With this in place, we can replace most of our Zips by Packages.
Extensions and Dictionaries are on the radar as well.

To move an installed file from zip to filelist, do:
1. Convert the Zip_foo.mk to Package_foo.mk :
   - change destination paths of all files to the same ones they have in
     the installation (you can find that in scp2)
   - use gb_Package_set_outdir to place the files under $INSTDIR, where
     they are expected.
2. Change the scp2 record:
   - change filename to <package-name>.filelist
   - replace ARCHIVE by FILELIST in Styles
   - change Dir to FILELIST_DIR.

Change-Id: Ie17d0765406081b03dcd44a6a23cf517f2067dd3
Reviewed-on: https://gerrit.libreoffice.org/3149
Reviewed-by: Fridrich Strba <fridrich@documentfoundation.org>
Tested-by: Fridrich Strba <fridrich@documentfoundation.org>

8 files changed, 129 insertions, 5 deletions

diff --git a/instsetoo_native/util/openoffice.lst.in b/instsetoo_native/util/openoffice.lst.in
index c6afdbdce070..5cf9840226b4 100644
--- a/instsetoo_native/util/openoffice.lst.in
+++ b/instsetoo_native/util/openoffice.lst.in
@@ -83,7 +83,7 @@ LibreOffice
         downloadname    LibreOffice_{productversion}_{os}_install_{languages}
         langpackdownloadname    LibreOffice_{productversion}_languagepack_{os}_install_{languages}
         helppackdownloadname    LibreOffice_{productversion}_helppack_{os}_install_{languages}
-        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin,{solarpath}/bin.{minor}/ooowoure,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external
+        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin,{solarpath}/bin.{minor}/ooowoure,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external,{filelistpath}
     }
 }
 
@@ -140,7 +140,7 @@ LibreOffice_Dev
         downloadname    LibreOfficeDev_{productversion}_{os}_install_{languages}
         langpackdownloadname    LibreOfficeDev_{productversion}_languagepack_{os}_install_{languages}
         helppackdownloadname    LibreOfficeDev_{productversion}_helppack_{os}_install_{languages}
-        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin/dev,{localpath}/bin,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice_dev,{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external
+        include         {solarenvpath}/{os}/loader2,.,{localpath}/bin/dev,{localpath}/bin,{solarpath}/bin.{minor}/osl,{solarpath}/bin.{minor}/desktop-integration/{pkgtype},{solarpath}/bin.{minor},{solarpath}/lib.{minor},{solarpath}/pck.{minor}/openoffice_dev,{solarpath}/pck.{minor}/openoffice,{solarpath}/pck.{minor},{solarpath}/xml.{minor}/office/instance,{solarpath}/xml.{minor},../../external/common,{solarenvpath}/{os}/OOo_external,{filelistpath}
     }
 }
 
diff --git a/scp2/inc/macros.inc b/scp2/inc/macros.inc
index 1b537aabaf38..dd4432c3966c 100755
--- a/scp2/inc/macros.inc
+++ b/scp2/inc/macros.inc
@@ -425,6 +425,12 @@ End
 #define SCP2_URE_JDL_NORMAL(n) SCP2_URE_DL_NORMAL(n)
 #endif
 
+#if defined MACOSX
+#define FILELIST_DIR gid_Dir_Bundle_Contents
+#else
+#define FILELIST_DIR gid_Dir_Brand_Root
+#endif
+
 #include <langmacros.inc>
 
 
diff --git a/solenv/bin/modules/installer.pm b/solenv/bin/modules/installer.pm
index c731d9e116fc..df0727f2af1f 100644
--- a/solenv/bin/modules/installer.pm
+++ b/solenv/bin/modules/installer.pm
@@ -41,6 +41,7 @@ use installer::download;
 use installer::environment;
 use installer::epmfile;
 use installer::files;
+use installer::filelists;
 use installer::globals;
 use installer::helppack;
 use installer::languagepack;
@@ -617,6 +618,14 @@ sub run {
             installer::scriptitems::quoting_illegal_filenames($filesinproductlanguageresolvedarrayref);
         }
 
+        ######################################################################################
+        # Processing files with flag FILELIST and putting listed files into the file list
+        ######################################################################################
+
+        installer::logger::print_message( "... analyzing files with flag FILELIST ...\n" );
+
+        $filesinproductlanguageresolvedarrayref = installer::filelists::resolve_filelist_flag($filesinproductlanguageresolvedarrayref, $ENV{'INSTDIR'});
+
         #####################################
         # Files with flag SUBST_FILENAME
         #####################################
diff --git a/solenv/bin/modules/installer/control.pm b/solenv/bin/modules/installer/control.pm
index 26ab41aa4724..87b961548e82 100644
--- a/solenv/bin/modules/installer/control.pm
+++ b/solenv/bin/modules/installer/control.pm
@@ -280,6 +280,7 @@ sub check_system_environment
         OUTPATH
         LOCAL_OUT
         LOCAL_COMMON_OUT
+        WORKDIR
     );
 
     for my $key ( @environmentvariables )
diff --git a/solenv/bin/modules/installer/environment.pm b/solenv/bin/modules/installer/environment.pm
index 646bf1649e00..cfc8fdf084c4 100644
--- a/solenv/bin/modules/installer/environment.pm
+++ b/solenv/bin/modules/installer/environment.pm
@@ -67,6 +67,9 @@ sub create_pathvariables
     my $platformname  = $environment->{'OUTPATH'};
     $variables{'platformname'} = $platformname;
 
+    my $filelistpath = $environment->{'WORKDIR'} . $installer::globals::separator . 'Package';
+    $variables{'filelistpath'} = $filelistpath;
+
     return \%variables;
 }
 
diff --git a/solenv/bin/modules/installer/filelists.pm b/solenv/bin/modules/installer/filelists.pm
new file mode 100644
index 000000000000..d3447be79163
--- /dev/null
+++ b/solenv/bin/modules/installer/filelists.pm
@@ -0,0 +1,95 @@
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+
+package installer::filelists;
+
+use installer::files;
+use installer::globals;
+use installer::logger;
+use installer::pathanalyzer;
+
+sub resolve_filelist_flag
+{
+    my ($files, $outdir) = @_;
+    my @newfiles = ();
+
+    foreach my $file (@{$files})
+    {
+        my $is_filelist = 0;
+        if ($file->{'Styles'})
+        {
+            if ($file->{'Styles'} =~ /\bFILELIST\b/)
+            {
+                $is_filelist = 1;
+            }
+        }
+
+        if ($is_filelist)
+        {
+            my $filelist_path = $file->{'sourcepath'};
+            my $filelist = read_filelist($filelist_path);
+            if (@{$filelist})
+            {
+                my $destination = $file->{'destination'};
+                installer::pathanalyzer::get_path_from_fullqualifiedname(\$destination);
+
+                foreach my $path (@{$filelist})
+                {
+                    # TODO: check that the file is really under $outdir
+                    # TODO: check existence of the file
+                    my $subpath = substr $path, ((length $outdir) + 1); # drop separator too
+
+                    my %newfile = ();
+                    %newfile = %{$file};
+                    $newfile{'Name'} = $subpath;
+                    $newfile{'sourcepath'} = $path;
+                    $newfile{'destination'} = $destination . $subpath;
+                    $newfile{'filelistname'} = $file->{'Name'};
+                    $newfile{'filelistpath'} = $file->{'sourcepath'};
+
+                    push @newfiles, \%newfile;
+                }
+            }
+            else
+            {
+                installer::logger::print_message("filelist $filelist_path is empty\n");
+            }
+        }
+        else # not a filelist, just pass the current file over
+        {
+            push @newfiles, $file;
+        }
+    }
+
+    return \@newfiles;
+}
+
+sub read_filelist
+{
+    my ($path) = @_;
+    my $content = installer::files::read_file($path);
+    my @filelist = ();
+
+    foreach my $line (@{$content})
+    {
+        chomp $line;
+        foreach my $file (split /\s+/, $line)
+        {
+            if ($file ne "")
+            {
+                push @filelist, $file;
+            }
+        }
+    }
+
+    return \@filelist;
+}
+
+1;
+
+# vim: set expandtab shiftwidth=4 tabstop=4:
diff --git a/solenv/gbuild/Package.mk b/solenv/gbuild/Package.mk
index c239897b3cf6..ed68d8f9e489 100644
--- a/solenv/gbuild/Package.mk
+++ b/solenv/gbuild/Package.mk
@@ -56,6 +56,12 @@ endef
 
 # Package class
 
+$(dir $(call gb_Package_get_target,%)).dir :
+	$(if $(wildcard $(dir $@)),,mkdir -p $(dir $@))
+
+$(dir $(call gb_Package_get_target,%))%/.dir :
+	$(if $(wildcard $(dir $@)),,mkdir -p $(dir $@))
+
 .PHONY : $(call gb_Package_get_clean_target,%)
 $(call gb_Package_get_clean_target,%) :
 	$(call gb_Output_announce,$*,$(false),PKG,2)
@@ -68,14 +74,17 @@ $(call gb_Package_get_preparation_target,%) :
 
 $(call gb_Package_get_target,%) :
 	$(call gb_Output_announce,$*,$(true),PKG,2)
-	mkdir -p $(dir $@) && touch $@
+	rm -f $@ && \
+	mv $(call var2file,$@.tmp,100,$(FILES)) $@
 
 # for other targets that want to create Packages, does not register at Module
 define gb_Package_Package_internal
 gb_Package_SOURCEDIR_$(1) := $(2)
 gb_Package_OUTDIR_$(1) := $(OUTDIR)
+$(call gb_Package_get_target,$(1)) : FILES :=
 $(call gb_Package_get_clean_target,$(1)) : FILES := $(call gb_Package_get_target,$(1)) $(call gb_Package_get_preparation_target,$(1))
 $(call gb_Package_get_target,$(1)) : $(call gb_Package_get_preparation_target,$(1))
+$(call gb_Package_get_target,$(1)) :| $(dir $(call gb_Package_get_target,$(1))).dir
 
 endef
 
@@ -99,6 +108,7 @@ endef
 
 define gb_Package_add_file
 $(call gb_Package_get_target,$(1)) : $$(gb_Package_OUTDIR_$(1))/$(2)
+$(call gb_Package_get_target,$(1)) : FILES += $$(gb_Package_OUTDIR_$(1))/$(2)
 $(call gb_Package_get_clean_target,$(1)) : FILES += $$(gb_Package_OUTDIR_$(1))/$(2)
 $(call gb_PackagePart_PackagePart,$(2),$$(gb_Package_SOURCEDIR_$(1))/$(3),$(call gb_Package_get_preparation_target,$(1)),$$(gb_Package_OUTDIR_$(1)))
 
diff --git a/solenv/gbuild/TargetLocations.mk b/solenv/gbuild/TargetLocations.mk
index a8cf8a431363..7f50cf6a7574 100644
--- a/solenv/gbuild/TargetLocations.mk
+++ b/solenv/gbuild/TargetLocations.mk
@@ -163,8 +163,8 @@ gb_ObjCxxObject_get_target = $(WORKDIR)/ObjCxxObject/$(1).o
 gb_ObjCObject_get_target = $(WORKDIR)/ObjCObject/$(1).o
 gb_Pagein_get_target = $(WORKDIR)/Pagein/$(1)
 gb_Package_get_preparation_target = $(WORKDIR)/Package/prepared/$(1)
-gb_Package_get_target = $(WORKDIR)/Package/$(1)
-gb_Package_get_target_for_build = $(WORKDIR_FOR_BUILD)/Package/$(1)
+gb_Package_get_target = $(WORKDIR)/Package/$(1).filelist
+gb_Package_get_target_for_build = $(WORKDIR_FOR_BUILD)/Package/$(1).filelist
 gb_Postprocess_get_target = $(WORKDIR)/Postprocess/$(1)
 gb_PrecompiledHeader_get_dep_target = $(WORKDIR)/Dep/PrecompiledHeader/$(gb_PrecompiledHeader_DEBUGDIR)/$(1).hxx.gch.d
 gb_PrecompiledHeader_get_target = $(WORKDIR)/PrecompiledHeader/$(gb_PrecompiledHeader_DEBUGDIR)/$(1).hxx.gch