Fast PCH generator and optimized PCH files

Ported update_pch.sh to Python with improved performance
and features. The new script is invoked from the same
update_pch.sh which calls it for each library in
parallel, although it can be invoked directly.

The ported script (update_pch) updates all PCH files
in ~15 seconds where the old script took ~4500 seconds.
In addition, the new script supports 3-tiered headers
(system, module, and local) and is very flexible to
support other improvement. It has a per-library
optimal configuration settings that can be updated
using another new scripts (update_pch_autotune.sh)
which finds optimal per-PCH settings.

PCH files have been generated using the new scripts
which builds significantly faster (2-3x, depending
on module and configuration) and the intermediate
binaries are noticably smaller (by several GBs).

The new script stamps each generated PCH file with
the command that generated it to make it trivial
for users to update them, and also adds the command
to invoke another script (update_pch_bisect) that
helps find missing headers or conflicting headers
that may break the build after updating the PCH.

Finally update_pch has built-in unit-tests for
makefile parsing and other core functionality.

Change-Id: Ib933b50e50374d7e2e7e3e95ba8799b0cc8a27fa
Reviewed-on: https://gerrit.libreoffice.org/19965
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Caolán McNamara <caolanm@redhat.com>
Tested-by: Caolán McNamara <caolanm@redhat.com>

2 files changed, 170 insertions, 89 deletions

diff --git a/package/inc/pch/precompiled_package2.hxx b/package/inc/pch/precompiled_package2.hxx
index 32525b766a85..95d7613ac4e7 100644
--- a/package/inc/pch/precompiled_package2.hxx
+++ b/package/inc/pch/precompiled_package2.hxx
@@ -8,92 +8,137 @@
  */
 
 /*
- This file has been autogenerated by update_pch.sh . It is possible to edit it
- manually (such as when an include file has been moved/renamed/removed. All such
+ This file has been autogenerated by update_pch.sh. It is possible to edit it
+ manually (such as when an include file has been moved/renamed/removed). All such
  manual changes will be rewritten by the next run of update_pch.sh (which presumably
  also fixes all possible problems, so it's usually better to use it).
+
+ Generated on 2015-11-14 14:16:36 using:
+ ./bin/update_pch package package2 --cutoff=3 --exclude:system --include:module --include:local
+
+ If after updating build fails, use the following command to locate conflicting headers:
+ ./bin/update_pch_bisect ./package/inc/pch/precompiled_package2.hxx "/opt/lo/bin/make package.build" --find-conflicts
 */
 
-#include "com/sun/star/io/XAsyncOutputMonitor.hpp"
 #include <algorithm>
-#include <com/sun/star/beans/NamedValue.hpp>
+#include <cassert>
+#include <config_global.h>
+#include <config_typesizes.h>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <exception>
+#include <functional>
+#include <iomanip>
+#include <new>
+#include <ostream>
+#include <sstream>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+#include <boost/optional/optional.hpp>
+#include <osl/diagnose.h>
+#include <osl/diagnose.hxx>
+#include <osl/interlck.h>
+#include <osl/mutex.h>
+#include <osl/mutex.hxx>
+#include <osl/time.h>
+#include <rtl/alloc.h>
+#include <rtl/cipher.h>
+#include <rtl/crc.h>
+#include <rtl/digest.h>
+#include <rtl/instance.hxx>
+#include <rtl/random.h>
+#include <rtl/ref.hxx>
+#include <rtl/string.h>
+#include <rtl/string.hxx>
+#include <rtl/stringutils.hxx>
+#include <rtl/textcvt.h>
+#include <rtl/textenc.h>
+#include <rtl/uri.hxx>
+#include <rtl/ustrbuf.h>
+#include <rtl/ustrbuf.hxx>
+#include <rtl/ustring.h>
+#include <rtl/ustring.hxx>
+#include <sal/config.h>
+#include <sal/detail/log.h>
+#include <sal/log.hxx>
+#include <sal/macros.h>
+#include <sal/saldllapi.h>
+#include <sal/types.h>
+#include <sal/typesizes.h>
+#include <CRC32.hxx>
+#include <EncryptedDataHeader.hxx>
+#include <EncryptionData.hxx>
+#include <PackageConstants.hxx>
+#include <ZipEntry.hxx>
+#include <ZipEnumeration.hxx>
+#include <ZipFile.hxx>
+#include <ZipOutputEntry.hxx>
+#include <ZipOutputStream.hxx>
+#include <ZipPackage.hxx>
+#include <ZipPackageBuffer.hxx>
+#include <ZipPackageFolder.hxx>
+#include <ZipPackageStream.hxx>
 #include <com/sun/star/beans/PropertyValue.hpp>
-#include <com/sun/star/container/XNameContainer.hpp>
 #include <com/sun/star/embed/StorageFormats.hpp>
-#include <com/sun/star/embed/UseBackupException.hpp>
-#include <com/sun/star/embed/XTransactedObject.hpp>
 #include <com/sun/star/io/TempFile.hpp>
-#include <com/sun/star/io/XActiveDataSink.hpp>
-#include <com/sun/star/io/XActiveDataSource.hpp>
-#include <com/sun/star/io/XActiveDataStreamer.hpp>
 #include <com/sun/star/io/XInputStream.hpp>
 #include <com/sun/star/io/XOutputStream.hpp>
 #include <com/sun/star/io/XSeekable.hpp>
 #include <com/sun/star/io/XStream.hpp>
-#include <com/sun/star/io/XTruncate.hpp>
 #include <com/sun/star/lang/DisposedException.hpp>
-#include <com/sun/star/lang/IllegalArgumentException.hpp>
-#include <com/sun/star/lang/WrappedTargetRuntimeException.hpp>
+#include <com/sun/star/lang/EventObject.hpp>
 #include <com/sun/star/lang/XMultiServiceFactory.hpp>
-#include <com/sun/star/lang/XSingleServiceFactory.hpp>
-#include <com/sun/star/packages/manifest/ManifestReader.hpp>
-#include <com/sun/star/packages/manifest/ManifestWriter.hpp>
+#include <com/sun/star/lang/XServiceInfo.hpp>
+#include <com/sun/star/lang/XTypeProvider.hpp>
 #include <com/sun/star/packages/zip/ZipConstants.hpp>
-#include <com/sun/star/packages/zip/ZipIOException.hpp>
-#include <com/sun/star/registry/XRegistryKey.hpp>
-#include <com/sun/star/ucb/IOErrorCode.hpp>
-#include <com/sun/star/ucb/NameClash.hpp>
-#include <com/sun/star/ucb/OpenCommandArgument2.hpp>
-#include <com/sun/star/ucb/OpenMode.hpp>
-#include <com/sun/star/ucb/SimpleFileAccess.hpp>
-#include <com/sun/star/ucb/TransferInfo.hpp>
-#include <com/sun/star/ucb/UniversalContentBroker.hpp>
-#include <com/sun/star/ucb/XCommandEnvironment.hpp>
-#include <com/sun/star/ucb/XProgressHandler.hpp>
+#include <com/sun/star/uno/Any.h>
+#include <com/sun/star/uno/Any.hxx>
+#include <com/sun/star/uno/Reference.h>
+#include <com/sun/star/uno/Reference.hxx>
 #include <com/sun/star/uno/RuntimeException.hpp>
+#include <com/sun/star/uno/Sequence.h>
+#include <com/sun/star/uno/Sequence.hxx>
+#include <com/sun/star/uno/Type.h>
+#include <com/sun/star/uno/Type.hxx>
+#include <com/sun/star/uno/TypeClass.hdl>
+#include <com/sun/star/uno/XAggregation.hpp>
+#include <com/sun/star/uno/XComponentContext.hpp>
+#include <com/sun/star/uno/XInterface.hpp>
+#include <com/sun/star/uno/XWeak.hpp>
+#include <com/sun/star/uno/genfunc.h>
+#include <com/sun/star/uno/genfunc.hxx>
 #include <com/sun/star/xml/crypto/CipherID.hpp>
 #include <com/sun/star/xml/crypto/DigestID.hpp>
-#include <com/sun/star/xml/crypto/NSSInitializer.hpp>
-#include <com/sun/star/xml/crypto/XCipherContext.hpp>
-#include <com/sun/star/xml/crypto/XCipherContextSupplier.hpp>
-#include <com/sun/star/xml/crypto/XDigestContext.hpp>
-#include <com/sun/star/xml/crypto/XDigestContextSupplier.hpp>
-#include <com/sun/star/xml/sax/Parser.hpp>
-#include <com/sun/star/xml/sax/SAXException.hpp>
-#include <com/sun/star/xml/sax/SAXParseException.hpp>
-#include <com/sun/star/xml/sax/Writer.hpp>
-#include <com/sun/star/xml/sax/XAttributeList.hpp>
-#include <com/sun/star/xml/sax/XDocumentHandler.hpp>
-#include <com/sun/star/xml/sax/XExtendedDocumentHandler.hpp>
-#include <comphelper/attributelist.hxx>
-#include <comphelper/documentconstants.hxx>
-#include <comphelper/ofopxmlhelper.hxx>
+#include <comphelper/comphelperdllapi.h>
 #include <comphelper/processfactory.hxx>
-#include <comphelper/seekableinput.hxx>
-#include <comphelper/sequenceashashmap.hxx>
 #include <comphelper/storagehelper.hxx>
+#include <cppu/cppudllapi.h>
+#include <cppu/unotype.hxx>
+#include <cppuhelper/cppuhelperdllapi.h>
 #include <cppuhelper/exc_hlp.hxx>
 #include <cppuhelper/factory.hxx>
 #include <cppuhelper/implbase.hxx>
+#include <cppuhelper/implbase_ex.hxx>
+#include <cppuhelper/implbase_ex_post.hxx>
+#include <cppuhelper/implbase_ex_pre.hxx>
 #include <cppuhelper/supportsservice.hxx>
 #include <cppuhelper/typeprovider.hxx>
-#include <cstring>
-#include <osl/diagnose.h>
-#include <osl/diagnose.hxx>
-#include <osl/mutex.hxx>
-#include <osl/time.h>
-#include <rtl/cipher.h>
-#include <rtl/crc.h>
-#include <rtl/digest.h>
-#include <rtl/instance.hxx>
-#include <rtl/random.h>
-#include <rtl/ref.hxx>
-#include <rtl/uri.hxx>
-#include <rtl/ustrbuf.hxx>
-#include <sax/tools/converter.hxx>
-#include <string.h>
-#include <ucbhelper/content.hxx>
-#include <vector>
-#include <zlib.h>
+#include <cppuhelper/weak.hxx>
+#include <cppuhelper/weakagg.hxx>
+#include <cppuhelper/weakref.hxx>
+#include <typelib/typeclass.h>
+#include <typelib/typedescription.h>
+#include <typelib/uik.h>
+#include <uno/any2.h>
+#include <uno/data.h>
+#include <uno/sequence2.h>
+#include <package/packagedllapi.hxx>
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/package/inc/pch/precompiled_xstor.hxx b/package/inc/pch/precompiled_xstor.hxx
index 28988a6cac57..ec4bc6359ce6 100644
--- a/package/inc/pch/precompiled_xstor.hxx
+++ b/package/inc/pch/precompiled_xstor.hxx
@@ -8,55 +8,91 @@
  */
 
 /*
- This file has been autogenerated by update_pch.sh . It is possible to edit it
- manually (such as when an include file has been moved/renamed/removed. All such
+ This file has been autogenerated by update_pch.sh. It is possible to edit it
+ manually (such as when an include file has been moved/renamed/removed). All such
  manual changes will be rewritten by the next run of update_pch.sh (which presumably
  also fixes all possible problems, so it's usually better to use it).
+
+ Generated on 2015-11-14 14:16:40 using:
+ ./bin/update_pch package xstor --cutoff=2 --exclude:system --include:module --exclude:local
+
+ If after updating build fails, use the following command to locate conflicting headers:
+ ./bin/update_pch_bisect ./package/inc/pch/precompiled_xstor.hxx "/opt/lo/bin/make package.build" --find-conflicts
 */
 
+#include <cassert>
+#include <config_typesizes.h>
+#include <cstddef>
+#include <cstdlib>
+#include <cstring>
+#include <iomanip>
+#include <new>
+#include <ostream>
+#include <sstream>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <string>
 #include <boost/checked_delete.hpp>
-#include <com/sun/star/beans/NamedValue.hpp>
+#include <osl/diagnose.h>
+#include <osl/interlck.h>
+#include <osl/time.h>
+#include <rtl/alloc.h>
+#include <rtl/digest.h>
+#include <rtl/instance.hxx>
+#include <rtl/string.h>
+#include <rtl/string.hxx>
+#include <rtl/stringutils.hxx>
+#include <rtl/textcvt.h>
+#include <rtl/textenc.h>
+#include <rtl/unload.h>
+#include <rtl/ustring.h>
+#include <rtl/ustring.hxx>
+#include <sal/config.h>
+#include <sal/detail/log.h>
+#include <sal/log.hxx>
+#include <sal/macros.h>
+#include <sal/saldllapi.h>
+#include <sal/types.h>
+#include <sal/typesizes.h>
 #include <com/sun/star/beans/PropertyValue.hpp>
-#include <com/sun/star/container/XEnumerationAccess.hpp>
-#include <com/sun/star/container/XHierarchicalNameAccess.hpp>
-#include <com/sun/star/container/XNamed.hpp>
 #include <com/sun/star/embed/ElementModes.hpp>
 #include <com/sun/star/embed/StorageFormats.hpp>
-#include <com/sun/star/embed/UseBackupException.hpp>
-#include <com/sun/star/embed/XHierarchicalStorageAccess2.hpp>
-#include <com/sun/star/embed/XTransactedObject.hpp>
-#include <com/sun/star/embed/XTransactionBroadcaster.hpp>
-#include <com/sun/star/io/IOException.hpp>
 #include <com/sun/star/io/TempFile.hpp>
-#include <com/sun/star/io/XInputStream.hpp>
-#include <com/sun/star/io/XSeekable.hpp>
 #include <com/sun/star/lang/DisposedException.hpp>
 #include <com/sun/star/lang/WrappedTargetRuntimeException.hpp>
-#include <com/sun/star/lang/XComponent.hpp>
 #include <com/sun/star/lang/XMultiServiceFactory.hpp>
-#include <com/sun/star/lang/XTypeProvider.hpp>
+#include <com/sun/star/lang/XSingleComponentFactory.hpp>
+#include <com/sun/star/lang/XSingleServiceFactory.hpp>
 #include <com/sun/star/lang/XUnoTunnel.hpp>
 #include <com/sun/star/logging/DocumentIOLogRing.hpp>
-#include <com/sun/star/registry/InvalidRegistryException.hpp>
 #include <com/sun/star/registry/XRegistryKey.hpp>
 #include <com/sun/star/ucb/SimpleFileAccess.hpp>
-#include <com/sun/star/ucb/XCommandEnvironment.hpp>
-#include <com/sun/star/ucb/XProgressHandler.hpp>
-#include <com/sun/star/uno/Reference.hxx>
+#include <com/sun/star/uno/Any.h>
+#include <com/sun/star/uno/Any.hxx>
+#include <com/sun/star/uno/Reference.h>
+#include <com/sun/star/uno/RuntimeException.hpp>
+#include <com/sun/star/uno/Type.h>
+#include <com/sun/star/uno/Type.hxx>
+#include <com/sun/star/uno/TypeClass.hdl>
 #include <com/sun/star/uno/XComponentContext.hpp>
-#include <com/sun/star/util/XChangesBatch.hpp>
-#include <com/sun/star/util/XCloneable.hpp>
+#include <com/sun/star/uno/XInterface.hpp>
+#include <com/sun/star/uno/genfunc.h>
+#include <com/sun/star/uno/genfunc.hxx>
 #include <comphelper/ofopxmlhelper.hxx>
 #include <comphelper/processfactory.hxx>
 #include <comphelper/storagehelper.hxx>
+#include <cppu/cppudllapi.h>
+#include <cppu/unotype.hxx>
+#include <cppuhelper/cppuhelperdllapi.h>
 #include <cppuhelper/exc_hlp.hxx>
-#include <cppuhelper/factory.hxx>
 #include <cppuhelper/queryinterface.hxx>
-#include <cppuhelper/supportsservice.hxx>
 #include <cppuhelper/typeprovider.hxx>
-#include <osl/diagnose.h>
-#include <rtl/digest.h>
-#include <rtl/instance.hxx>
-#include <sal/config.h>
+#include <typelib/typeclass.h>
+#include <typelib/typedescription.h>
+#include <typelib/uik.h>
+#include <uno/any2.h>
+#include <uno/data.h>
+#include <uno/sequence2.h>
 
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */