summaryrefslogtreecommitdiff
path: root/thirdparty/snappy
diff options
context:
space:
mode:
authorJosé Fonseca <jfonseca@vmware.com>2014-10-01 22:36:20 +0100
committerJosé Fonseca <jfonseca@vmware.com>2014-10-02 12:38:51 +0100
commite455db6d32a4d1f08942f77df548120323f584f4 (patch)
tree6b0950d1fcb30efbbabc2d9257ba213d3ba3cfb2 /thirdparty/snappy
parent4f9042d3a705307849a6f63961eaaaa2e1d85d77 (diff)
snappy: Update to version 1.1.2.
Diffstat (limited to 'thirdparty/snappy')
-rw-r--r--thirdparty/snappy/COPYING26
-rw-r--r--thirdparty/snappy/ChangeLog699
-rw-r--r--thirdparty/snappy/NEWS39
-rw-r--r--thirdparty/snappy/config.h68
-rw-r--r--thirdparty/snappy/framing_format.txt35
-rw-r--r--thirdparty/snappy/snappy-c.h2
-rw-r--r--thirdparty/snappy/snappy-internal.h4
-rw-r--r--thirdparty/snappy/snappy-stubs-internal.h84
-rw-r--r--thirdparty/snappy/snappy-stubs-public.h17
-rw-r--r--thirdparty/snappy/snappy-stubs-public.h.in13
-rw-r--r--thirdparty/snappy/snappy-test.cc50
-rw-r--r--thirdparty/snappy/snappy-test.h99
-rw-r--r--thirdparty/snappy/snappy.cc347
-rw-r--r--thirdparty/snappy/snappy.h47
-rw-r--r--thirdparty/snappy/snappy_unittest.cc270
15 files changed, 1522 insertions, 278 deletions
diff --git a/thirdparty/snappy/COPYING b/thirdparty/snappy/COPYING
index 8d6bd9fe..4816c430 100644
--- a/thirdparty/snappy/COPYING
+++ b/thirdparty/snappy/COPYING
@@ -26,3 +26,29 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+===
+
+Some of the benchmark data in util/zippy/testdata is licensed differently:
+
+ - fireworks.jpeg is Copyright 2013 Steinar H. Gunderson, and
+ is licensed under the Creative Commons Attribution 3.0 license
+ (CC-BY-3.0). See https://creativecommons.org/licenses/by/3.0/
+ for more information.
+
+ - kppkn.gtb is taken from the Gaviota chess tablebase set, and
+ is licensed under the MIT License. See
+ https://sites.google.com/site/gaviotachessengine/Home/endgame-tablebases-1
+ for more information.
+
+ - paper-100k.pdf is an excerpt (bytes 92160 to 194560) from the paper
+ “Combinatorial Modeling of Chromatin Features Quantitatively Predicts DNA
+ Replication Timing in _Drosophila_” by Federico Comoglio and Renato Paro,
+ which is licensed under the CC-BY license. See
+ http://www.ploscompbiol.org/static/license for more ifnormation.
+
+ - alice29.txt, asyoulik.txt, plrabn12.txt and lcet10.txt are from Project
+ Gutenberg. The first three have expired copyrights and are in the public
+ domain; the latter does not have expired copyright, but is still in the
+ public domain according to the license information
+ (http://www.gutenberg.org/ebooks/53).
diff --git a/thirdparty/snappy/ChangeLog b/thirdparty/snappy/ChangeLog
index f79491b5..edd46dd7 100644
--- a/thirdparty/snappy/ChangeLog
+++ b/thirdparty/snappy/ChangeLog
@@ -1,4 +1,703 @@
------------------------------------------------------------------------
+r83 | snappy.mirrorbot@gmail.com | 2014-02-19 11:31:49 +0100 (Wed, 19 Feb 2014) | 9 lines
+
+Fix public issue 82: Stop distributing benchmark data files that have
+unclear or unsuitable licensing.
+
+In general, we replace the files we can with liberally licensed data,
+and remove all the others (in particular all the parts of the Canterbury
+corpus that are not clearly in the public domain). The replacements
+do not always have the exact same characteristics as the original ones,
+but they are more than good enough to be useful for benchmarking.
+
+------------------------------------------------------------------------
+r82 | snappy.mirrorbot@gmail.com | 2013-10-25 15:31:27 +0200 (Fri, 25 Oct 2013) | 8 lines
+
+Add support for padding in the Snappy framed format.
+
+This is specifically motivated by DICOM's demands that embedded data
+must be of an even number of bytes, but could in principle be used for
+any sort of padding/alignment needed.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r81 | snappy.mirrorbot@gmail.com | 2013-10-15 17:21:31 +0200 (Tue, 15 Oct 2013) | 4 lines
+
+Release Snappy 1.1.1.
+
+R=jeff
+
+------------------------------------------------------------------------
+r80 | snappy.mirrorbot@gmail.com | 2013-08-13 14:55:00 +0200 (Tue, 13 Aug 2013) | 6 lines
+
+Add autoconf tests for size_t and ssize_t. Sort-of resolves public issue 79;
+it would solve the problem if MSVC typically used autoconf. However, it gives
+a natural place (config.h) to put the typedef even for MSVC.
+
+R=jsbell
+
+------------------------------------------------------------------------
+r79 | snappy.mirrorbot@gmail.com | 2013-07-29 13:06:44 +0200 (Mon, 29 Jul 2013) | 14 lines
+
+When we compare the number of bytes produced with the offset for a
+backreference, make the signedness of the bytes produced clear,
+by sticking it into a size_t. This avoids a signed/unsigned compare
+warning from MSVC (public issue 71), and also is slightly clearer.
+
+Since the line is now so long the explanatory comment about the -1u
+trick has to go somewhere else anyway, I used the opportunity to
+explain it in slightly more detail.
+
+This is a purely stylistic change; the emitted assembler from GCC
+is identical.
+
+R=jeff
+
+------------------------------------------------------------------------
+r78 | snappy.mirrorbot@gmail.com | 2013-06-30 21:24:03 +0200 (Sun, 30 Jun 2013) | 111 lines
+
+In the fast path for decompressing literals, instead of checking
+whether there's 16 bytes free and then checking right afterwards
+(when having subtracted the literal size) that there are now
+5 bytes free, just check once for 21 bytes. This skips a compare
+and a branch; although it is easily predictable, it is still
+a few cycles on a fast path that we would like to get rid of.
+
+Benchmarking this yields very confusing results. On open-source
+GCC 4.8.1 on Haswell, we get exactly the expected results; the
+benchmarks where we hit the fast path for literals (in particular
+the two HTML benchmarks and the protobuf benchmark) give very nice
+speedups, and the others are not really affected.
+
+However, benchmarks with Google's GCC branch on other hardware
+is much less clear. It seems that we have a weak loss in some cases
+(and the win for the “typical” win cases are not nearly as clear),
+but that it depends on microarchitecture and plain luck in how we run
+the benchmark. Looking at the generated assembler, it seems that
+the removal of the if causes other large-scale changes in how the
+function is laid out, which makes it likely that this is just bad luck.
+
+Thus, we should keep this change, even though its exact current impact is
+unclear; it's a sensible change per se, and dropping it on the basis of
+microoptimization for a given compiler (or even branch of a compiler)
+would seem like a bad strategy in the long run.
+
+Microbenchmark results (all in 64-bit, opt mode):
+
+ Nehalem, Google GCC:
+
+ Benchmark Base (ns) New (ns) Improvement
+ ------------------------------------------------------------------------------
+ BM_UFlat/0 76747 75591 1.3GB/s html +1.5%
+ BM_UFlat/1 765756 757040 886.3MB/s urls +1.2%
+ BM_UFlat/2 10867 10893 10.9GB/s jpg -0.2%
+ BM_UFlat/3 124 131 1.4GB/s jpg_200 -5.3%
+ BM_UFlat/4 31663 31596 2.8GB/s pdf +0.2%
+ BM_UFlat/5 314162 308176 1.2GB/s html4 +1.9%
+ BM_UFlat/6 29668 29746 790.6MB/s cp -0.3%
+ BM_UFlat/7 12958 13386 796.4MB/s c -3.2%
+ BM_UFlat/8 3596 3682 966.0MB/s lsp -2.3%
+ BM_UFlat/9 1019193 1033493 953.3MB/s xls -1.4%
+ BM_UFlat/10 239 247 775.3MB/s xls_200 -3.2%
+ BM_UFlat/11 236411 240271 606.9MB/s txt1 -1.6%
+ BM_UFlat/12 206639 209768 571.2MB/s txt2 -1.5%
+ BM_UFlat/13 627803 635722 641.4MB/s txt3 -1.2%
+ BM_UFlat/14 845932 857816 538.2MB/s txt4 -1.4%
+ BM_UFlat/15 402107 391670 1.2GB/s bin +2.7%
+ BM_UFlat/16 283 279 683.6MB/s bin_200 +1.4%
+ BM_UFlat/17 46070 46815 781.5MB/s sum -1.6%
+ BM_UFlat/18 5053 5163 782.0MB/s man -2.1%
+ BM_UFlat/19 79721 76581 1.4GB/s pb +4.1%
+ BM_UFlat/20 251158 252330 697.5MB/s gaviota -0.5%
+ Sum of all benchmarks 4966150 4980396 -0.3%
+
+
+ Sandy Bridge, Google GCC:
+
+ Benchmark Base (ns) New (ns) Improvement
+ ------------------------------------------------------------------------------
+ BM_UFlat/0 42850 42182 2.3GB/s html +1.6%
+ BM_UFlat/1 525660 515816 1.3GB/s urls +1.9%
+ BM_UFlat/2 7173 7283 16.3GB/s jpg -1.5%
+ BM_UFlat/3 92 91 2.1GB/s jpg_200 +1.1%
+ BM_UFlat/4 15147 14872 5.9GB/s pdf +1.8%
+ BM_UFlat/5 199936 192116 2.0GB/s html4 +4.1%
+ BM_UFlat/6 12796 12443 1.8GB/s cp +2.8%
+ BM_UFlat/7 6588 6400 1.6GB/s c +2.9%
+ BM_UFlat/8 2010 1951 1.8GB/s lsp +3.0%
+ BM_UFlat/9 761124 763049 1.3GB/s xls -0.3%
+ BM_UFlat/10 186 189 1016.1MB/s xls_200 -1.6%
+ BM_UFlat/11 159354 158460 918.6MB/s txt1 +0.6%
+ BM_UFlat/12 139732 139950 856.1MB/s txt2 -0.2%
+ BM_UFlat/13 429917 425027 961.7MB/s txt3 +1.2%
+ BM_UFlat/14 585255 587324 785.8MB/s txt4 -0.4%
+ BM_UFlat/15 276186 266173 1.8GB/s bin +3.8%
+ BM_UFlat/16 205 207 925.5MB/s bin_200 -1.0%
+ BM_UFlat/17 24925 24935 1.4GB/s sum -0.0%
+ BM_UFlat/18 2632 2576 1.5GB/s man +2.2%
+ BM_UFlat/19 40546 39108 2.8GB/s pb +3.7%
+ BM_UFlat/20 175803 168209 1048.9MB/s gaviota +4.5%
+ Sum of all benchmarks 3408117 3368361 +1.2%
+
+
+ Haswell, upstream GCC 4.8.1:
+
+ Benchmark Base (ns) New (ns) Improvement
+ ------------------------------------------------------------------------------
+ BM_UFlat/0 46308 40641 2.3GB/s html +13.9%
+ BM_UFlat/1 513385 514706 1.3GB/s urls -0.3%
+ BM_UFlat/2 6197 6151 19.2GB/s jpg +0.7%
+ BM_UFlat/3 61 61 3.0GB/s jpg_200 +0.0%
+ BM_UFlat/4 13551 13429 6.5GB/s pdf +0.9%
+ BM_UFlat/5 198317 190243 2.0GB/s html4 +4.2%
+ BM_UFlat/6 14768 12560 1.8GB/s cp +17.6%
+ BM_UFlat/7 6453 6447 1.6GB/s c +0.1%
+ BM_UFlat/8 1991 1980 1.8GB/s lsp +0.6%
+ BM_UFlat/9 766947 770424 1.2GB/s xls -0.5%
+ BM_UFlat/10 170 169 1.1GB/s xls_200 +0.6%
+ BM_UFlat/11 164350 163554 888.7MB/s txt1 +0.5%
+ BM_UFlat/12 145444 143830 832.1MB/s txt2 +1.1%
+ BM_UFlat/13 437849 438413 929.2MB/s txt3 -0.1%
+ BM_UFlat/14 603587 605309 759.8MB/s txt4 -0.3%
+ BM_UFlat/15 249799 248067 1.9GB/s bin +0.7%
+ BM_UFlat/16 191 188 1011.4MB/s bin_200 +1.6%
+ BM_UFlat/17 26064 24778 1.4GB/s sum +5.2%
+ BM_UFlat/18 2620 2601 1.5GB/s man +0.7%
+ BM_UFlat/19 44551 37373 3.0GB/s pb +19.2%
+ BM_UFlat/20 165408 164584 1.0GB/s gaviota +0.5%
+ Sum of all benchmarks 3408011 3385508 +0.7%
+
+------------------------------------------------------------------------
+r77 | snappy.mirrorbot@gmail.com | 2013-06-14 23:42:26 +0200 (Fri, 14 Jun 2013) | 92 lines
+
+Make the two IncrementalCopy* functions take in an ssize_t instead of a len,
+in order to avoid having to do 32-to-64-bit signed conversions on a hot path
+during decompression. (Also fixes some MSVC warnings, mentioned in public
+issue 75, but more of those remain.) They cannot be size_t because we expect
+them to go negative and test for that.
+
+This saves a few movzwl instructions, yielding ~2% speedup in decompression.
+
+
+Sandy Bridge:
+
+Benchmark Base (ns) New (ns) Improvement
+-------------------------------------------------------------------------------------------------
+BM_UFlat/0 48009 41283 2.3GB/s html +16.3%
+BM_UFlat/1 531274 513419 1.3GB/s urls +3.5%
+BM_UFlat/2 7378 7062 16.8GB/s jpg +4.5%
+BM_UFlat/3 92 92 2.0GB/s jpg_200 +0.0%
+BM_UFlat/4 15057 14974 5.9GB/s pdf +0.6%
+BM_UFlat/5 204323 193140 2.0GB/s html4 +5.8%
+BM_UFlat/6 13282 12611 1.8GB/s cp +5.3%
+BM_UFlat/7 6511 6504 1.6GB/s c +0.1%
+BM_UFlat/8 2014 2030 1.7GB/s lsp -0.8%
+BM_UFlat/9 775909 768336 1.3GB/s xls +1.0%
+BM_UFlat/10 182 184 1043.2MB/s xls_200 -1.1%
+BM_UFlat/11 167352 161630 901.2MB/s txt1 +3.5%
+BM_UFlat/12 147393 142246 842.8MB/s txt2 +3.6%
+BM_UFlat/13 449960 432853 944.4MB/s txt3 +4.0%
+BM_UFlat/14 620497 594845 775.9MB/s txt4 +4.3%
+BM_UFlat/15 265610 267356 1.8GB/s bin -0.7%
+BM_UFlat/16 206 205 932.7MB/s bin_200 +0.5%
+BM_UFlat/17 25561 24730 1.4GB/s sum +3.4%
+BM_UFlat/18 2620 2644 1.5GB/s man -0.9%
+BM_UFlat/19 45766 38589 2.9GB/s pb +18.6%
+BM_UFlat/20 171107 169832 1039.5MB/s gaviota +0.8%
+Sum of all benchmarks 3500103 3394565 +3.1%
+
+
+Westmere:
+
+Benchmark Base (ns) New (ns) Improvement
+-------------------------------------------------------------------------------------------------
+BM_UFlat/0 72624 71526 1.3GB/s html +1.5%
+BM_UFlat/1 735821 722917 930.8MB/s urls +1.8%
+BM_UFlat/2 10450 10172 11.7GB/s jpg +2.7%
+BM_UFlat/3 117 117 1.6GB/s jpg_200 +0.0%
+BM_UFlat/4 29817 29648 3.0GB/s pdf +0.6%
+BM_UFlat/5 297126 293073 1.3GB/s html4 +1.4%
+BM_UFlat/6 28252 27994 842.0MB/s cp +0.9%
+BM_UFlat/7 12672 12391 862.1MB/s c +2.3%
+BM_UFlat/8 3507 3425 1040.9MB/s lsp +2.4%
+BM_UFlat/9 1004268 969395 1018.0MB/s xls +3.6%
+BM_UFlat/10 233 227 844.8MB/s xls_200 +2.6%
+BM_UFlat/11 230054 224981 647.8MB/s txt1 +2.3%
+BM_UFlat/12 201229 196447 610.5MB/s txt2 +2.4%
+BM_UFlat/13 609547 596761 685.3MB/s txt3 +2.1%
+BM_UFlat/14 824362 804821 573.8MB/s txt4 +2.4%
+BM_UFlat/15 371095 374899 1.3GB/s bin -1.0%
+BM_UFlat/16 267 267 717.8MB/s bin_200 +0.0%
+BM_UFlat/17 44623 43828 835.9MB/s sum +1.8%
+BM_UFlat/18 5077 4815 841.0MB/s man +5.4%
+BM_UFlat/19 74964 73210 1.5GB/s pb +2.4%
+BM_UFlat/20 237987 236745 746.0MB/s gaviota +0.5%
+Sum of all benchmarks 4794092 4697659 +2.1%
+
+
+Istanbul:
+
+Benchmark Base (ns) New (ns) Improvement
+-------------------------------------------------------------------------------------------------
+BM_UFlat/0 98614 96376 1020.4MB/s html +2.3%
+BM_UFlat/1 963740 953241 707.2MB/s urls +1.1%
+BM_UFlat/2 25042 24769 4.8GB/s jpg +1.1%
+BM_UFlat/3 180 180 1065.6MB/s jpg_200 +0.0%
+BM_UFlat/4 45942 45403 1.9GB/s pdf +1.2%
+BM_UFlat/5 400135 390226 1008.2MB/s html4 +2.5%
+BM_UFlat/6 37768 37392 631.9MB/s cp +1.0%
+BM_UFlat/7 18585 18200 588.2MB/s c +2.1%
+BM_UFlat/8 5751 5690 627.7MB/s lsp +1.1%
+BM_UFlat/9 1543154 1542209 641.4MB/s xls +0.1%
+BM_UFlat/10 381 388 494.6MB/s xls_200 -1.8%
+BM_UFlat/11 339715 331973 440.1MB/s txt1 +2.3%
+BM_UFlat/12 294807 289418 415.4MB/s txt2 +1.9%
+BM_UFlat/13 906160 884094 463.3MB/s txt3 +2.5%
+BM_UFlat/14 1224221 1198435 386.1MB/s txt4 +2.2%
+BM_UFlat/15 516277 502923 979.5MB/s bin +2.7%
+BM_UFlat/16 405 402 477.2MB/s bin_200 +0.7%
+BM_UFlat/17 61640 60621 605.6MB/s sum +1.7%
+BM_UFlat/18 7326 7383 549.5MB/s man -0.8%
+BM_UFlat/19 94720 92653 1.2GB/s pb +2.2%
+BM_UFlat/20 360435 346687 510.6MB/s gaviota +4.0%
+Sum of all benchmarks 6944998 6828663 +1.7%
+
+------------------------------------------------------------------------
+r76 | snappy.mirrorbot@gmail.com | 2013-06-13 18:19:52 +0200 (Thu, 13 Jun 2013) | 9 lines
+
+Add support for uncompressing to iovecs (scatter I/O).
+Windows does not have struct iovec defined anywhere,
+so we define our own version that's equal to what UNIX
+typically has.
+
+The bulk of this patch was contributed by Mohit Aron.
+
+R=jeff
+
+------------------------------------------------------------------------
+r75 | snappy.mirrorbot@gmail.com | 2013-06-12 21:51:15 +0200 (Wed, 12 Jun 2013) | 4 lines
+
+Some code reorganization needed for an internal change.
+
+R=fikes
+
+------------------------------------------------------------------------
+r74 | snappy.mirrorbot@gmail.com | 2013-04-09 17:33:30 +0200 (Tue, 09 Apr 2013) | 4 lines
+
+Supports truncated test data in zippy benchmark.
+
+R=sesse
+
+------------------------------------------------------------------------
+r73 | snappy.mirrorbot@gmail.com | 2013-02-05 15:36:15 +0100 (Tue, 05 Feb 2013) | 4 lines
+
+Release Snappy 1.1.0.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r72 | snappy.mirrorbot@gmail.com | 2013-02-05 15:30:05 +0100 (Tue, 05 Feb 2013) | 9 lines
+
+Make ./snappy_unittest pass without "srcdir" being defined.
+
+Previously, snappy_unittests would read from an absolute path /testdata/..;
+convert it to use a relative path instead.
+
+Patch from Marc-Antonie Ruel.
+
+R=maruel
+
+------------------------------------------------------------------------
+r71 | snappy.mirrorbot@gmail.com | 2013-01-18 13:16:36 +0100 (Fri, 18 Jan 2013) | 287 lines
+
+Increase the Zippy block size from 32 kB to 64 kB, winning ~3% density
+while being effectively performance neutral.
+
+The longer story about density is that we win 3-6% density on the benchmarks
+where this has any effect at all; many of the benchmarks (cp, c, lsp, man)
+are smaller than 32 kB and thus will have no effect. Binary data also seems
+to win little or nothing; of course, the already-compressed data wins nothing.
+The protobuf benchmark wins as much as ~18% depending on architecture,
+but I wouldn't be too sure that this is representative of protobuf data in
+general.
+
+As of performance, we lose a tiny amount since we get more tags (e.g., a long
+literal might be broken up into literal-copy-literal), but we win it back with
+less clearing of the hash table, and more opportunities to skip incompressible
+data (e.g. in the jpg benchmark). Decompression seems to get ever so slightly
+slower, again due to more tags. The total net change is about as close to zero
+as we can get, so the end effect seems to be simply more density and no
+real performance change.
+
+The comment about not changing kBlockSize, scary as it is, is not really
+relevant, since we're never going to have a block-level decompressor without
+explicitly marked blocks. Replace it with something more appropriate.
+
+This affects the framing format, but it's okay to change it since it basically
+has no users yet.
+
+
+Density (note that cp, c, lsp and man are all smaller than 32 kB):
+
+ Benchmark Description Base (%) New (%) Improvement
+ --------------------------------------------------------------
+ ZFlat/0 html 22.57 22.31 +5.6%
+ ZFlat/1 urls 50.89 47.77 +6.5%
+ ZFlat/2 jpg 99.88 99.87 +0.0%
+ ZFlat/3 pdf 82.13 82.07 +0.1%
+ ZFlat/4 html4 23.55 22.51 +4.6%
+ ZFlat/5 cp 48.12 48.12 +0.0%
+ ZFlat/6 c 42.40 42.40 +0.0%
+ ZFlat/7 lsp 48.37 48.37 +0.0%
+ ZFlat/8 xls 41.34 41.23 +0.3%
+ ZFlat/9 txt1 59.81 57.87 +3.4%
+ ZFlat/10 txt2 64.07 61.93 +3.5%
+ ZFlat/11 txt3 57.11 54.92 +4.0%
+ ZFlat/12 txt4 68.35 66.22 +3.2%
+ ZFlat/13 bin 18.21 18.11 +0.6%
+ ZFlat/14 sum 51.88 48.96 +6.0%
+ ZFlat/15 man 59.36 59.36 +0.0%
+ ZFlat/16 pb 23.15 19.64 +17.9%
+ ZFlat/17 gaviota 38.27 37.72 +1.5%
+ Geometric mean 45.51 44.15 +3.1%
+
+
+Microbenchmarks (64-bit, opt):
+
+Westmere 2.8 GHz:
+
+ Benchmark Base (ns) New (ns) Improvement
+ -------------------------------------------------------------------------------------------------
+ BM_UFlat/0 75342 75027 1.3GB/s html +0.4%
+ BM_UFlat/1 723767 744269 899.6MB/s urls -2.8%
+ BM_UFlat/2 10072 10072 11.7GB/s jpg +0.0%
+ BM_UFlat/3 30747 30388 2.9GB/s pdf +1.2%
+ BM_UFlat/4 307353 306063 1.2GB/s html4 +0.4%
+ BM_UFlat/5 28593 28743 816.3MB/s cp -0.5%
+ BM_UFlat/6 12958 12998 818.1MB/s c -0.3%
+ BM_UFlat/7 3700 3792 935.8MB/s lsp -2.4%
+ BM_UFlat/8 999685 999905 982.1MB/s xls -0.0%
+ BM_UFlat/9 232954 230079 630.4MB/s txt1 +1.2%
+ BM_UFlat/10 200785 201468 592.6MB/s txt2 -0.3%
+ BM_UFlat/11 617267 610968 666.1MB/s txt3 +1.0%
+ BM_UFlat/12 821595 822475 558.7MB/s txt4 -0.1%
+ BM_UFlat/13 377097 377632 1.3GB/s bin -0.1%
+ BM_UFlat/14 45476 45260 805.8MB/s sum +0.5%
+ BM_UFlat/15 4985 5003 805.7MB/s man -0.4%
+ BM_UFlat/16 80813 77494 1.4GB/s pb +4.3%
+ BM_UFlat/17 251792 241553 727.7MB/s gaviota +4.2%
+ BM_UValidate/0 40343 40354 2.4GB/s html -0.0%
+ BM_UValidate/1 426890 451574 1.4GB/s urls -5.5%
+ BM_UValidate/2 187 179 661.9GB/s jpg +4.5%
+ BM_UValidate/3 13783 13827 6.4GB/s pdf -0.3%
+ BM_UValidate/4 162393 163335 2.3GB/s html4 -0.6%
+ BM_UDataBuffer/0 93756 93302 1046.7MB/s html +0.5%
+ BM_UDataBuffer/1 886714 916292 730.7MB/s urls -3.2%
+ BM_UDataBuffer/2 15861 16401 7.2GB/s jpg -3.3%
+ BM_UDataBuffer/3 38934 39224 2.2GB/s pdf -0.7%
+ BM_UDataBuffer/4 381008 379428 1029.5MB/s html4 +0.4%
+ BM_UCord/0 92528 91098 1072.0MB/s html +1.6%
+ BM_UCord/1 858421 885287 756.3MB/s urls -3.0%
+ BM_UCord/2 13140 13464 8.8GB/s jpg -2.4%
+ BM_UCord/3 39012 37773 2.3GB/s pdf +3.3%
+ BM_UCord/4 376869 371267 1052.1MB/s html4 +1.5%
+ BM_UCordString/0 75810 75303 1.3GB/s html +0.7%
+ BM_UCordString/1 735290 753841 888.2MB/s urls -2.5%
+ BM_UCordString/2 11945 13113 9.0GB/s jpg -8.9%
+ BM_UCordString/3 33901 32562 2.7GB/s pdf +4.1%
+ BM_UCordString/4 310985 309390 1.2GB/s html4 +0.5%
+ BM_UCordValidate/0 40952 40450 2.4GB/s html +1.2%
+ BM_UCordValidate/1 433842 456531 1.4GB/s urls -5.0%
+ BM_UCordValidate/2 1179 1173 100.8GB/s jpg +0.5%
+ BM_UCordValidate/3 14481 14392 6.1GB/s pdf +0.6%
+ BM_UCordValidate/4 164364 164151 2.3GB/s html4 +0.1%
+ BM_ZFlat/0 160610 156601 623.6MB/s html (22.31 %) +2.6%
+ BM_ZFlat/1 1995238 1993582 335.9MB/s urls (47.77 %) +0.1%
+ BM_ZFlat/2 30133 24983 4.7GB/s jpg (99.87 %) +20.6%
+ BM_ZFlat/3 74453 73128 1.2GB/s pdf (82.07 %) +1.8%
+ BM_ZFlat/4 647674 633729 616.4MB/s html4 (22.51 %) +2.2%
+ BM_ZFlat/5 76259 76090 308.4MB/s cp (48.12 %) +0.2%
+ BM_ZFlat/6 31106 31084 342.1MB/s c (42.40 %) +0.1%
+ BM_ZFlat/7 10507 10443 339.8MB/s lsp (48.37 %) +0.6%
+ BM_ZFlat/8 1811047 1793325 547.6MB/s xls (41.23 %) +1.0%
+ BM_ZFlat/9 597903 581793 249.3MB/s txt1 (57.87 %) +2.8%
+ BM_ZFlat/10 525320 514522 232.0MB/s txt2 (61.93 %) +2.1%
+ BM_ZFlat/11 1596591 1551636 262.3MB/s txt3 (54.92 %) +2.9%
+ BM_ZFlat/12 2134523 2094033 219.5MB/s txt4 (66.22 %) +1.9%
+ BM_ZFlat/13 593024 587869 832.6MB/s bin (18.11 %) +0.9%
+ BM_ZFlat/14 114746 110666 329.5MB/s sum (48.96 %) +3.7%
+ BM_ZFlat/15 14376 14485 278.3MB/s man (59.36 %) -0.8%
+ BM_ZFlat/16 167908 150070 753.6MB/s pb (19.64 %) +11.9%
+ BM_ZFlat/17 460228 442253 397.5MB/s gaviota (37.72 %) +4.1%
+ BM_ZCord/0 164896 160241 609.4MB/s html +2.9%
+ BM_ZCord/1 2070239 2043492 327.7MB/s urls +1.3%
+ BM_ZCord/2 54402 47002 2.5GB/s jpg +15.7%
+ BM_ZCord/3 85871 83832 1073.1MB/s pdf +2.4%
+ BM_ZCord/4 664078 648825 602.0MB/s html4 +2.4%
+ BM_ZDataBuffer/0 174874 172549 566.0MB/s html +1.3%
+ BM_ZDataBuffer/1 2134410 2139173 313.0MB/s urls -0.2%
+ BM_ZDataBuffer/2 71911 69551 1.7GB/s jpg +3.4%
+ BM_ZDataBuffer/3 98236 99727 902.1MB/s pdf -1.5%
+ BM_ZDataBuffer/4 710776 699104 558.8MB/s html4 +1.7%
+ Sum of all benchmarks 27358908 27200688 +0.6%
+
+
+Sandy Bridge 2.6 GHz:
+
+ Benchmark Base (ns) New (ns) Improvement
+ -------------------------------------------------------------------------------------------------
+ BM_UFlat/0 49356 49018 1.9GB/s html +0.7%
+ BM_UFlat/1 516764 531955 1.2GB/s urls -2.9%
+ BM_UFlat/2 6982 7304 16.2GB/s jpg -4.4%
+ BM_UFlat/3 15285 15598 5.6GB/s pdf -2.0%
+ BM_UFlat/4 206557 206669 1.8GB/s html4 -0.1%
+ BM_UFlat/5 13681 13567 1.7GB/s cp +0.8%
+ BM_UFlat/6 6571 6592 1.6GB/s c -0.3%
+ BM_UFlat/7 2008 1994 1.7GB/s lsp +0.7%
+ BM_UFlat/8 775700 773286 1.2GB/s xls +0.3%
+ BM_UFlat/9 165578 164480 881.8MB/s txt1 +0.7%
+ BM_UFlat/10 143707 144139 828.2MB/s txt2 -0.3%
+ BM_UFlat/11 443026 436281 932.8MB/s txt3 +1.5%
+ BM_UFlat/12 603129 595856 771.2MB/s txt4 +1.2%
+ BM_UFlat/13 271682 270450 1.8GB/s bin +0.5%
+ BM_UFlat/14 26200 25666 1.4GB/s sum +2.1%
+ BM_UFlat/15 2620 2608 1.5GB/s man +0.5%
+ BM_UFlat/16 48908 47756 2.3GB/s pb +2.4%
+ BM_UFlat/17 174638 170346 1031.9MB/s gaviota +2.5%
+ BM_UValidate/0 31922 31898 3.0GB/s html +0.1%
+ BM_UValidate/1 341265 363554 1.8GB/s urls -6.1%
+ BM_UValidate/2 160 151 782.8GB/s jpg +6.0%
+ BM_UValidate/3 10402 10380 8.5GB/s pdf +0.2%
+ BM_UValidate/4 129490 130587 2.9GB/s html4 -0.8%
+ BM_UDataBuffer/0 59383 58736 1.6GB/s html +1.1%
+ BM_UDataBuffer/1 619222 637786 1049.8MB/s urls -2.9%
+ BM_UDataBuffer/2 10775 11941 9.9GB/s jpg -9.8%
+ BM_UDataBuffer/3 18002 17930 4.9GB/s pdf +0.4%
+ BM_UDataBuffer/4 259182 259306 1.5GB/s html4 -0.0%
+ BM_UCord/0 59379 57814 1.6GB/s html +2.7%
+ BM_UCord/1 598456 615162 1088.4MB/s urls -2.7%
+ BM_UCord/2 8519 8628 13.7GB/s jpg -1.3%
+ BM_UCord/3 18123 17537 5.0GB/s pdf +3.3%
+ BM_UCord/4 252375 252331 1.5GB/s html4 +0.0%
+ BM_UCordString/0 49494 49790 1.9GB/s html -0.6%
+ BM_UCordString/1 524659 541803 1.2GB/s urls -3.2%
+ BM_UCordString/2 8206 8354 14.2GB/s jpg -1.8%
+ BM_UCordString/3 17235 16537 5.3GB/s pdf +4.2%
+ BM_UCordString/4 210188 211072 1.8GB/s html4 -0.4%
+ BM_UCordValidate/0 31956 31587 3.0GB/s html +1.2%
+ BM_UCordValidate/1 340828 362141 1.8GB/s urls -5.9%
+ BM_UCordValidate/2 783 744 158.9GB/s jpg +5.2%
+ BM_UCordValidate/3 10543 10462 8.4GB/s pdf +0.8%
+ BM_UCordValidate/4 130150 129789 2.9GB/s html4 +0.3%
+ BM_ZFlat/0 113873 111200 878.2MB/s html (22.31 %) +2.4%
+ BM_ZFlat/1 1473023 1489858 449.4MB/s urls (47.77 %) -1.1%
+ BM_ZFlat/2 23569 19486 6.1GB/s jpg (99.87 %) +21.0%
+ BM_ZFlat/3 49178 48046 1.8GB/s pdf (82.07 %) +2.4%
+ BM_ZFlat/4 475063 469394 832.2MB/s html4 (22.51 %) +1.2%
+ BM_ZFlat/5 46910 46816 501.2MB/s cp (48.12 %) +0.2%
+ BM_ZFlat/6 16883 16916 628.6MB/s c (42.40 %) -0.2%
+ BM_ZFlat/7 5381 5447 651.5MB/s lsp (48.37 %) -1.2%
+ BM_ZFlat/8 1466870 1473861 666.3MB/s xls (41.23 %) -0.5%
+ BM_ZFlat/9 468006 464101 312.5MB/s txt1 (57.87 %) +0.8%
+ BM_ZFlat/10 408157 408957 291.9MB/s txt2 (61.93 %) -0.2%
+ BM_ZFlat/11 1253348 1232910 330.1MB/s txt3 (54.92 %) +1.7%
+ BM_ZFlat/12 1702373 1702977 269.8MB/s txt4 (66.22 %) -0.0%
+ BM_ZFlat/13 439792 438557 1116.0MB/s bin (18.11 %) +0.3%
+ BM_ZFlat/14 80766 78851 462.5MB/s sum (48.96 %) +2.4%
+ BM_ZFlat/15 7420 7542 534.5MB/s man (59.36 %) -1.6%
+ BM_ZFlat/16 112043 100126 1.1GB/s pb (19.64 %) +11.9%
+ BM_ZFlat/17 368877 357703 491.4MB/s gaviota (37.72 %) +3.1%
+ BM_ZCord/0 116402 113564 859.9MB/s html +2.5%
+ BM_ZCord/1 1507156 1519911 440.5MB/s urls -0.8%
+ BM_ZCord/2 39860 33686 3.5GB/s jpg +18.3%
+ BM_ZCord/3 56211 54694 1.6GB/s pdf +2.8%
+ BM_ZCord/4 485594 479212 815.1MB/s html4 +1.3%
+ BM_ZDataBuffer/0 123185 121572 803.3MB/s html +1.3%
+ BM_ZDataBuffer/1 1569111 1589380 421.3MB/s urls -1.3%
+ BM_ZDataBuffer/2 53143 49556 2.4GB/s jpg +7.2%
+ BM_ZDataBuffer/3 65725 66826 1.3GB/s pdf -1.6%
+ BM_ZDataBuffer/4 517871 514750 758.9MB/s html4 +0.6%
+ Sum of all benchmarks 20258879 20315484 -0.3%
+
+
+AMD Instanbul 2.4 GHz:
+
+ Benchmark Base (ns) New (ns) Improvement
+ -------------------------------------------------------------------------------------------------
+ BM_UFlat/0 97120 96585 1011.1MB/s html +0.6%
+ BM_UFlat/1 917473 948016 706.3MB/s urls -3.2%
+ BM_UFlat/2 21496 23938 4.9GB/s jpg -10.2%
+ BM_UFlat/3 44751 45639 1.9GB/s pdf -1.9%
+ BM_UFlat/4 391950 391413 998.0MB/s html4 +0.1%
+ BM_UFlat/5 37366 37201 630.7MB/s cp +0.4%
+ BM_UFlat/6 18350 18318 580.5MB/s c +0.2%
+ BM_UFlat/7 5672 5661 626.9MB/s lsp +0.2%
+ BM_UFlat/8 1533390 1529441 642.1MB/s xls +0.3%
+ BM_UFlat/9 335477 336553 431.0MB/s txt1 -0.3%
+ BM_UFlat/10 285140 292080 408.7MB/s txt2 -2.4%
+ BM_UFlat/11 888507 894758 454.9MB/s txt3 -0.7%
+ BM_UFlat/12 1187643 1210928 379.5MB/s txt4 -1.9%
+ BM_UFlat/13 493717 507447 964.5MB/s bin -2.7%
+ BM_UFlat/14 61740 60870 599.1MB/s sum +1.4%
+ BM_UFlat/15 7211 7187 560.9MB/s man +0.3%
+ BM_UFlat/16 97435 93100 1.2GB/s pb +4.7%
+ BM_UFlat/17 362662 356395 493.2MB/s gaviota +1.8%
+ BM_UValidate/0 47475 47118 2.0GB/s html +0.8%
+ BM_UValidate/1 501304 529741 1.2GB/s urls -5.4%
+ BM_UValidate/2 276 243 486.2GB/s jpg +13.6%
+ BM_UValidate/3 16361 16261 5.4GB/s pdf +0.6%
+ BM_UValidate/4 190741 190353 2.0GB/s html4 +0.2%
+ BM_UDataBuffer/0 111080 109771 889.6MB/s html +1.2%
+ BM_UDataBuffer/1 1051035 1085999 616.5MB/s urls -3.2%
+ BM_UDataBuffer/2 25801 25463 4.6GB/s jpg +1.3%
+ BM_UDataBuffer/3 50493 49946 1.8GB/s pdf +1.1%
+ BM_UDataBuffer/4 447258 444138 879.5MB/s html4 +0.7%
+ BM_UCord/0 109350 107909 905.0MB/s html +1.3%
+ BM_UCord/1 1023396 1054964 634.7MB/s urls -3.0%
+ BM_UCord/2 25292 24371 4.9GB/s jpg +3.8%
+ BM_UCord/3 48955 49736 1.8GB/s pdf -1.6%
+ BM_UCord/4 440452 437331 893.2MB/s html4 +0.7%
+ BM_UCordString/0 98511 98031 996.2MB/s html +0.5%
+ BM_UCordString/1 933230 963495 694.9MB/s urls -3.1%
+ BM_UCordString/2 23311 24076 4.9GB/s jpg -3.2%
+ BM_UCordString/3 45568 46196 1.9GB/s pdf -1.4%
+ BM_UCordString/4 397791 396934 984.1MB/s html4 +0.2%
+ BM_UCordValidate/0 47537 46921 2.0GB/s html +1.3%
+ BM_UCordValidate/1 505071 532716 1.2GB/s urls -5.2%
+ BM_UCordValidate/2 1663 1621 72.9GB/s jpg +2.6%
+ BM_UCordValidate/3 16890 16926 5.2GB/s pdf -0.2%
+ BM_UCordValidate/4 192365 191984 2.0GB/s html4 +0.2%
+ BM_ZFlat/0 184708 179103 545.3MB/s html (22.31 %) +3.1%
+ BM_ZFlat/1 2293864 2302950 290.7MB/s urls (47.77 %) -0.4%
+ BM_ZFlat/2 52852 47618 2.5GB/s jpg (99.87 %) +11.0%
+ BM_ZFlat/3 100766 96179 935.3MB/s pdf (82.07 %) +4.8%
+ BM_ZFlat/4 741220 727977 536.6MB/s html4 (22.51 %) +1.8%
+ BM_ZFlat/5 85402 85418 274.7MB/s cp (48.12 %) -0.0%
+ BM_ZFlat/6 36558 36494 291.4MB/s c (42.40 %) +0.2%
+ BM_ZFlat/7 12706 12507 283.7MB/s lsp (48.37 %) +1.6%
+ BM_ZFlat/8 2336823 2335688 420.5MB/s xls (41.23 %) +0.0%
+ BM_ZFlat/9 701804 681153 212.9MB/s txt1 (57.87 %) +3.0%
+ BM_ZFlat/10 606700 597194 199.9MB/s txt2 (61.93 %) +1.6%
+ BM_ZFlat/11 1852283 1803238 225.7MB/s txt3 (54.92 %) +2.7%
+ BM_ZFlat/12 2475527 2443354 188.1MB/s txt4 (66.22 %) +1.3%
+ BM_ZFlat/13 694497 696654 702.6MB/s bin (18.11 %) -0.3%
+ BM_ZFlat/14 136929 129855 280.8MB/s sum (48.96 %) +5.4%
+ BM_ZFlat/15 17172 17124 235.4MB/s man (59.36 %) +0.3%
+ BM_ZFlat/16 190364 171763 658.4MB/s pb (19.64 %) +10.8%
+ BM_ZFlat/17 567285 555190 316.6MB/s gaviota (37.72 %) +2.2%
+ BM_ZCord/0 193490 187031 522.1MB/s html +3.5%
+ BM_ZCord/1 2427537 2415315 277.2MB/s urls +0.5%
+ BM_ZCord/2 85378 81412 1.5GB/s jpg +4.9%
+ BM_ZCord/3 121898 119419 753.3MB/s pdf +2.1%
+ BM_ZCord/4 779564 762961 512.0MB/s html4 +2.2%
+ BM_ZDataBuffer/0 213820 207272 471.1MB/s html +3.2%
+ BM_ZDataBuffer/1 2589010 2586495 258.9MB/s urls +0.1%
+ BM_ZDataBuffer/2 121871 118885 1018.4MB/s jpg +2.5%
+ BM_ZDataBuffer/3 145382 145986 616.2MB/s pdf -0.4%
+ BM_ZDataBuffer/4 868117 852754 458.1MB/s html4 +1.8%
+ Sum of all benchmarks 33771833 33744763 +0.1%
+
+------------------------------------------------------------------------
+r70 | snappy.mirrorbot@gmail.com | 2013-01-06 20:21:26 +0100 (Sun, 06 Jan 2013) | 6 lines
+
+Adjust the Snappy open-source distribution for the changes in Google's
+internal file API.
+
+R=sanjay
+
+
+------------------------------------------------------------------------
+r69 | snappy.mirrorbot@gmail.com | 2013-01-04 12:54:20 +0100 (Fri, 04 Jan 2013) | 15 lines
+
+Change a few ORs to additions where they don't matter. This helps the compiler
+use the LEA instruction more efficiently, since e.g. a + (b << 2) can be encoded
+as one instruction. Even more importantly, it can constant-fold the
+COPY_* enums together with the shifted negative constants, which also saves
+some instructions. (We don't need it for LITERAL, since it happens to be 0.)
+
+I am unsure why the compiler couldn't do this itself, but the theory is that
+it cannot prove that len-1 and len-4 cannot underflow/wrap, and thus can't
+do the optimization safely.
+
+The gains are small but measurable; 0.5-1.0% over the BM_Z* benchmarks
+(measured on Westmere, Sandy Bridge and Istanbul).
+
+R=sanjay
+
+------------------------------------------------------------------------
+r68 | snappy.mirrorbot@gmail.com | 2012-10-08 13:37:16 +0200 (Mon, 08 Oct 2012) | 5 lines
+
+Stop giving -Werror to automake, due to an incompatibility between current
+versions of libtool and automake on non-GNU platforms (e.g. Mac OS X).
+
+R=sanjay
+
+------------------------------------------------------------------------
+r67 | snappy.mirrorbot@gmail.com | 2012-08-17 15:54:47 +0200 (Fri, 17 Aug 2012) | 5 lines
+
+Fix public issue 66: Document GetUncompressedLength better, in particular that
+it leaves the source in a state that's not appropriate for RawUncompress.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r66 | snappy.mirrorbot@gmail.com | 2012-07-31 13:44:44 +0200 (Tue, 31 Jul 2012) | 5 lines
+
+Fix public issue 64: Check for <sys/time.h> at configure time,
+since MSVC seemingly does not have it.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r65 | snappy.mirrorbot@gmail.com | 2012-07-04 11:34:48 +0200 (Wed, 04 Jul 2012) | 10 lines
+
+Handle the case where gettimeofday() goes backwards or returns the same value
+twice; it could cause division by zero in the unit test framework.
+(We already had one fix for this in place, but it was incomplete.)
+
+This could in theory happen on any system, since there are few guarantees
+about gettimeofday(), but seems to only happen in practice on GNU/Hurd, where
+gettimeofday() is cached and only updated ever so often.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r64 | snappy.mirrorbot@gmail.com | 2012-07-04 11:28:33 +0200 (Wed, 04 Jul 2012) | 6 lines
+
+Mark ARMv4 as not supporting unaligned accesses (not just ARMv5 and ARMv6);
+apparently Debian still targets these by default, giving us segfaults on
+armel.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r63 | snappy.mirrorbot@gmail.com | 2012-05-22 11:46:05 +0200 (Tue, 22 May 2012) | 5 lines
+
+Fix public bug #62: Remove an extraneous comma at the end of an enum list,
+causing compile errors when embedded in Mozilla on OpenBSD.
+
+R=sanjay
+
+------------------------------------------------------------------------
+r62 | snappy.mirrorbot@gmail.com | 2012-05-22 11:32:50 +0200 (Tue, 22 May 2012) | 8 lines
+
+Snappy library no longer depends on iostream.
+
+Achieved by moving logging macro definitions to a test-only
+header file, and by changing non-test code to use assert,
+fprintf, and abort instead of LOG/CHECK macros.
+
+R=sesse
+
+------------------------------------------------------------------------
+r61 | snappy.mirrorbot@gmail.com | 2012-02-24 16:46:37 +0100 (Fri, 24 Feb 2012) | 4 lines
+
+Release Snappy 1.0.5.
+
+R=sanjay
+
+------------------------------------------------------------------------
r60 | snappy.mirrorbot@gmail.com | 2012-02-23 18:00:36 +0100 (Thu, 23 Feb 2012) | 57 lines
For 32-bit platforms, do not try to accelerate multiple neighboring
diff --git a/thirdparty/snappy/NEWS b/thirdparty/snappy/NEWS
index 60bbd17d..27a5b176 100644
--- a/thirdparty/snappy/NEWS
+++ b/thirdparty/snappy/NEWS
@@ -1,3 +1,42 @@
+Snappy v1.1.2, February 28th 2014:
+
+This is a maintenance release with no changes to the actual library
+source code.
+
+ * Stop distributing benchmark data files that have unclear
+ or unsuitable licensing.
+
+ * Add support for padding chunks in the framing format.
+
+
+Snappy v1.1.1, October 15th 2013:
+
+ * Add support for uncompressing to iovecs (scatter I/O).
+ The bulk of this patch was contributed by Mohit Aron.
+
+ * Speed up decompression by ~2%; much more so (~13-20%) on
+ a few benchmarks on given compilers and CPUs.
+
+ * Fix a few issues with MSVC compilation.
+
+ * Support truncated test data in the benchmark.
+
+
+Snappy v1.1.0, January 18th 2013:
+
+ * Snappy now uses 64 kB block size instead of 32 kB. On average,
+ this means it compresses about 3% denser (more so for some
+ inputs), at the same or better speeds.
+
+ * libsnappy no longer depends on iostream.
+
+ * Some small performance improvements in compression on x86
+ (0.5–1%).
+
+ * Various portability fixes for ARM-based platforms, for MSVC,
+ and for GNU/Hurd.
+
+
Snappy v1.0.5, February 24th 2012:
* More speed improvements. Exactly how big will depend on
diff --git a/thirdparty/snappy/config.h b/thirdparty/snappy/config.h
index 6e6e7cf6..ab70985d 100644
--- a/thirdparty/snappy/config.h
+++ b/thirdparty/snappy/config.h
@@ -18,34 +18,34 @@
#endif
/* Define to 1 if you have the <dlfcn.h> header file. */
-#undef HAVE_DLFCN_H
+/* #undef HAVE_DLFCN_H */
/* Use the gflags package for command-line parsing. */
-#undef HAVE_GFLAGS
+/* #undef HAVE_GFLAGS */
/* Defined when Google Test is available. */
-#undef HAVE_GTEST
+/* #undef HAVE_GTEST */
/* Define to 1 if you have the <inttypes.h> header file. */
#define HAVE_INTTYPES_H 1
/* Define to 1 if you have the `fastlz' library (-lfastlz). */
-#undef HAVE_LIBFASTLZ
+/* #undef HAVE_LIBFASTLZ */
/* Define to 1 if you have the `lzf' library (-llzf). */
-#undef HAVE_LIBLZF
+/* #undef HAVE_LIBLZF */
/* Define to 1 if you have the `lzo2' library (-llzo2). */
-#undef HAVE_LIBLZO2
+/* #undef HAVE_LIBLZO2 */
/* Define to 1 if you have the `quicklz' library (-lquicklz). */
-#undef HAVE_LIBQUICKLZ
+/* #undef HAVE_LIBQUICKLZ */
/* Define to 1 if you have the `z' library (-lz). */
-#undef HAVE_LIBZ
+/* #undef HAVE_LIBZ */
/* Define to 1 if you have the <memory.h> header file. */
-#undef HAVE_MEMORY_H
+/* #undef HAVE_MEMORY_H */
/* Define to 1 if you have the <stddef.h> header file. */
#define HAVE_STDDEF_H 1
@@ -57,31 +57,36 @@
#define HAVE_STDLIB_H 1
/* Define to 1 if you have the <strings.h> header file. */
-#undef HAVE_STRINGS_H
+/* #undef HAVE_STRINGS_H */
/* Define to 1 if you have the <string.h> header file. */
#define HAVE_STRING_H 1
/* Define to 1 if you have the <sys/byteswap.h> header file. */
-#undef HAVE_SYS_BYTESWAP_H
+/* #undef HAVE_SYS_BYTESWAP_H */
/* Define to 1 if you have the <sys/endian.h> header file. */
-#undef HAVE_SYS_ENDIAN_H
+/* #undef HAVE_SYS_ENDIAN_H */
/* Define to 1 if you have the <sys/mman.h> header file. */
-#undef HAVE_SYS_MMAN_H
+/* #undef HAVE_SYS_MMAN_H */
/* Define to 1 if you have the <sys/resource.h> header file. */
-#undef HAVE_SYS_RESOURCE_H
+/* #undef HAVE_SYS_RESOURCE_H */
/* Define to 1 if you have the <sys/stat.h> header file. */
-#undef HAVE_SYS_STAT_H
+/* #undef HAVE_SYS_STAT_H */
+
+/* Define to 1 if you have the <sys/time.h> header file. */
+/* #undef HAVE_SYS_TIME_H */
/* Define to 1 if you have the <sys/types.h> header file. */
-#define HAVE_SYS_TYPES_H 1
+#if !defined(_MSC_VER)
+# define HAVE_SYS_TYPES_H 1
+#endif
/* Define to 1 if you have the <unistd.h> header file. */
-#if !defined(_WIN32)
+#if !defined(_MSC_VER)
# define HAVE_UNISTD_H 1
#endif
@@ -92,37 +97,34 @@
/* Define to the sub-directory in which libtool stores uninstalled libraries.
*/
-#undef LT_OBJDIR
+/* #undef LT_OBJDIR */
/* Name of package */
-#undef PACKAGE
+/* #undef PACKAGE */
/* Define to the address where bug reports for this package should be sent. */
-#undef PACKAGE_BUGREPORT
+/* #undef PACKAGE_BUGREPORT */
/* Define to the full name of this package. */
-#undef PACKAGE_NAME
+/* #undef PACKAGE_NAME */
/* Define to the full name and version of this package. */
-#undef PACKAGE_STRING
+/* #undef PACKAGE_STRING */
/* Define to the one symbol short name of this package. */
-#undef PACKAGE_TARNAME
-
-/* Define to the home page for this package. */
-#undef PACKAGE_URL
+/* #undef PACKAGE_TARNAME */
/* Define to the version of this package. */
-#undef PACKAGE_VERSION
+/* #undef PACKAGE_VERSION */
/* Define to 1 if you have the ANSI C header files. */
#define STDC_HEADERS 1
/* Version number of package */
-#undef VERSION
+/* #undef VERSION */
/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
- significant byte first (like Motorola and SPARC, unlike Intel). */
+ significant byte first (like Motorola and SPARC, unlike Intel and VAX). */
#if defined(__GLIBC__)
# include <endian.h>
# if __BYTE_ORDER == __BIG_ENDIAN
@@ -134,3 +136,11 @@
# define WORDS_BIGENDIAN 1
# endif
#endif
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+/* #undef size_t */
+
+/* Define to `int' if <sys/types.h> does not define. */
+#if defined(_MSC_VER)
+# define ssize_t int
+#endif
diff --git a/thirdparty/snappy/framing_format.txt b/thirdparty/snappy/framing_format.txt
index 08fda03f..9764e83d 100644
--- a/thirdparty/snappy/framing_format.txt
+++ b/thirdparty/snappy/framing_format.txt
@@ -1,5 +1,5 @@
Snappy framing format description
-Last revised: 2011-12-15
+Last revised: 2013-10-25
This format decribes a framing format for Snappy, allowing compressing to
files or streams that can then more easily be decompressed without having
@@ -15,9 +15,9 @@ decompressor; it is not part of the Snappy core specification.
The file consists solely of chunks, lying back-to-back with no padding
in between. Each chunk consists first a single byte of chunk identifier,
-then a two-byte little-endian length of the chunk in bytes (from 0 to 65535,
-inclusive), and then the data if any. The three bytes of chunk header is not
-counted in the data length.
+then a three-byte little-endian length of the chunk in bytes (from 0 to
+16777215, inclusive), and then the data if any. The four bytes of chunk
+header is not counted in the data length.
The different chunk types are listed below. The first chunk must always
be the stream identifier chunk (see section 4.1, below). The stream
@@ -71,7 +71,7 @@ The stream identifier is always the first element in the stream.
It is exactly six bytes long and contains "sNaPpY" in ASCII. This means that
a valid Snappy framed stream always starts with the bytes
- 0xff 0x06 0x00 0x73 0x4e 0x61 0x50 0x70 0x59
+ 0xff 0x06 0x00 0x00 0x73 0x4e 0x61 0x50 0x70 0x59
The stream identifier chunk can come multiple times in the stream besides
the first; if such a chunk shows up, it should simply be ignored, assuming
@@ -86,9 +86,9 @@ see the compressed format specification. The compressed data is preceded by
the CRC-32C (see section 3) of the _uncompressed_ data.
Note that the data portion of the chunk, i.e., the compressed contents,
-can be at most 65531 bytes (2^16 - 1, minus the checksum).
+can be at most 16777211 bytes (2^24 - 1, minus the checksum).
However, we place an additional restriction that the uncompressed data
-in a chunk must be no longer than 32768 bytes. This allows consumers to
+in a chunk must be no longer than 65536 bytes. This allows consumers to
easily use small fixed-size buffers.
@@ -102,11 +102,22 @@ As in the compressed chunks, the data is preceded by its own masked
CRC-32C (see section 3).
An uncompressed data chunk, like compressed data chunks, should contain
-no more than 32768 data bytes, so the maximum legal chunk length with the
-checksum is 32772.
+no more than 65536 data bytes, so the maximum legal chunk length with the
+checksum is 65540.
-4.4. Reserved unskippable chunks (chunk types 0x02-0x7f)
+4.4. Padding (chunk type 0xfe)
+
+Padding chunks allow a compressor to increase the size of the data stream
+so that it complies with external demands, e.g. that the total number of
+bytes is a multiple of some value.
+
+All bytes of the padding chunk, except the chunk byte itself and the length,
+should be zero, but decompressors must not try to interpret or verify the
+padding data in any way.
+
+
+4.5. Reserved unskippable chunks (chunk types 0x02-0x7f)
These are reserved for future expansion. A decoder that sees such a chunk
should immediately return an error, as it must assume it cannot decode the
@@ -115,10 +126,10 @@ stream correctly.
Future versions of this specification may define meanings for these chunks.
-4.5. Reserved skippable chunks (chunk types 0x80-0xfe)
+4.6. Reserved skippable chunks (chunk types 0x80-0xfd)
These are also reserved for future expansion, but unlike the chunks
-described in 4.4, a decoder seeing these must skip them and continue
+described in 4.5, a decoder seeing these must skip them and continue
decoding.
Future versions of this specification may define meanings for these chunks.
diff --git a/thirdparty/snappy/snappy-c.h b/thirdparty/snappy/snappy-c.h
index 88e1122d..c6c2a860 100644
--- a/thirdparty/snappy/snappy-c.h
+++ b/thirdparty/snappy/snappy-c.h
@@ -46,7 +46,7 @@ extern "C" {
typedef enum {
SNAPPY_OK = 0,
SNAPPY_INVALID_INPUT = 1,
- SNAPPY_BUFFER_TOO_SMALL = 2,
+ SNAPPY_BUFFER_TOO_SMALL = 2
} snappy_status;
/*
diff --git a/thirdparty/snappy/snappy-internal.h b/thirdparty/snappy/snappy-internal.h
index a32eda59..c99d3313 100644
--- a/thirdparty/snappy/snappy-internal.h
+++ b/thirdparty/snappy/snappy-internal.h
@@ -85,7 +85,7 @@ char* CompressFragment(const char* input,
static inline int FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit) {
- DCHECK_GE(s2_limit, s2);
+ assert(s2_limit >= s2);
int matched = 0;
// Find out how long the match is. We loop over the data 64 bits at a
@@ -122,7 +122,7 @@ static inline int FindMatchLength(const char* s1,
const char* s2,
const char* s2_limit) {
// Implementation based on the x86-64 version, above.
- DCHECK_GE(s2_limit, s2);
+ assert(s2_limit >= s2);
int matched = 0;
while (s2 <= s2_limit - 4 &&
diff --git a/thirdparty/snappy/snappy-stubs-internal.h b/thirdparty/snappy/snappy-stubs-internal.h
index 6033cdfb..12393b62 100644
--- a/thirdparty/snappy/snappy-stubs-internal.h
+++ b/thirdparty/snappy/snappy-stubs-internal.h
@@ -35,7 +35,6 @@
#include "config.h"
#endif
-#include <iostream>
#include <string>
#include <assert.h>
@@ -95,87 +94,6 @@ namespace snappy {
static const uint32 kuint32max = static_cast<uint32>(0xFFFFFFFF);
static const int64 kint64max = static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
-// Logging.
-
-#define LOG(level) LogMessage()
-#define VLOG(level) true ? (void)0 : \
- snappy::LogMessageVoidify() & snappy::LogMessage()
-
-class LogMessage {
- public:
- LogMessage() { }
- ~LogMessage() {
- cerr << endl;
- }
-
- LogMessage& operator<<(const std::string& msg) {
- cerr << msg;
- return *this;
- }
- LogMessage& operator<<(int x) {
- cerr << x;
- return *this;
- }
-};
-
-// Asserts, both versions activated in debug mode only,
-// and ones that are always active.
-
-#define CRASH_UNLESS(condition) \
- PREDICT_TRUE(condition) ? (void)0 : \
- snappy::LogMessageVoidify() & snappy::LogMessageCrash()
-
-class LogMessageCrash : public LogMessage {
- public:
- LogMessageCrash() { }
- ~LogMessageCrash() {
- cerr << endl;
- abort();
- }
-};
-
-// This class is used to explicitly ignore values in the conditional
-// logging macros. This avoids compiler warnings like "value computed
-// is not used" and "statement has no effect".
-
-class LogMessageVoidify {
- public:
- LogMessageVoidify() { }
- // This has to be an operator with a precedence lower than << but
- // higher than ?:
- void operator&(const LogMessage&) { }
-};
-
-#define CHECK(cond) CRASH_UNLESS(cond)
-#define CHECK_LE(a, b) CRASH_UNLESS((a) <= (b))
-#define CHECK_GE(a, b) CRASH_UNLESS((a) >= (b))
-#define CHECK_EQ(a, b) CRASH_UNLESS((a) == (b))
-#define CHECK_NE(a, b) CRASH_UNLESS((a) != (b))
-#define CHECK_LT(a, b) CRASH_UNLESS((a) < (b))
-#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b))
-
-#ifdef NDEBUG
-
-#define DCHECK(cond) CRASH_UNLESS(true)
-#define DCHECK_LE(a, b) CRASH_UNLESS(true)
-#define DCHECK_GE(a, b) CRASH_UNLESS(true)
-#define DCHECK_EQ(a, b) CRASH_UNLESS(true)
-#define DCHECK_NE(a, b) CRASH_UNLESS(true)
-#define DCHECK_LT(a, b) CRASH_UNLESS(true)
-#define DCHECK_GT(a, b) CRASH_UNLESS(true)
-
-#else
-
-#define DCHECK(cond) CHECK(cond)
-#define DCHECK_LE(a, b) CHECK_LE(a, b)
-#define DCHECK_GE(a, b) CHECK_GE(a, b)
-#define DCHECK_EQ(a, b) CHECK_EQ(a, b)
-#define DCHECK_NE(a, b) CHECK_NE(a, b)
-#define DCHECK_LT(a, b) CHECK_LT(a, b)
-#define DCHECK_GT(a, b) CHECK_GT(a, b)
-
-#endif
-
// Potentially unaligned loads and stores.
// x86 and PowerPC can simply do these loads and stores native.
@@ -200,6 +118,8 @@ class LogMessageVoidify {
// This is a mess, but there's not much we can do about it.
#elif defined(__arm__) && \
+ !defined(__ARM_ARCH_4__) && \
+ !defined(__ARM_ARCH_4T__) && \
!defined(__ARM_ARCH_5__) && \
!defined(__ARM_ARCH_5T__) && \
!defined(__ARM_ARCH_5TE__) && \
diff --git a/thirdparty/snappy/snappy-stubs-public.h b/thirdparty/snappy/snappy-stubs-public.h
index 9ee4ca52..6b41bbe9 100644
--- a/thirdparty/snappy/snappy-stubs-public.h
+++ b/thirdparty/snappy/snappy-stubs-public.h
@@ -44,9 +44,13 @@
#include <stddef.h>
#endif
+#if 0
+#include <sys/uio.h>
+#endif
+
#define SNAPPY_MAJOR 1
-#define SNAPPY_MINOR 0
-#define SNAPPY_PATCHLEVEL 5
+#define SNAPPY_MINOR 1
+#define SNAPPY_PATCHLEVEL 2
#define SNAPPY_VERSION \
((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL)
@@ -80,6 +84,15 @@ typedef std::string string;
TypeName(const TypeName&); \
void operator=(const TypeName&)
+#if !0
+// Windows does not have an iovec type, yet the concept is universally useful.
+// It is simple to define it ourselves, so we put it inside our own namespace.
+struct iovec {
+ void* iov_base;
+ size_t iov_len;
+};
+#endif
+
} // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
diff --git a/thirdparty/snappy/snappy-stubs-public.h.in b/thirdparty/snappy/snappy-stubs-public.h.in
index f0babcbe..6c181a1c 100644
--- a/thirdparty/snappy/snappy-stubs-public.h.in
+++ b/thirdparty/snappy/snappy-stubs-public.h.in
@@ -44,6 +44,10 @@
#include <stddef.h>
#endif
+#if @ac_cv_have_sys_uio_h@
+#include <sys/uio.h>
+#endif
+
#define SNAPPY_MAJOR @SNAPPY_MAJOR@
#define SNAPPY_MINOR @SNAPPY_MINOR@
#define SNAPPY_PATCHLEVEL @SNAPPY_PATCHLEVEL@
@@ -80,6 +84,15 @@ typedef std::string string;
TypeName(const TypeName&); \
void operator=(const TypeName&)
+#if !@ac_cv_have_sys_uio_h@
+// Windows does not have an iovec type, yet the concept is universally useful.
+// It is simple to define it ourselves, so we put it inside our own namespace.
+struct iovec {
+ void* iov_base;
+ size_t iov_len;
+};
+#endif
+
} // namespace snappy
#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_
diff --git a/thirdparty/snappy/snappy-test.cc b/thirdparty/snappy/snappy-test.cc
index 223cd92d..46194109 100644
--- a/thirdparty/snappy/snappy-test.cc
+++ b/thirdparty/snappy/snappy-test.cc
@@ -42,18 +42,25 @@ DEFINE_bool(run_microbenchmarks, true,
namespace snappy {
-string ReadTestDataFile(const string& base) {
+string ReadTestDataFile(const string& base, size_t size_limit) {
string contents;
const char* srcdir = getenv("srcdir"); // This is set by Automake.
+ string prefix;
if (srcdir) {
- File::ReadFileToStringOrDie(
- string(srcdir) + "/testdata/" + base, &contents);
- } else {
- File::ReadFileToStringOrDie("testdata/" + base, &contents);
+ prefix = string(srcdir) + "/";
+ }
+ file::GetContents(prefix + "testdata/" + base, &contents, file::Defaults()
+ ).CheckSuccess();
+ if (size_limit > 0) {
+ contents = contents.substr(0, size_limit);
}
return contents;
}
+string ReadTestDataFile(const string& base) {
+ return ReadTestDataFile(base, 0);
+}
+
string StringPrintf(const char* format, ...) {
char buf[4096];
va_list ap;
@@ -204,27 +211,32 @@ void Benchmark::Run() {
benchmark_runs[run].cpu_time_us = benchmark_cpu_time_us;
}
+ string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
+ string human_readable_speed;
+
nth_element(benchmark_runs,
benchmark_runs + kMedianPos,
benchmark_runs + kNumRuns,
BenchmarkCompareCPUTime());
int64 real_time_us = benchmark_runs[kMedianPos].real_time_us;
int64 cpu_time_us = benchmark_runs[kMedianPos].cpu_time_us;
- int64 bytes_per_second = benchmark_bytes_processed * 1000000 / cpu_time_us;
-
- string heading = StringPrintf("%s/%d", name_.c_str(), test_case_num);
- string human_readable_speed;
- if (bytes_per_second < 1024) {
- human_readable_speed = StringPrintf("%dB/s", bytes_per_second);
- } else if (bytes_per_second < 1024 * 1024) {
- human_readable_speed = StringPrintf(
- "%.1fkB/s", bytes_per_second / 1024.0f);
- } else if (bytes_per_second < 1024 * 1024 * 1024) {
- human_readable_speed = StringPrintf(
- "%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
+ if (cpu_time_us <= 0) {
+ human_readable_speed = "?";
} else {
- human_readable_speed = StringPrintf(
- "%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
+ int64 bytes_per_second =
+ benchmark_bytes_processed * 1000000 / cpu_time_us;
+ if (bytes_per_second < 1024) {
+ human_readable_speed = StringPrintf("%dB/s", bytes_per_second);
+ } else if (bytes_per_second < 1024 * 1024) {
+ human_readable_speed = StringPrintf(
+ "%.1fkB/s", bytes_per_second / 1024.0f);
+ } else if (bytes_per_second < 1024 * 1024 * 1024) {
+ human_readable_speed = StringPrintf(
+ "%.1fMB/s", bytes_per_second / (1024.0f * 1024.0f));
+ } else {
+ human_readable_speed = StringPrintf(
+ "%.1fGB/s", bytes_per_second / (1024.0f * 1024.0f * 1024.0f));
+ }
}
fprintf(stderr,
diff --git a/thirdparty/snappy/snappy-test.h b/thirdparty/snappy/snappy-test.h
index ef6a9553..0f18bf14 100644
--- a/thirdparty/snappy/snappy-test.h
+++ b/thirdparty/snappy/snappy-test.h
@@ -31,6 +31,9 @@
#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_TEST_H_
+#include <iostream>
+#include <string>
+
#include "snappy-stubs-internal.h"
#include <stdio.h>
@@ -44,7 +47,9 @@
#include <sys/resource.h>
#endif
+#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
+#endif
#ifdef HAVE_WINDOWS_H
#define WIN32_LEAN_AND_MEAN
@@ -121,13 +126,23 @@ extern "C" {
#endif
namespace {
+
namespace File {
void Init() { }
+} // namespace File
+
+namespace file {
+ int Defaults() { }
- void ReadFileToStringOrDie(const char* filename, string* data) {
- FILE* fp = fopen(filename, "rb");
+ class DummyStatus {
+ public:
+ void CheckSuccess() { }
+ };
+
+ DummyStatus GetContents(const string& filename, string* data, int unused) {
+ FILE* fp = fopen(filename.c_str(), "rb");
if (fp == NULL) {
- perror(filename);
+ perror(filename.c_str());
exit(1);
}
@@ -145,14 +160,12 @@ namespace File {
fclose(fp);
}
- void ReadFileToStringOrDie(const string& filename, string* data) {
- ReadFileToStringOrDie(filename.c_str(), data);
- }
-
- void WriteStringToFileOrDie(const string& str, const char* filename) {
- FILE* fp = fopen(filename, "wb");
+ DummyStatus SetContents(const string& filename,
+ const string& str,
+ int unused) {
+ FILE* fp = fopen(filename.c_str(), "wb");
if (fp == NULL) {
- perror(filename);
+ perror(filename.c_str());
exit(1);
}
@@ -164,7 +177,8 @@ namespace File {
fclose(fp);
}
-} // namespace File
+} // namespace file
+
} // namespace
namespace snappy {
@@ -183,6 +197,8 @@ void Test_Snappy_ReadPastEndOfBuffer();
void Test_Snappy_FindMatchLength();
void Test_Snappy_FindMatchLengthRandom();
+string ReadTestDataFile(const string& base, size_t size_limit);
+
string ReadTestDataFile(const string& base);
// A sprintf() variant that returns a std::string.
@@ -312,6 +328,7 @@ class Benchmark {
(new Benchmark(#benchmark_name, benchmark_name))
extern Benchmark* Benchmark_BM_UFlat;
+extern Benchmark* Benchmark_BM_UIOVec;
extern Benchmark* Benchmark_BM_UValidate;
extern Benchmark* Benchmark_BM_ZFlat;
@@ -462,6 +479,7 @@ static void RunSpecifiedBenchmarks() {
fprintf(stderr, "---------------------------------------------------\n");
snappy::Benchmark_BM_UFlat->Run();
+ snappy::Benchmark_BM_UIOVec->Run();
snappy::Benchmark_BM_UValidate->Run();
snappy::Benchmark_BM_ZFlat->Run();
@@ -496,6 +514,65 @@ static void CompressFile(const char* fname);
static void UncompressFile(const char* fname);
static void MeasureFile(const char* fname);
+// Logging.
+
+#define LOG(level) LogMessage()
+#define VLOG(level) true ? (void)0 : \
+ snappy::LogMessageVoidify() & snappy::LogMessage()
+
+class LogMessage {
+ public:
+ LogMessage() { }
+ ~LogMessage() {
+ cerr << endl;
+ }
+
+ LogMessage& operator<<(const std::string& msg) {
+ cerr << msg;
+ return *this;
+ }
+ LogMessage& operator<<(int x) {
+ cerr << x;
+ return *this;
+ }
+};
+
+// Asserts, both versions activated in debug mode only,
+// and ones that are always active.
+
+#define CRASH_UNLESS(condition) \
+ PREDICT_TRUE(condition) ? (void)0 : \
+ snappy::LogMessageVoidify() & snappy::LogMessageCrash()
+
+class LogMessageCrash : public LogMessage {
+ public:
+ LogMessageCrash() { }
+ ~LogMessageCrash() {
+ cerr << endl;
+ abort();
+ }
+};
+
+// This class is used to explicitly ignore values in the conditional
+// logging macros. This avoids compiler warnings like "value computed
+// is not used" and "statement has no effect".
+
+class LogMessageVoidify {
+ public:
+ LogMessageVoidify() { }
+ // This has to be an operator with a precedence lower than << but
+ // higher than ?:
+ void operator&(const LogMessage&) { }
+};
+
+#define CHECK(cond) CRASH_UNLESS(cond)
+#define CHECK_LE(a, b) CRASH_UNLESS((a) <= (b))
+#define CHECK_GE(a, b) CRASH_UNLESS((a) >= (b))
+#define CHECK_EQ(a, b) CRASH_UNLESS((a) == (b))
+#define CHECK_NE(a, b) CRASH_UNLESS((a) != (b))
+#define CHECK_LT(a, b) CRASH_UNLESS((a) < (b))
+#define CHECK_GT(a, b) CRASH_UNLESS((a) > (b))
+
} // namespace
using snappy::CompressFile;
diff --git a/thirdparty/snappy/snappy.cc b/thirdparty/snappy/snappy.cc
index 4d4eb42a..f8d0d23d 100644
--- a/thirdparty/snappy/snappy.cc
+++ b/thirdparty/snappy/snappy.cc
@@ -82,6 +82,7 @@ enum {
COPY_2_BYTE_OFFSET = 2,
COPY_4_BYTE_OFFSET = 3
};
+static const int kMaximumTagLength = 5; // COPY_4_BYTE_OFFSET plus the actual offset.
// Copy "len" bytes from "src" to "op", one byte at a time. Used for
// handling COPY operations where the input and output regions may
@@ -94,8 +95,8 @@ enum {
// ababababababababababab
// Note that this does not match the semantics of either memcpy()
// or memmove().
-static inline void IncrementalCopy(const char* src, char* op, int len) {
- DCHECK_GT(len, 0);
+static inline void IncrementalCopy(const char* src, char* op, ssize_t len) {
+ assert(len > 0);
do {
*op++ = *src++;
} while (--len > 0);
@@ -136,9 +137,7 @@ namespace {
const int kMaxIncrementCopyOverflow = 10;
-} // namespace
-
-static inline void IncrementalCopyFastPath(const char* src, char* op, int len) {
+inline void IncrementalCopyFastPath(const char* src, char* op, ssize_t len) {
while (op - src < 8) {
UnalignedCopy64(src, op);
len -= op - src;
@@ -152,6 +151,8 @@ static inline void IncrementalCopyFastPath(const char* src, char* op, int len) {
}
}
+} // namespace
+
static inline char* EmitLiteral(char* op,
const char* literal,
int len,
@@ -195,17 +196,17 @@ static inline char* EmitLiteral(char* op,
}
static inline char* EmitCopyLessThan64(char* op, size_t offset, int len) {
- DCHECK_LE(len, 64);
- DCHECK_GE(len, 4);
- DCHECK_LT(offset, 65536);
+ assert(len <= 64);
+ assert(len >= 4);
+ assert(offset < 65536);
if ((len < 12) && (offset < 2048)) {
size_t len_minus_4 = len - 4;
assert(len_minus_4 < 8); // Must fit in 3 bits
- *op++ = COPY_1_BYTE_OFFSET | ((len_minus_4) << 2) | ((offset >> 8) << 5);
+ *op++ = COPY_1_BYTE_OFFSET + ((len_minus_4) << 2) + ((offset >> 8) << 5);
*op++ = offset & 0xff;
} else {
- *op++ = COPY_2_BYTE_OFFSET | ((len-1) << 2);
+ *op++ = COPY_2_BYTE_OFFSET + ((len-1) << 2);
LittleEndian::Store16(op, offset);
op += 2;
}
@@ -253,8 +254,6 @@ uint16* WorkingMemory::GetHashTable(size_t input_size, int* table_size) {
while (htsize < kMaxHashTableSize && htsize < input_size) {
htsize <<= 1;
}
- CHECK_EQ(0, htsize & (htsize - 1)) << ": must be power of two";
- CHECK_LE(htsize, kMaxHashTableSize) << ": hash table too large";
uint16* table;
if (htsize <= ARRAYSIZE(small_table_)) {
@@ -294,8 +293,8 @@ static inline EightBytesReference GetEightBytesAt(const char* ptr) {
}
static inline uint32 GetUint32AtOffset(uint64 v, int offset) {
- DCHECK_GE(offset, 0);
- DCHECK_LE(offset, 4);
+ assert(offset >= 0);
+ assert(offset <= 4);
return v >> (LittleEndian::IsLittleEndian() ? 8 * offset : 32 - 8 * offset);
}
@@ -308,8 +307,8 @@ static inline EightBytesReference GetEightBytesAt(const char* ptr) {
}
static inline uint32 GetUint32AtOffset(const char* v, int offset) {
- DCHECK_GE(offset, 0);
- DCHECK_LE(offset, 4);
+ assert(offset >= 0);
+ assert(offset <= 4);
return UNALIGNED_LOAD32(v + offset);
}
@@ -334,10 +333,10 @@ char* CompressFragment(const char* input,
const int table_size) {
// "ip" is the input pointer, and "op" is the output pointer.
const char* ip = input;
- CHECK_LE(input_size, kBlockSize);
- CHECK_EQ(table_size & (table_size - 1), 0) << ": table must be power of two";
+ assert(input_size <= kBlockSize);
+ assert((table_size & (table_size - 1)) == 0); // table must be power of two
const int shift = 32 - Bits::Log2Floor(table_size);
- DCHECK_EQ(static_cast<int>(kuint32max >> shift), table_size - 1);
+ assert(static_cast<int>(kuint32max >> shift) == table_size - 1);
const char* ip_end = input + input_size;
const char* base_ip = ip;
// Bytes in [next_emit, ip) will be emitted as literal bytes. Or
@@ -349,7 +348,7 @@ char* CompressFragment(const char* input,
const char* ip_limit = input + input_size - kInputMarginBytes;
for (uint32 next_hash = Hash(++ip, shift); ; ) {
- DCHECK_LT(next_emit, ip);
+ assert(next_emit < ip);
// The body of this loop calls EmitLiteral once and then EmitCopy one or
// more times. (The exception is that when we're close to exhausting
// the input we goto emit_remainder.)
@@ -382,7 +381,7 @@ char* CompressFragment(const char* input,
do {
ip = next_ip;
uint32 hash = next_hash;
- DCHECK_EQ(hash, Hash(ip, shift));
+ assert(hash == Hash(ip, shift));
uint32 bytes_between_hash_lookups = skip++ >> 5;
next_ip = ip + bytes_between_hash_lookups;
if (PREDICT_FALSE(next_ip > ip_limit)) {
@@ -390,8 +389,8 @@ char* CompressFragment(const char* input,
}
next_hash = Hash(next_ip, shift);
candidate = base_ip + table[hash];
- DCHECK_GE(candidate, base_ip);
- DCHECK_LT(candidate, ip);
+ assert(candidate >= base_ip);
+ assert(candidate < ip);
table[hash] = ip - base_ip;
} while (PREDICT_TRUE(UNALIGNED_LOAD32(ip) !=
@@ -400,7 +399,7 @@ char* CompressFragment(const char* input,
// Step 2: A 4-byte match has been found. We'll later see if more
// than 4 bytes match. But, prior to the match, input
// bytes [next_emit, ip) are unmatched. Emit them as "literal bytes."
- DCHECK_LE(next_emit + 16, ip_end);
+ assert(next_emit + 16 <= ip_end);
op = EmitLiteral(op, next_emit, ip - next_emit, true);
// Step 3: Call EmitCopy, and then see if another EmitCopy could
@@ -421,7 +420,7 @@ char* CompressFragment(const char* input,
int matched = 4 + FindMatchLength(candidate + 4, ip + 4, ip_end);
ip += matched;
size_t offset = base - candidate;
- DCHECK_EQ(0, memcmp(base, candidate, matched));
+ assert(0 == memcmp(base, candidate, matched));
op = EmitCopy(op, offset, matched);
// We could immediately start working at ip now, but to improve
// compression we first update table[Hash(ip - 1, ...)].
@@ -471,21 +470,26 @@ char* CompressFragment(const char* input,
// bool Append(const char* ip, size_t length);
// bool AppendFromSelf(uint32 offset, size_t length);
//
-// // The difference between TryFastAppend and Append is that TryFastAppend
-// // is allowed to read up to <available> bytes from the input buffer,
-// // whereas Append is allowed to read <length>.
+// // The rules for how TryFastAppend differs from Append are somewhat
+// // convoluted:
// //
-// // Also, TryFastAppend is allowed to return false, declining the append,
-// // without it being a fatal error -- just "return false" would be
-// // a perfectly legal implementation of TryFastAppend. The intention
-// // is for TryFastAppend to allow a fast path in the common case of
-// // a small append.
+// // - TryFastAppend is allowed to decline (return false) at any
+// // time, for any reason -- just "return false" would be
+// // a perfectly legal implementation of TryFastAppend.
+// // The intention is for TryFastAppend to allow a fast path
+// // in the common case of a small append.
+// // - TryFastAppend is allowed to read up to <available> bytes
+// // from the input buffer, whereas Append is allowed to read
+// // <length>. However, if it returns true, it must leave
+// // at least five (kMaximumTagLength) bytes in the input buffer
+// // afterwards, so that there is always enough space to read the
+// // next tag without checking for a refill.
+// // - TryFastAppend must always return decline (return false)
+// // if <length> is 61 or more, as in this case the literal length is not
+// // decoded fully. In practice, this should not be a big problem,
+// // as it is unlikely that one would implement a fast path accepting
+// // this much data.
// //
-// // NOTE(user): TryFastAppend must always return decline (return false)
-// // if <length> is 61 or more, as in this case the literal length is not
-// // decoded fully. In practice, this should not be a big problem,
-// // as it is unlikely that one would implement a fast path accepting
-// // this much data.
// bool TryFastAppend(const char* ip, size_t available, size_t length);
// };
@@ -554,9 +558,9 @@ static uint16 MakeEntry(unsigned int extra,
unsigned int len,
unsigned int copy_offset) {
// Check that all of the fields fit within the allocated space
- DCHECK_EQ(extra, extra & 0x7); // At most 3 bits
- DCHECK_EQ(copy_offset, copy_offset & 0x7); // At most 3 bits
- DCHECK_EQ(len, len & 0x7f); // At most 7 bits
+ assert(extra == (extra & 0x7)); // At most 3 bits
+ assert(copy_offset == (copy_offset & 0x7)); // At most 3 bits
+ assert(len == (len & 0x7f)); // At most 7 bits
return len | (copy_offset << 8) | (extra << 11);
}
@@ -614,9 +618,15 @@ static void ComputeTable() {
}
// Check that each entry was initialized exactly once.
- CHECK_EQ(assigned, 256);
+ if (assigned != 256) {
+ fprintf(stderr, "ComputeTable: assigned only %d of 256\n", assigned);
+ abort();
+ }
for (int i = 0; i < 256; i++) {
- CHECK_NE(dst[i], 0xffff);
+ if (dst[i] == 0xffff) {
+ fprintf(stderr, "ComputeTable: did not assign byte %d\n", i);
+ abort();
+ }
}
if (FLAGS_snappy_dump_decompression_table) {
@@ -631,7 +641,11 @@ static void ComputeTable() {
// Check that computed table matched recorded table
for (int i = 0; i < 256; i++) {
- CHECK_EQ(dst[i], char_table[i]);
+ if (dst[i] != char_table[i]) {
+ fprintf(stderr, "ComputeTable: byte %d: computed (%x), expect (%x)\n",
+ i, static_cast<int>(dst[i]), static_cast<int>(char_table[i]));
+ abort();
+ }
}
}
#endif /* !NDEBUG */
@@ -644,7 +658,7 @@ class SnappyDecompressor {
const char* ip_limit_; // Points just past buffered bytes
uint32 peeked_; // Bytes peeked from reader (need to skip)
bool eof_; // Hit end of input without an error?
- char scratch_[5]; // Temporary buffer for PeekFast() boundaries
+ char scratch_[kMaximumTagLength]; // See RefillTag().
// Ensure that all of the tag metadata for the next tag is available
// in [ip_..ip_limit_-1]. Also ensures that [ip,ip+4] is readable even
@@ -676,7 +690,7 @@ class SnappyDecompressor {
// On succcess, stores the length in *result and returns true.
// On failure, returns false.
bool ReadUncompressedLength(uint32* result) {
- DCHECK(ip_ == NULL); // Must not have read anything yet
+ assert(ip_ == NULL); // Must not have read anything yet
// Length is encoded in 1..5 bytes
*result = 0;
uint32 shift = 0;
@@ -707,7 +721,7 @@ class SnappyDecompressor {
// scope to optimize the <ip_limit_ - ip> expression based on the local
// context, which overall increases speed.
#define MAYBE_REFILL() \
- if (ip_limit_ - ip < 5) { \
+ if (ip_limit_ - ip < kMaximumTagLength) { \
ip_ = ip; \
if (!RefillTag()) return; \
ip = ip_; \
@@ -720,9 +734,11 @@ class SnappyDecompressor {
if ((c & 0x3) == LITERAL) {
size_t literal_length = (c >> 2) + 1u;
if (writer->TryFastAppend(ip, ip_limit_ - ip, literal_length)) {
- DCHECK_LT(literal_length, 61);
+ assert(literal_length < 61);
ip += literal_length;
- MAYBE_REFILL();
+ // NOTE(user): There is no MAYBE_REFILL() here, as TryFastAppend()
+ // will not return true unless there's already at least five spare
+ // bytes in addition to the literal.
continue;
}
if (PREDICT_FALSE(literal_length >= 61)) {
@@ -787,11 +803,11 @@ bool SnappyDecompressor::RefillTag() {
}
// Read the tag character
- DCHECK_LT(ip, ip_limit_);
+ assert(ip < ip_limit_);
const unsigned char c = *(reinterpret_cast<const unsigned char*>(ip));
const uint32 entry = char_table[c];
const uint32 needed = (entry >> 11) + 1; // +1 byte for 'c'
- DCHECK_LE(needed, sizeof(scratch_));
+ assert(needed <= sizeof(scratch_));
// Read more bytes from reader if needed
uint32 nbuf = ip_limit_ - ip;
@@ -812,10 +828,10 @@ bool SnappyDecompressor::RefillTag() {
nbuf += to_add;
reader_->Skip(to_add);
}
- DCHECK_EQ(nbuf, needed);
+ assert(nbuf == needed);
ip_ = scratch_;
ip_limit_ = scratch_ + needed;
- } else if (nbuf < 5) {
+ } else if (nbuf < kMaximumTagLength) {
// Have enough bytes, but move into scratch_ so that we do not
// read past end of input
memmove(scratch_, ip, nbuf);
@@ -831,27 +847,18 @@ bool SnappyDecompressor::RefillTag() {
}
template <typename Writer>
-static bool InternalUncompress(Source* r,
- Writer* writer,
- uint32 max_len) {
+static bool InternalUncompress(Source* r, Writer* writer) {
// Read the uncompressed length from the front of the compressed input
SnappyDecompressor decompressor(r);
uint32 uncompressed_len = 0;
if (!decompressor.ReadUncompressedLength(&uncompressed_len)) return false;
- return InternalUncompressAllTags(
- &decompressor, writer, uncompressed_len, max_len);
+ return InternalUncompressAllTags(&decompressor, writer, uncompressed_len);
}
template <typename Writer>
static bool InternalUncompressAllTags(SnappyDecompressor* decompressor,
Writer* writer,
- uint32 uncompressed_len,
- uint32 max_len) {
- // Protect against possible DoS attack
- if (static_cast<uint64>(uncompressed_len) > max_len) {
- return false;
- }
-
+ uint32 uncompressed_len) {
writer->SetExpectedLength(uncompressed_len);
// Process the entire input
@@ -880,7 +887,7 @@ size_t Compress(Source* reader, Sink* writer) {
// Get next block to compress (without copying if possible)
size_t fragment_size;
const char* fragment = reader->Peek(&fragment_size);
- DCHECK_NE(fragment_size, 0) << ": premature end of input";
+ assert(fragment_size != 0); // premature end of input
const size_t num_to_read = min(N, kBlockSize);
size_t bytes_read = fragment_size;
@@ -907,11 +914,11 @@ size_t Compress(Source* reader, Sink* writer) {
bytes_read += n;
reader->Skip(n);
}
- DCHECK_EQ(bytes_read, num_to_read);
+ assert(bytes_read == num_to_read);
fragment = scratch;
fragment_size = num_to_read;
}
- DCHECK_EQ(fragment_size, num_to_read);
+ assert(fragment_size == num_to_read);
// Get encoding table for compression
int table_size;
@@ -946,6 +953,183 @@ size_t Compress(Source* reader, Sink* writer) {
}
// -----------------------------------------------------------------------
+// IOVec interfaces
+// -----------------------------------------------------------------------
+
+// A type that writes to an iovec.
+// Note that this is not a "ByteSink", but a type that matches the
+// Writer template argument to SnappyDecompressor::DecompressAllTags().
+class SnappyIOVecWriter {
+ private:
+ const struct iovec* output_iov_;
+ const size_t output_iov_count_;
+
+ // We are currently writing into output_iov_[curr_iov_index_].
+ int curr_iov_index_;
+
+ // Bytes written to output_iov_[curr_iov_index_] so far.
+ size_t curr_iov_written_;
+
+ // Total bytes decompressed into output_iov_ so far.
+ size_t total_written_;
+
+ // Maximum number of bytes that will be decompressed into output_iov_.
+ size_t output_limit_;
+
+ inline char* GetIOVecPointer(int index, size_t offset) {
+ return reinterpret_cast<char*>(output_iov_[index].iov_base) +
+ offset;
+ }
+
+ public:
+ // Does not take ownership of iov. iov must be valid during the
+ // entire lifetime of the SnappyIOVecWriter.
+ inline SnappyIOVecWriter(const struct iovec* iov, size_t iov_count)
+ : output_iov_(iov),
+ output_iov_count_(iov_count),
+ curr_iov_index_(0),
+ curr_iov_written_(0),
+ total_written_(0),
+ output_limit_(-1) {
+ }
+
+ inline void SetExpectedLength(size_t len) {
+ output_limit_ = len;
+ }
+
+ inline bool CheckLength() const {
+ return total_written_ == output_limit_;
+ }
+
+ inline bool Append(const char* ip, size_t len) {
+ if (total_written_ + len > output_limit_) {
+ return false;
+ }
+
+ while (len > 0) {
+ assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len);
+ if (curr_iov_written_ >= output_iov_[curr_iov_index_].iov_len) {
+ // This iovec is full. Go to the next one.
+ if (curr_iov_index_ + 1 >= output_iov_count_) {
+ return false;
+ }
+ curr_iov_written_ = 0;
+ ++curr_iov_index_;
+ }
+
+ const size_t to_write = std::min(
+ len, output_iov_[curr_iov_index_].iov_len - curr_iov_written_);
+ memcpy(GetIOVecPointer(curr_iov_index_, curr_iov_written_),
+ ip,
+ to_write);
+ curr_iov_written_ += to_write;
+ total_written_ += to_write;
+ ip += to_write;
+ len -= to_write;
+ }
+
+ return true;
+ }
+
+ inline bool TryFastAppend(const char* ip, size_t available, size_t len) {
+ const size_t space_left = output_limit_ - total_written_;
+ if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16 &&
+ output_iov_[curr_iov_index_].iov_len - curr_iov_written_ >= 16) {
+ // Fast path, used for the majority (about 95%) of invocations.
+ char* ptr = GetIOVecPointer(curr_iov_index_, curr_iov_written_);
+ UnalignedCopy64(ip, ptr);
+ UnalignedCopy64(ip + 8, ptr + 8);
+ curr_iov_written_ += len;
+ total_written_ += len;
+ return true;
+ }
+
+ return false;
+ }
+
+ inline bool AppendFromSelf(size_t offset, size_t len) {
+ if (offset > total_written_ || offset == 0) {
+ return false;
+ }
+ const size_t space_left = output_limit_ - total_written_;
+ if (len > space_left) {
+ return false;
+ }
+
+ // Locate the iovec from which we need to start the copy.
+ int from_iov_index = curr_iov_index_;
+ size_t from_iov_offset = curr_iov_written_;
+ while (offset > 0) {
+ if (from_iov_offset >= offset) {
+ from_iov_offset -= offset;
+ break;
+ }
+
+ offset -= from_iov_offset;
+ --from_iov_index;
+ assert(from_iov_index >= 0);
+ from_iov_offset = output_iov_[from_iov_index].iov_len;
+ }
+
+ // Copy <len> bytes starting from the iovec pointed to by from_iov_index to
+ // the current iovec.
+ while (len > 0) {
+ assert(from_iov_index <= curr_iov_index_);
+ if (from_iov_index != curr_iov_index_) {
+ const size_t to_copy = std::min(
+ output_iov_[from_iov_index].iov_len - from_iov_offset,
+ len);
+ Append(GetIOVecPointer(from_iov_index, from_iov_offset), to_copy);
+ len -= to_copy;
+ if (len > 0) {
+ ++from_iov_index;
+ from_iov_offset = 0;
+ }
+ } else {
+ assert(curr_iov_written_ <= output_iov_[curr_iov_index_].iov_len);
+ size_t to_copy = std::min(output_iov_[curr_iov_index_].iov_len -
+ curr_iov_written_,
+ len);
+ if (to_copy == 0) {
+ // This iovec is full. Go to the next one.
+ if (curr_iov_index_ + 1 >= output_iov_count_) {
+ return false;
+ }
+ ++curr_iov_index_;
+ curr_iov_written_ = 0;
+ continue;
+ }
+ if (to_copy > len) {
+ to_copy = len;
+ }
+ IncrementalCopy(GetIOVecPointer(from_iov_index, from_iov_offset),
+ GetIOVecPointer(curr_iov_index_, curr_iov_written_),
+ to_copy);
+ curr_iov_written_ += to_copy;
+ from_iov_offset += to_copy;
+ total_written_ += to_copy;
+ len -= to_copy;
+ }
+ }
+
+ return true;
+ }
+
+};
+
+bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
+ const struct iovec* iov, size_t iov_cnt) {
+ ByteArraySource reader(compressed, compressed_length);
+ return RawUncompressToIOVec(&reader, iov, iov_cnt);
+}
+
+bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
+ size_t iov_cnt) {
+ SnappyIOVecWriter output(iov, iov_cnt);
+ return InternalUncompress(compressed, &output);
+}
+
+// -----------------------------------------------------------------------
// Flat array interfaces
// -----------------------------------------------------------------------
@@ -986,7 +1170,7 @@ class SnappyArrayWriter {
inline bool TryFastAppend(const char* ip, size_t available, size_t len) {
char* op = op_;
const size_t space_left = op_limit_ - op;
- if (len <= 16 && available >= 16 && space_left >= 16) {
+ if (len <= 16 && available >= 16 + kMaximumTagLength && space_left >= 16) {
// Fast path, used for the majority (about 95%) of invocations.
UnalignedCopy64(ip, op);
UnalignedCopy64(ip + 8, op + 8);
@@ -1001,7 +1185,16 @@ class SnappyArrayWriter {
char* op = op_;
const size_t space_left = op_limit_ - op;
- if (op - base_ <= offset - 1u) { // -1u catches offset==0
+ // Check if we try to append from before the start of the buffer.
+ // Normally this would just be a check for "produced < offset",
+ // but "produced <= offset - 1u" is equivalent for every case
+ // except the one where offset==0, where the right side will wrap around
+ // to a very big number. This is convenient, as offset==0 is another
+ // invalid case that we also want to catch, so that we do not go
+ // into an infinite loop.
+ assert(op >= base_);
+ size_t produced = op - base_;
+ if (produced <= offset - 1u) {
return false;
}
if (len <= 16 && offset >= 8 && space_left >= 16) {
@@ -1031,7 +1224,7 @@ bool RawUncompress(const char* compressed, size_t n, char* uncompressed) {
bool RawUncompress(Source* compressed, char* uncompressed) {
SnappyArrayWriter output(uncompressed);
- return InternalUncompress(compressed, &output, kuint32max);
+ return InternalUncompress(compressed, &output);
}
bool Uncompress(const char* compressed, size_t n, string* uncompressed) {
@@ -1039,9 +1232,9 @@ bool Uncompress(const char* compressed, size_t n, string* uncompressed) {
if (!GetUncompressedLength(compressed, n, &ulength)) {
return false;
}
- // Protect against possible DoS attack
- if ((static_cast<uint64>(ulength) + uncompressed->size()) >
- uncompressed->max_size()) {
+ // On 32-bit builds: max_size() < kuint32max. Check for that instead
+ // of crashing (e.g., consider externally specified compressed data).
+ if (ulength > uncompressed->max_size()) {
return false;
}
STLStringResizeUninitialized(uncompressed, ulength);
@@ -1071,7 +1264,9 @@ class SnappyDecompressionValidator {
return false;
}
inline bool AppendFromSelf(size_t offset, size_t len) {
- if (produced_ <= offset - 1u) return false; // -1u catches offset==0
+ // See SnappyArrayWriter::AppendFromSelf for an explanation of
+ // the "offset - 1u" trick.
+ if (produced_ <= offset - 1u) return false;
produced_ += len;
return produced_ <= expected_;
}
@@ -1080,7 +1275,7 @@ class SnappyDecompressionValidator {
bool IsValidCompressedBuffer(const char* compressed, size_t n) {
ByteArraySource reader(compressed, n);
SnappyDecompressionValidator writer;
- return InternalUncompress(&reader, &writer, kuint32max);
+ return InternalUncompress(&reader, &writer);
}
void RawCompress(const char* input,
diff --git a/thirdparty/snappy/snappy.h b/thirdparty/snappy/snappy.h
index 8c2075fe..e879e794 100644
--- a/thirdparty/snappy/snappy.h
+++ b/thirdparty/snappy/snappy.h
@@ -56,6 +56,13 @@ namespace snappy {
// number of bytes written.
size_t Compress(Source* source, Sink* sink);
+ // Find the uncompressed length of the given stream, as given by the header.
+ // Note that the true length could deviate from this; the stream could e.g.
+ // be truncated.
+ //
+ // Also note that this leaves "*source" in a state that is unsuitable for
+ // further operations, such as RawUncompress(). You will need to rewind
+ // or recreate the source yourself before attempting any further calls.
bool GetUncompressedLength(Source* source, uint32* result);
// ------------------------------------------------------------------------
@@ -117,6 +124,28 @@ namespace snappy {
// returns false if the message is corrupted and could not be decrypted
bool RawUncompress(Source* compressed, char* uncompressed);
+ // Given data in "compressed[0..compressed_length-1]" generated by
+ // calling the Snappy::Compress routine, this routine
+ // stores the uncompressed data to the iovec "iov". The number of physical
+ // buffers in "iov" is given by iov_cnt and their cumulative size
+ // must be at least GetUncompressedLength(compressed). The individual buffers
+ // in "iov" must not overlap with each other.
+ //
+ // returns false if the message is corrupted and could not be decrypted
+ bool RawUncompressToIOVec(const char* compressed, size_t compressed_length,
+ const struct iovec* iov, size_t iov_cnt);
+
+ // Given data from the byte source 'compressed' generated by calling
+ // the Snappy::Compress routine, this routine stores the uncompressed
+ // data to the iovec "iov". The number of physical
+ // buffers in "iov" is given by iov_cnt and their cumulative size
+ // must be at least GetUncompressedLength(compressed). The individual buffers
+ // in "iov" must not overlap with each other.
+ //
+ // returns false if the message is corrupted and could not be decrypted
+ bool RawUncompressToIOVec(Source* compressed, const struct iovec* iov,
+ size_t iov_cnt);
+
// Returns the maximal size of the compressed representation of
// input data that is "source_bytes" bytes in length;
size_t MaxCompressedLength(size_t source_bytes);
@@ -135,20 +164,20 @@ namespace snappy {
bool IsValidCompressedBuffer(const char* compressed,
size_t compressed_length);
- // *** DO NOT CHANGE THE VALUE OF kBlockSize ***
+ // The size of a compression block. Note that many parts of the compression
+ // code assumes that kBlockSize <= 65536; in particular, the hash table
+ // can only store 16-bit offsets, and EmitCopy() also assumes the offset
+ // is 65535 bytes or less. Note also that if you change this, it will
+ // affect the framing format (see framing_format.txt).
//
- // New Compression code chops up the input into blocks of at most
- // the following size. This ensures that back-references in the
- // output never cross kBlockSize block boundaries. This can be
- // helpful in implementing blocked decompression. However the
- // decompression code should not rely on this guarantee since older
- // compression code may not obey it.
- static const int kBlockLog = 15;
+ // Note that there might be older data around that is compressed with larger
+ // block sizes, so the decompression code should not rely on the
+ // non-existence of long backreferences.
+ static const int kBlockLog = 16;
static const size_t kBlockSize = 1 << kBlockLog;
static const int kMaxHashTableBits = 14;
static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits;
-
} // end namespace snappy
diff --git a/thirdparty/snappy/snappy_unittest.cc b/thirdparty/snappy/snappy_unittest.cc
index f3b9c83c..be7bba6b 100644
--- a/thirdparty/snappy/snappy_unittest.cc
+++ b/thirdparty/snappy/snappy_unittest.cc
@@ -492,6 +492,46 @@ static int VerifyString(const string& input) {
}
+static void VerifyIOVec(const string& input) {
+ string compressed;
+ DataEndingAtUnreadablePage i(input);
+ const size_t written = snappy::Compress(i.data(), i.size(), &compressed);
+ CHECK_EQ(written, compressed.size());
+ CHECK_LE(compressed.size(),
+ snappy::MaxCompressedLength(input.size()));
+ CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
+
+ // Try uncompressing into an iovec containing a random number of entries
+ // ranging from 1 to 10.
+ char* buf = new char[input.size()];
+ ACMRandom rnd(input.size());
+ int num = rnd.Next() % 10 + 1;
+ if (input.size() < num) {
+ num = input.size();
+ }
+ struct iovec* iov = new iovec[num];
+ int used_so_far = 0;
+ for (int i = 0; i < num; ++i) {
+ iov[i].iov_base = buf + used_so_far;
+ if (i == num - 1) {
+ iov[i].iov_len = input.size() - used_so_far;
+ } else {
+ // Randomly choose to insert a 0 byte entry.
+ if (rnd.OneIn(5)) {
+ iov[i].iov_len = 0;
+ } else {
+ iov[i].iov_len = rnd.Uniform(input.size());
+ }
+ }
+ used_so_far += iov[i].iov_len;
+ }
+ CHECK(snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, num));
+ CHECK(!memcmp(buf, input.data(), input.size()));
+ delete[] iov;
+ delete[] buf;
+}
+
// Test that data compressed by a compressor that does not
// obey block sizes is uncompressed properly.
static void VerifyNonBlockedCompression(const string& input) {
@@ -542,8 +582,11 @@ static int Verify(const string& input) {
VerifyNonBlockedCompression(input);
+ VerifyIOVec(input);
if (!input.empty()) {
- VerifyNonBlockedCompression(Expand(input));
+ const string expanded = Expand(input);
+ VerifyNonBlockedCompression(expanded);
+ VerifyIOVec(input);
}
@@ -611,7 +654,8 @@ TYPED_TEST(CorruptedTest, VerifyCorrupted) {
// try reading stuff in from a bad file.
for (int i = 1; i <= 3; ++i) {
- string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str());
+ string data = ReadTestDataFile(StringPrintf("baddata%d.snappy", i).c_str(),
+ 0);
string uncmp;
// check that we don't return a crazy length
size_t ulen;
@@ -663,7 +707,7 @@ static void AppendCopy(string* dst, int offset, int length) {
}
length -= to_copy;
- if ((to_copy < 12) && (offset < 2048)) {
+ if ((to_copy >= 4) && (to_copy < 12) && (offset < 2048)) {
assert(to_copy-4 < 8); // Must fit in 3 bits
dst->push_back(1 | ((to_copy-4) << 2) | ((offset >> 8) << 5));
dst->push_back(offset & 0xff);
@@ -768,10 +812,123 @@ TEST(Snappy, FourByteOffset) {
string uncompressed;
CHECK(snappy::IsValidCompressedBuffer(compressed.data(), compressed.size()));
- CHECK(snappy::Uncompress(compressed.data(), compressed.size(), &uncompressed));
+ CHECK(snappy::Uncompress(compressed.data(), compressed.size(),
+ &uncompressed));
CHECK_EQ(uncompressed, src);
}
+TEST(Snappy, IOVecEdgeCases) {
+ // Test some tricky edge cases in the iovec output that are not necessarily
+ // exercised by random tests.
+
+ // Our output blocks look like this initially (the last iovec is bigger
+ // than depicted):
+ // [ ] [ ] [ ] [ ] [ ]
+ static const int kLengths[] = { 2, 1, 4, 8, 128 };
+
+ struct iovec iov[ARRAYSIZE(kLengths)];
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ iov[i].iov_base = new char[kLengths[i]];
+ iov[i].iov_len = kLengths[i];
+ }
+
+ string compressed;
+ Varint::Append32(&compressed, 22);
+
+ // A literal whose output crosses three blocks.
+ // [ab] [c] [123 ] [ ] [ ]
+ AppendLiteral(&compressed, "abc123");
+
+ // A copy whose output crosses two blocks (source and destination
+ // segments marked).
+ // [ab] [c] [1231] [23 ] [ ]
+ // ^--^ --
+ AppendCopy(&compressed, 3, 3);
+
+ // A copy where the input is, at first, in the block before the output:
+ //
+ // [ab] [c] [1231] [231231 ] [ ]
+ // ^--- ^---
+ // Then during the copy, the pointers move such that the input and
+ // output pointers are in the same block:
+ //
+ // [ab] [c] [1231] [23123123] [ ]
+ // ^- ^-
+ // And then they move again, so that the output pointer is no longer
+ // in the same block as the input pointer:
+ // [ab] [c] [1231] [23123123] [123 ]
+ // ^-- ^--
+ AppendCopy(&compressed, 6, 9);
+
+ // Finally, a copy where the input is from several blocks back,
+ // and it also crosses three blocks:
+ //
+ // [ab] [c] [1231] [23123123] [123b ]
+ // ^ ^
+ // [ab] [c] [1231] [23123123] [123bc ]
+ // ^ ^
+ // [ab] [c] [1231] [23123123] [123bc12 ]
+ // ^- ^-
+ AppendCopy(&compressed, 17, 4);
+
+ CHECK(snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, ARRAYSIZE(iov)));
+ CHECK_EQ(0, memcmp(iov[0].iov_base, "ab", 2));
+ CHECK_EQ(0, memcmp(iov[1].iov_base, "c", 1));
+ CHECK_EQ(0, memcmp(iov[2].iov_base, "1231", 4));
+ CHECK_EQ(0, memcmp(iov[3].iov_base, "23123123", 8));
+ CHECK_EQ(0, memcmp(iov[4].iov_base, "123bc12", 7));
+
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ delete[] reinterpret_cast<char *>(iov[i].iov_base);
+ }
+}
+
+TEST(Snappy, IOVecLiteralOverflow) {
+ static const int kLengths[] = { 3, 4 };
+
+ struct iovec iov[ARRAYSIZE(kLengths)];
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ iov[i].iov_base = new char[kLengths[i]];
+ iov[i].iov_len = kLengths[i];
+ }
+
+ string compressed;
+ Varint::Append32(&compressed, 8);
+
+ AppendLiteral(&compressed, "12345678");
+
+ CHECK(!snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, ARRAYSIZE(iov)));
+
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ delete[] reinterpret_cast<char *>(iov[i].iov_base);
+ }
+}
+
+TEST(Snappy, IOVecCopyOverflow) {
+ static const int kLengths[] = { 3, 4 };
+
+ struct iovec iov[ARRAYSIZE(kLengths)];
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ iov[i].iov_base = new char[kLengths[i]];
+ iov[i].iov_len = kLengths[i];
+ }
+
+ string compressed;
+ Varint::Append32(&compressed, 8);
+
+ AppendLiteral(&compressed, "123");
+ AppendCopy(&compressed, 3, 5);
+
+ CHECK(!snappy::RawUncompressToIOVec(
+ compressed.data(), compressed.size(), iov, ARRAYSIZE(iov)));
+
+ for (int i = 0; i < ARRAYSIZE(kLengths); ++i) {
+ delete[] reinterpret_cast<char *>(iov[i].iov_base);
+ }
+}
+
static bool CheckUncompressedLength(const string& compressed,
size_t* ulength) {
@@ -971,18 +1128,18 @@ TEST(Snappy, FindMatchLengthRandom) {
static void CompressFile(const char* fname) {
string fullinput;
- File::ReadFileToStringOrDie(fname, &fullinput);
+ file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
string compressed;
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);
- File::WriteStringToFileOrDie(compressed,
- string(fname).append(".comp").c_str());
+ file::SetContents(string(fname).append(".comp"), compressed, file::Defaults())
+ .CheckSuccess();
}
static void UncompressFile(const char* fname) {
string fullinput;
- File::ReadFileToStringOrDie(fname, &fullinput);
+ file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
size_t uncompLength;
CHECK(CheckUncompressedLength(fullinput, &uncompLength));
@@ -991,13 +1148,13 @@ static void UncompressFile(const char* fname) {
uncompressed.resize(uncompLength);
CHECK(snappy::Uncompress(fullinput.data(), fullinput.size(), &uncompressed));
- File::WriteStringToFileOrDie(uncompressed,
- string(fname).append(".uncomp").c_str());
+ file::SetContents(string(fname).append(".uncomp"), uncompressed,
+ file::Defaults()).CheckSuccess();
}
static void MeasureFile(const char* fname) {
string fullinput;
- File::ReadFileToStringOrDie(fname, &fullinput);
+ file::GetContents(fname, &fullinput, file::Defaults()).CheckSuccess();
printf("%-40s :\n", fname);
int start_len = (FLAGS_start_len < 0) ? fullinput.size() : FLAGS_start_len;
@@ -1030,25 +1187,20 @@ static void MeasureFile(const char* fname) {
static struct {
const char* label;
const char* filename;
+ size_t size_limit;
} files[] = {
- { "html", "html" },
- { "urls", "urls.10K" },
- { "jpg", "house.jpg" },
- { "pdf", "mapreduce-osdi-1.pdf" },
- { "html4", "html_x_4" },
- { "cp", "cp.html" },
- { "c", "fields.c" },
- { "lsp", "grammar.lsp" },
- { "xls", "kennedy.xls" },
- { "txt1", "alice29.txt" },
- { "txt2", "asyoulik.txt" },
- { "txt3", "lcet10.txt" },
- { "txt4", "plrabn12.txt" },
- { "bin", "ptt5" },
- { "sum", "sum" },
- { "man", "xargs.1" },
- { "pb", "geo.protodata" },
- { "gaviota", "kppkn.gtb" },
+ { "html", "html", 0 },
+ { "urls", "urls.10K", 0 },
+ { "jpg", "fireworks.jpeg", 0 },
+ { "jpg_200", "fireworks.jpeg", 200 },
+ { "pdf", "paper-100k.pdf", 0 },
+ { "html4", "html_x_4", 0 },
+ { "txt1", "alice29.txt", 0 },
+ { "txt2", "asyoulik.txt", 0 },
+ { "txt3", "lcet10.txt", 0 },
+ { "txt4", "plrabn12.txt", 0 },
+ { "pb", "geo.protodata", 0 },
+ { "gaviota", "kppkn.gtb", 0 },
};
static void BM_UFlat(int iters, int arg) {
@@ -1057,7 +1209,8 @@ static void BM_UFlat(int iters, int arg) {
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
- string contents = ReadTestDataFile(files[arg].filename);
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents);
@@ -1074,7 +1227,7 @@ static void BM_UFlat(int iters, int arg) {
delete[] dst;
}
-BENCHMARK(BM_UFlat)->DenseRange(0, 17);
+BENCHMARK(BM_UFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
static void BM_UValidate(int iters, int arg) {
StopBenchmarkTiming();
@@ -1082,7 +1235,8 @@ static void BM_UValidate(int iters, int arg) {
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
- string contents = ReadTestDataFile(files[arg].filename);
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
string zcontents;
snappy::Compress(contents.data(), contents.size(), &zcontents);
@@ -1098,6 +1252,52 @@ static void BM_UValidate(int iters, int arg) {
}
BENCHMARK(BM_UValidate)->DenseRange(0, 4);
+static void BM_UIOVec(int iters, int arg) {
+ StopBenchmarkTiming();
+
+ // Pick file to process based on "arg"
+ CHECK_GE(arg, 0);
+ CHECK_LT(arg, ARRAYSIZE(files));
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
+
+ string zcontents;
+ snappy::Compress(contents.data(), contents.size(), &zcontents);
+
+ // Uncompress into an iovec containing ten entries.
+ const int kNumEntries = 10;
+ struct iovec iov[kNumEntries];
+ char *dst = new char[contents.size()];
+ int used_so_far = 0;
+ for (int i = 0; i < kNumEntries; ++i) {
+ iov[i].iov_base = dst + used_so_far;
+ if (used_so_far == contents.size()) {
+ iov[i].iov_len = 0;
+ continue;
+ }
+
+ if (i == kNumEntries - 1) {
+ iov[i].iov_len = contents.size() - used_so_far;
+ } else {
+ iov[i].iov_len = contents.size() / kNumEntries;
+ }
+ used_so_far += iov[i].iov_len;
+ }
+
+ SetBenchmarkBytesProcessed(static_cast<int64>(iters) *
+ static_cast<int64>(contents.size()));
+ SetBenchmarkLabel(files[arg].label);
+ StartBenchmarkTiming();
+ while (iters-- > 0) {
+ CHECK(snappy::RawUncompressToIOVec(zcontents.data(), zcontents.size(), iov,
+ kNumEntries));
+ }
+ StopBenchmarkTiming();
+
+ delete[] dst;
+}
+BENCHMARK(BM_UIOVec)->DenseRange(0, 4);
+
static void BM_ZFlat(int iters, int arg) {
StopBenchmarkTiming();
@@ -1105,7 +1305,8 @@ static void BM_ZFlat(int iters, int arg) {
// Pick file to process based on "arg"
CHECK_GE(arg, 0);
CHECK_LT(arg, ARRAYSIZE(files));
- string contents = ReadTestDataFile(files[arg].filename);
+ string contents = ReadTestDataFile(files[arg].filename,
+ files[arg].size_limit);
char* dst = new char[snappy::MaxCompressedLength(contents.size())];
@@ -1126,7 +1327,7 @@ static void BM_ZFlat(int iters, int arg) {
files[arg].label, contents.size(), zsize);
delete[] dst;
}
-BENCHMARK(BM_ZFlat)->DenseRange(0, 17);
+BENCHMARK(BM_ZFlat)->DenseRange(0, ARRAYSIZE(files) - 1);
} // namespace snappy
@@ -1134,7 +1335,6 @@ BENCHMARK(BM_ZFlat)->DenseRange(0, 17);
int main(int argc, char** argv) {
InitGoogle(argv[0], &argc, &argv, true);
- File::Init();
RunSpecifiedBenchmarks();
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-04 08:32:55 +0000