From a77f2925f342f27c8164213cdf0a8086f97e3765 Mon Sep 17 00:00:00 2001 From: Jose Fonseca Date: Wed, 23 Mar 2016 12:59:48 +0000 Subject: common: Support reading Brotli compressed traces. --- common/CMakeLists.txt | 3 + common/trace_file.hpp | 1 + common/trace_file_brotli.cpp | 150 +++++++++++++++++++++++++++++++++++++++++++ common/trace_file_read.cpp | 4 +- 4 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 common/trace_file_brotli.cpp diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index be72f5f7..49a635f7 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -1,5 +1,6 @@ include_directories ( ${CMAKE_SOURCE_DIR}/guids + ${CMAKE_SOURCE_DIR}/thirdparty ) if (WIN32) @@ -17,6 +18,7 @@ add_convenience_library (common trace_file.cpp trace_file_read.cpp trace_file_zlib.cpp + trace_file_brotli.cpp trace_file_snappy.cpp trace_model.cpp trace_parser.cpp @@ -38,6 +40,7 @@ add_convenience_library (common target_link_libraries (common guids ${LIBBACKTRACE_LIBRARIES} + brotli_dec_bundled ) if (WIN32) target_link_libraries (common diff --git a/common/trace_file.hpp b/common/trace_file.hpp index 1eda7ac3..26b525c3 100644 --- a/common/trace_file.hpp +++ b/common/trace_file.hpp @@ -45,6 +45,7 @@ public: public: static File *createZLib(void); + static File *createBrotli(void); static File *createSnappy(void); static File *createForRead(const char *filename); public: diff --git a/common/trace_file_brotli.cpp b/common/trace_file_brotli.cpp new file mode 100644 index 00000000..e99b010e --- /dev/null +++ b/common/trace_file_brotli.cpp @@ -0,0 +1,150 @@ +/************************************************************************** + * + * Copyright 2016 VMware, Inc. + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + **************************************************************************/ + + +#include "trace_file.hpp" + + +#include +#include + +#include + +#include + +#include "os.hpp" + + +using namespace trace; + + +class BrotliFile : public File { +public: + BrotliFile(void); + virtual ~BrotliFile(); + +protected: + virtual bool rawOpen(const char *filename) override; + virtual size_t rawRead(void *buffer, size_t length) override; + virtual int rawGetc(void) override; + virtual void rawClose(void) override; + virtual bool rawSkip(size_t length) override; + virtual int rawPercentRead(void) override; +private: + BrotliState state; + std::ifstream m_stream; + static const size_t kFileBufferSize = 65536; + uint8_t input[kFileBufferSize]; + const uint8_t* next_in; + size_t available_in; +}; + +BrotliFile::BrotliFile(void) +{ + BrotliStateInit(&state); + available_in = 0; + next_in = input; +} + +BrotliFile::~BrotliFile() +{ + close(); + BrotliStateCleanup(&state); +} + +bool BrotliFile::rawOpen(const char *filename) +{ + std::ios_base::openmode fmode = std::fstream::binary + | std::fstream::in; + + m_stream.open(filename, fmode); + return m_stream.is_open(); +} + +size_t BrotliFile::rawRead(void *buffer, size_t length) +{ + uint8_t* output = (uint8_t *)buffer; + size_t total_out; + size_t available_out = length; + uint8_t* next_out = output; + + while (true) { + BrotliResult result; + result = BrotliDecompressStream(&available_in, &next_in, + &available_out, &next_out, &total_out, + &state); + if (result == BROTLI_RESULT_NEEDS_MORE_INPUT) { + if (m_stream.fail()) { + break; + } + m_stream.read((char *)input, kFileBufferSize); + available_in = kFileBufferSize; + if (m_stream.fail()) { + available_in = m_stream.gcount(); + if (!available_in) { + break; + } + } + next_in = input; + } else { + assert(result == BROTLI_RESULT_NEEDS_MORE_OUTPUT || + result == BROTLI_RESULT_SUCCESS); + break; + } + } + + assert(next_out - output <= length); + + return next_out - output; +} + +int BrotliFile::rawGetc() +{ + unsigned char c; + if (rawRead(&c, 1) != 1) { + return -1; + } + return c; +} + +void BrotliFile::rawClose() +{ + m_stream.close(); +} + +bool BrotliFile::rawSkip(size_t) +{ + return false; +} + +int BrotliFile::rawPercentRead(void) +{ + return 0; +} + + +File * File::createBrotli(void) { + return new BrotliFile; +} diff --git a/common/trace_file_read.cpp b/common/trace_file_read.cpp index c4198550..fb303aa8 100644 --- a/common/trace_file_read.cpp +++ b/common/trace_file_read.cpp @@ -53,8 +53,8 @@ File::createForRead(const char *filename) } else if (byte1 == 0x1f && byte2 == 0x8b) { file = File::createZLib(); } else { - os::log("error: %s: unkwnown compression\n", filename); - return NULL; + // XXX: Brotli has no magic header + file = File::createBrotli(); } if (!file) { return NULL; -- cgit v1.2.3