From: José Fonseca Date: Wed, 26 Oct 2011 22:37:01 +0000 (+0100) Subject: Re-organize the Trace::File code. X-Git-Url: https://git.cworth.org/git?a=commitdiff_plain;h=4159a6196b23dae0be22dde17a85783fe4036517;p=apitrace Re-organize the Trace::File code. This should allow stop linking the zlib code from the wrappers. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index db3d9ab..3499fe4 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,7 +208,8 @@ endif () add_library (common STATIC common/trace_file.cpp - common/trace_snappyfile.cpp + common/trace_file_zlib.cpp + common/trace_file_snappy.cpp common/trace_model.cpp common/trace_parser.cpp common/trace_writer.cpp diff --git a/common/trace_file.cpp b/common/trace_file.cpp index 8eb3bd4..93c1fb2 100644 --- a/common/trace_file.cpp +++ b/common/trace_file.cpp @@ -26,8 +26,6 @@ #include "trace_file.hpp" -#include "trace_snappyfile.hpp" - #include #include @@ -62,112 +60,3 @@ void File::setCurrentOffset(const File::Offset &offset) assert(0); } -bool File::isZLibCompressed(const std::string &filename) -{ - std::fstream stream(filename.c_str(), - std::fstream::binary | std::fstream::in); - if (!stream.is_open()) - return false; - - unsigned char byte1, byte2; - stream >> byte1; - stream >> byte2; - stream.close(); - - return (byte1 == 0x1f && byte2 == 0x8b); -} - - -bool File::isSnappyCompressed(const std::string &filename) -{ - std::fstream stream(filename.c_str(), - std::fstream::binary | std::fstream::in); - if (!stream.is_open()) - return false; - - unsigned char byte1, byte2; - stream >> byte1; - stream >> byte2; - stream.close(); - - return (byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2); -} - -ZLibFile::ZLibFile(const std::string &filename, - File::Mode mode) - : File(filename, mode), - m_gzFile(NULL) -{ -} - -ZLibFile::~ZLibFile() -{ -} - -bool ZLibFile::rawOpen(const std::string &filename, File::Mode mode) -{ - m_gzFile = gzopen(filename.c_str(), - (mode == File::Write) ? "wb" : "rb"); - - if (mode == File::Read && m_gzFile) { - //XXX: unfortunately zlib doesn't support - // SEEK_END or we could've done: - //m_endOffset = gzseek(m_gzFile, 0, SEEK_END); - //gzrewind(m_gzFile); - gz_state *state = (gz_state *)m_gzFile; - off_t loc = lseek(state->fd, 0, SEEK_CUR); - m_endOffset = lseek(state->fd, 0, SEEK_END); - lseek(state->fd, loc, SEEK_SET); - } - - return m_gzFile != NULL; -} - -bool ZLibFile::rawWrite(const void *buffer, size_t length) -{ - return gzwrite(m_gzFile, buffer, length) != -1; -} - -bool ZLibFile::rawRead(void *buffer, size_t length) -{ - return gzread(m_gzFile, buffer, length) != -1; -} - -int ZLibFile::rawGetc() -{ - return gzgetc(m_gzFile); -} - -void ZLibFile::rawClose() -{ - if (m_gzFile) { - gzclose(m_gzFile); - m_gzFile = NULL; - } -} - -void ZLibFile::rawFlush() -{ - gzflush(m_gzFile, Z_SYNC_FLUSH); -} - -File::Offset ZLibFile::currentOffset() -{ - return File::Offset(gztell(m_gzFile)); -} - -bool ZLibFile::supportsOffsets() const -{ - return false; -} - -bool ZLibFile::rawSkip(size_t) -{ - return false; -} - -int ZLibFile::rawPercentRead() -{ - gz_state *state = (gz_state *)m_gzFile; - return 100 * (lseek(state->fd, 0, SEEK_CUR) / m_endOffset); -} diff --git a/common/trace_file.hpp b/common/trace_file.hpp index 4b1b70d..0105a2a 100644 --- a/common/trace_file.hpp +++ b/common/trace_file.hpp @@ -51,6 +51,8 @@ public: public: static bool isZLibCompressed(const std::string &filename); static bool isSnappyCompressed(const std::string &filename); + static File *createZLib(void); + static File *createSnappy(void); public: File(const std::string &filename = std::string(), File::Mode mode = File::Read); @@ -162,28 +164,6 @@ inline bool File::skip(size_t length) return rawSkip(length); } -class ZLibFile : public File { -public: - ZLibFile(const std::string &filename = std::string(), - File::Mode mode = File::Read); - virtual ~ZLibFile(); - - - virtual bool supportsOffsets() const; - virtual File::Offset currentOffset(); -protected: - virtual bool rawOpen(const std::string &filename, File::Mode mode); - virtual bool rawWrite(const void *buffer, size_t length); - virtual bool rawRead(void *buffer, size_t length); - virtual int rawGetc(); - virtual void rawClose(); - virtual void rawFlush(); - virtual bool rawSkip(size_t length); - virtual int rawPercentRead(); -private: - void *m_gzFile; - double m_endOffset; -}; inline bool operator<(const File::Offset &one, const File::Offset &two) diff --git a/common/trace_file_snappy.cpp b/common/trace_file_snappy.cpp new file mode 100644 index 0000000..60f003b --- /dev/null +++ b/common/trace_file_snappy.cpp @@ -0,0 +1,422 @@ +/************************************************************************** + * + * Copyright 2011 Zack Rusin + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + **************************************************************************/ + + +/* + * Snappy file format. + * ------------------- + * + * Snappy at its core is just a compressoin algorithm so we're + * creating a new file format which uses snappy compression + * to hold the trace data. + * + * The file is composed of a number of chunks, they are: + * chunk { + * uint32 - specifying the length of the compressed data + * compressed data, in little endian + * } + * File can contain any number of such chunks. + * The default size of an uncompressed chunk is specified in + * SNAPPY_CHUNK_SIZE. + * + * Note: + * Currently the default size for a a to-be-compressed data is + * 1mb, meaning that the compressed data will be <= 1mb. + * The reason it's 1mb is because it seems + * to offer a pretty good compression/disk io speed ratio + * but that might change. + * + */ + + +#include + +#include + +#include +#include + +#include "trace_file.hpp" + + +#define SNAPPY_CHUNK_SIZE (1 * 1024 * 1024) + +#define SNAPPY_BYTE1 'a' +#define SNAPPY_BYTE2 't' + + +using namespace Trace; + + +class SnappyFile : public File { +public: + SnappyFile(const std::string &filename = std::string(), + File::Mode mode = File::Read); + virtual ~SnappyFile(); + + virtual bool supportsOffsets() const; + virtual File::Offset currentOffset(); + virtual void setCurrentOffset(const File::Offset &offset); +protected: + virtual bool rawOpen(const std::string &filename, File::Mode mode); + virtual bool rawWrite(const void *buffer, size_t length); + virtual bool rawRead(void *buffer, size_t length); + virtual int rawGetc(); + virtual void rawClose(); + virtual void rawFlush(); + virtual bool rawSkip(size_t length); + virtual int rawPercentRead(); + +private: + inline size_t usedCacheSize() const + { + assert(m_cachePtr >= m_cache); + return m_cachePtr - m_cache; + } + inline size_t freeCacheSize() const + { + assert(m_cacheSize >= usedCacheSize()); + if (m_cacheSize > 0) { + return m_cacheSize - usedCacheSize(); + } else { + return 0; + } + } + inline bool endOfData() const + { + return m_stream.eof() && freeCacheSize() == 0; + } + void flushWriteCache(); + void flushReadCache(size_t skipLength = 0); + void createCache(size_t size); + void writeCompressedLength(size_t length); + size_t readCompressedLength(); +private: + std::fstream m_stream; + char *m_cache; + char *m_cachePtr; + size_t m_cacheSize; + + char *m_compressedCache; + + File::Offset m_currentOffset; + std::streampos m_endPos; +}; + +SnappyFile::SnappyFile(const std::string &filename, + File::Mode mode) + : File(), + m_cache(0), + m_cachePtr(0), + m_cacheSize(0) +{ + size_t maxCompressedLength = + snappy::MaxCompressedLength(SNAPPY_CHUNK_SIZE); + m_compressedCache = new char[maxCompressedLength]; +} + +SnappyFile::~SnappyFile() +{ + delete [] m_compressedCache; + delete [] m_cache; +} + +bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode) +{ + std::ios_base::openmode fmode = std::fstream::binary; + if (mode == File::Write) { + fmode |= (std::fstream::out | std::fstream::trunc); + createCache(SNAPPY_CHUNK_SIZE); + } else if (mode == File::Read) { + fmode |= std::fstream::in; + } + + m_stream.open(filename.c_str(), fmode); + + //read in the initial buffer if we're reading + if (m_stream.is_open() && mode == File::Read) { + m_stream.seekg(0, std::ios::end); + m_endPos = m_stream.tellg(); + m_stream.seekg(0, std::ios::beg); + + // read the snappy file identifier + unsigned char byte1, byte2; + m_stream >> byte1; + m_stream >> byte2; + assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2); + + flushReadCache(); + } else if (m_stream.is_open() && mode == File::Write) { + // write the snappy file identifier + m_stream << SNAPPY_BYTE1; + m_stream << SNAPPY_BYTE2; + } + return m_stream.is_open(); +} + +bool SnappyFile::rawWrite(const void *buffer, size_t length) +{ + if (freeCacheSize() > length) { + memcpy(m_cachePtr, buffer, length); + m_cachePtr += length; + } else if (freeCacheSize() == length) { + memcpy(m_cachePtr, buffer, length); + m_cachePtr += length; + flushWriteCache(); + } else { + size_t sizeToWrite = length; + + while (sizeToWrite >= freeCacheSize()) { + size_t endSize = freeCacheSize(); + size_t offset = length - sizeToWrite; + memcpy(m_cachePtr, (const char*)buffer + offset, endSize); + sizeToWrite -= endSize; + m_cachePtr += endSize; + flushWriteCache(); + } + if (sizeToWrite) { + size_t offset = length - sizeToWrite; + memcpy(m_cachePtr, (const char*)buffer + offset, sizeToWrite); + m_cachePtr += sizeToWrite; + } + } + + return true; +} + +bool SnappyFile::rawRead(void *buffer, size_t length) +{ + if (endOfData()) { + return false; + } + + if (freeCacheSize() >= length) { + memcpy(buffer, m_cachePtr, length); + m_cachePtr += length; + } else { + size_t sizeToRead = length; + size_t offset = 0; + while (sizeToRead) { + size_t chunkSize = std::min(freeCacheSize(), sizeToRead); + offset = length - sizeToRead; + memcpy((char*)buffer + offset, m_cachePtr, chunkSize); + m_cachePtr += chunkSize; + sizeToRead -= chunkSize; + if (sizeToRead > 0) { + flushReadCache(); + } + if (!m_cacheSize) { + break; + } + } + } + + return true; +} + +int SnappyFile::rawGetc() +{ + int c = 0; + if (!rawRead(&c, 1)) + return -1; + return c; +} + +void SnappyFile::rawClose() +{ + if (m_mode == File::Write) { + flushWriteCache(); + } + m_stream.close(); + delete [] m_cache; + m_cache = NULL; + m_cachePtr = NULL; +} + +void SnappyFile::rawFlush() +{ + assert(m_mode == File::Write); + flushWriteCache(); + m_stream.flush(); +} + +void SnappyFile::flushWriteCache() +{ + size_t inputLength = usedCacheSize(); + + if (inputLength) { + size_t compressedLength; + + ::snappy::RawCompress(m_cache, inputLength, + m_compressedCache, &compressedLength); + + writeCompressedLength(compressedLength); + m_stream.write(m_compressedCache, compressedLength); + m_cachePtr = m_cache; + } + assert(m_cachePtr == m_cache); +} + +void SnappyFile::flushReadCache(size_t skipLength) +{ + //assert(m_cachePtr == m_cache + m_cacheSize); + m_currentOffset.chunk = m_stream.tellg(); + size_t compressedLength; + compressedLength = readCompressedLength(); + + if (compressedLength) { + m_stream.read((char*)m_compressedCache, compressedLength); + ::snappy::GetUncompressedLength(m_compressedCache, compressedLength, + &m_cacheSize); + createCache(m_cacheSize); + if (skipLength < m_cacheSize) { + ::snappy::RawUncompress(m_compressedCache, compressedLength, + m_cache); + } + } else { + createCache(0); + } +} + +void SnappyFile::createCache(size_t size) +{ + // TODO: only re-allocate if the current buffer is not big enough + + if (m_cache) { + delete [] m_cache; + } + + if (size) { + m_cache = new char[size]; + } else { + m_cache = NULL; + } + + m_cachePtr = m_cache; + m_cacheSize = size; +} + +void SnappyFile::writeCompressedLength(size_t length) +{ + unsigned char buf[4]; + buf[0] = length & 0xff; length >>= 8; + buf[1] = length & 0xff; length >>= 8; + buf[2] = length & 0xff; length >>= 8; + buf[3] = length & 0xff; length >>= 8; + assert(length == 0); + m_stream.write((const char *)buf, sizeof buf); +} + +size_t SnappyFile::readCompressedLength() +{ + unsigned char buf[4]; + size_t length; + m_stream.read((char *)buf, sizeof buf); + if (m_stream.fail()) { + length = 0; + } else { + length = (size_t)buf[0]; + length |= ((size_t)buf[1] << 8); + length |= ((size_t)buf[2] << 16); + length |= ((size_t)buf[3] << 24); + } + return length; +} + +bool SnappyFile::supportsOffsets() const +{ + return true; +} + +File::Offset SnappyFile::currentOffset() +{ + m_currentOffset.offsetInChunk = m_cachePtr - m_cache; + return m_currentOffset; +} + +void SnappyFile::setCurrentOffset(const File::Offset &offset) +{ + // to remove eof bit + m_stream.clear(); + // seek to the start of a chunk + m_stream.seekg(offset.chunk, std::ios::beg); + // load the chunk + flushReadCache(); + assert(m_cacheSize >= offset.offsetInChunk); + // seek within our cache to the correct location within the chunk + m_cachePtr = m_cache + offset.offsetInChunk; + +} + +bool SnappyFile::rawSkip(size_t length) +{ + if (endOfData()) { + return false; + } + + if (freeCacheSize() >= length) { + m_cachePtr += length; + } else { + size_t sizeToRead = length; + while (sizeToRead) { + size_t chunkSize = std::min(freeCacheSize(), sizeToRead); + m_cachePtr += chunkSize; + sizeToRead -= chunkSize; + if (sizeToRead > 0) { + flushReadCache(sizeToRead); + } + if (!m_cacheSize) { + break; + } + } + } + + return true; +} + +int SnappyFile::rawPercentRead() +{ + return 100 * (double(m_stream.tellg()) / double(m_endPos)); +} + + +File* File::createSnappy(void) { + return new SnappyFile; +} + +bool File::isSnappyCompressed(const std::string &filename) +{ + std::fstream stream(filename.c_str(), + std::fstream::binary | std::fstream::in); + if (!stream.is_open()) + return false; + + unsigned char byte1, byte2; + stream >> byte1; + stream >> byte2; + stream.close(); + + return (byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2); +} diff --git a/common/trace_file_zlib.cpp b/common/trace_file_zlib.cpp new file mode 100644 index 0000000..6982c1e --- /dev/null +++ b/common/trace_file_zlib.cpp @@ -0,0 +1,165 @@ +/************************************************************************** + * + * Copyright 2011 Zack Rusin + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + **************************************************************************/ + + +#include "trace_file.hpp" + + +#include +#include + +#include +#include + +#include "os.hpp" + +#include + + +using namespace Trace; + + +class ZLibFile : public File { +public: + ZLibFile(const std::string &filename = std::string(), + File::Mode mode = File::Read); + virtual ~ZLibFile(); + + + virtual bool supportsOffsets() const; + virtual File::Offset currentOffset(); +protected: + virtual bool rawOpen(const std::string &filename, File::Mode mode); + virtual bool rawWrite(const void *buffer, size_t length); + virtual bool rawRead(void *buffer, size_t length); + virtual int rawGetc(); + virtual void rawClose(); + virtual void rawFlush(); + virtual bool rawSkip(size_t length); + virtual int rawPercentRead(); +private: + void *m_gzFile; + double m_endOffset; +}; + +ZLibFile::ZLibFile(const std::string &filename, + File::Mode mode) + : File(filename, mode), + m_gzFile(NULL) +{ +} + +ZLibFile::~ZLibFile() +{ +} + +bool ZLibFile::rawOpen(const std::string &filename, File::Mode mode) +{ + m_gzFile = gzopen(filename.c_str(), + (mode == File::Write) ? "wb" : "rb"); + + if (mode == File::Read && m_gzFile) { + //XXX: unfortunately zlib doesn't support + // SEEK_END or we could've done: + //m_endOffset = gzseek(m_gzFile, 0, SEEK_END); + //gzrewind(m_gzFile); + gz_state *state = (gz_state *)m_gzFile; + off_t loc = lseek(state->fd, 0, SEEK_CUR); + m_endOffset = lseek(state->fd, 0, SEEK_END); + lseek(state->fd, loc, SEEK_SET); + } + + return m_gzFile != NULL; +} + +bool ZLibFile::rawWrite(const void *buffer, size_t length) +{ + return gzwrite(m_gzFile, buffer, length) != -1; +} + +bool ZLibFile::rawRead(void *buffer, size_t length) +{ + return gzread(m_gzFile, buffer, length) != -1; +} + +int ZLibFile::rawGetc() +{ + return gzgetc(m_gzFile); +} + +void ZLibFile::rawClose() +{ + if (m_gzFile) { + gzclose(m_gzFile); + m_gzFile = NULL; + } +} + +void ZLibFile::rawFlush() +{ + gzflush(m_gzFile, Z_SYNC_FLUSH); +} + +File::Offset ZLibFile::currentOffset() +{ + return File::Offset(gztell(m_gzFile)); +} + +bool ZLibFile::supportsOffsets() const +{ + return false; +} + +bool ZLibFile::rawSkip(size_t) +{ + return false; +} + +int ZLibFile::rawPercentRead() +{ + gz_state *state = (gz_state *)m_gzFile; + return 100 * (lseek(state->fd, 0, SEEK_CUR) / m_endOffset); +} + + +File * File::createZLib(void) { + return new ZLibFile; +} + +bool File::isZLibCompressed(const std::string &filename) +{ + std::fstream stream(filename.c_str(), + std::fstream::binary | std::fstream::in); + if (!stream.is_open()) + return false; + + unsigned char byte1, byte2; + stream >> byte1; + stream >> byte2; + stream.close(); + + return (byte1 == 0x1f && byte2 == 0x8b); +} + diff --git a/common/trace_parser.cpp b/common/trace_parser.cpp index 17f4a15..0a28f7a 100644 --- a/common/trace_parser.cpp +++ b/common/trace_parser.cpp @@ -29,7 +29,6 @@ #include #include "trace_file.hpp" -#include "trace_snappyfile.hpp" #include "trace_parser.hpp" @@ -54,9 +53,9 @@ Parser::~Parser() { bool Parser::open(const char *filename) { assert(!file); if (File::isZLibCompressed(filename)) { - file = new ZLibFile; + file = File::createZLib(); } else { - file = new SnappyFile; + file = File::createSnappy(); } if (!file->open(filename, File::Read)) { diff --git a/common/trace_snappyfile.cpp b/common/trace_snappyfile.cpp deleted file mode 100644 index 60711d2..0000000 --- a/common/trace_snappyfile.cpp +++ /dev/null @@ -1,338 +0,0 @@ -/************************************************************************** - * - * Copyright 2011 Zack Rusin - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - **************************************************************************/ - - -#include "trace_snappyfile.hpp" - -#include - -#include - -#include -#include - -using namespace Trace; - -/* - * Snappy file format. - * ------------------- - * - * Snappy at its core is just a compressoin algorithm so we're - * creating a new file format which uses snappy compression - * to hold the trace data. - * - * The file is composed of a number of chunks, they are: - * chunk { - * uint32 - specifying the length of the compressed data - * compressed data, in little endian - * } - * File can contain any number of such chunks. - * The default size of an uncompressed chunk is specified in - * SNAPPY_CHUNK_SIZE. - * - * Note: - * Currently the default size for a a to-be-compressed data is - * 1mb, meaning that the compressed data will be <= 1mb. - * The reason it's 1mb is because it seems - * to offer a pretty good compression/disk io speed ratio - * but that might change. - * - */ - -SnappyFile::SnappyFile(const std::string &filename, - File::Mode mode) - : File(), - m_cache(0), - m_cachePtr(0), - m_cacheSize(0) -{ - size_t maxCompressedLength = - snappy::MaxCompressedLength(SNAPPY_CHUNK_SIZE); - m_compressedCache = new char[maxCompressedLength]; -} - -SnappyFile::~SnappyFile() -{ - delete [] m_compressedCache; - delete [] m_cache; -} - -bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode) -{ - std::ios_base::openmode fmode = std::fstream::binary; - if (mode == File::Write) { - fmode |= (std::fstream::out | std::fstream::trunc); - createCache(SNAPPY_CHUNK_SIZE); - } else if (mode == File::Read) { - fmode |= std::fstream::in; - } - - m_stream.open(filename.c_str(), fmode); - - //read in the initial buffer if we're reading - if (m_stream.is_open() && mode == File::Read) { - m_stream.seekg(0, std::ios::end); - m_endPos = m_stream.tellg(); - m_stream.seekg(0, std::ios::beg); - - // read the snappy file identifier - unsigned char byte1, byte2; - m_stream >> byte1; - m_stream >> byte2; - assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2); - - flushReadCache(); - } else if (m_stream.is_open() && mode == File::Write) { - // write the snappy file identifier - m_stream << SNAPPY_BYTE1; - m_stream << SNAPPY_BYTE2; - } - return m_stream.is_open(); -} - -bool SnappyFile::rawWrite(const void *buffer, size_t length) -{ - if (freeCacheSize() > length) { - memcpy(m_cachePtr, buffer, length); - m_cachePtr += length; - } else if (freeCacheSize() == length) { - memcpy(m_cachePtr, buffer, length); - m_cachePtr += length; - flushWriteCache(); - } else { - size_t sizeToWrite = length; - - while (sizeToWrite >= freeCacheSize()) { - size_t endSize = freeCacheSize(); - size_t offset = length - sizeToWrite; - memcpy(m_cachePtr, (const char*)buffer + offset, endSize); - sizeToWrite -= endSize; - m_cachePtr += endSize; - flushWriteCache(); - } - if (sizeToWrite) { - size_t offset = length - sizeToWrite; - memcpy(m_cachePtr, (const char*)buffer + offset, sizeToWrite); - m_cachePtr += sizeToWrite; - } - } - - return true; -} - -bool SnappyFile::rawRead(void *buffer, size_t length) -{ - if (endOfData()) { - return false; - } - - if (freeCacheSize() >= length) { - memcpy(buffer, m_cachePtr, length); - m_cachePtr += length; - } else { - size_t sizeToRead = length; - size_t offset = 0; - while (sizeToRead) { - size_t chunkSize = std::min(freeCacheSize(), sizeToRead); - offset = length - sizeToRead; - memcpy((char*)buffer + offset, m_cachePtr, chunkSize); - m_cachePtr += chunkSize; - sizeToRead -= chunkSize; - if (sizeToRead > 0) { - flushReadCache(); - } - if (!m_cacheSize) { - break; - } - } - } - - return true; -} - -int SnappyFile::rawGetc() -{ - int c = 0; - if (!rawRead(&c, 1)) - return -1; - return c; -} - -void SnappyFile::rawClose() -{ - if (m_mode == File::Write) { - flushWriteCache(); - } - m_stream.close(); - delete [] m_cache; - m_cache = NULL; - m_cachePtr = NULL; -} - -void SnappyFile::rawFlush() -{ - assert(m_mode == File::Write); - flushWriteCache(); - m_stream.flush(); -} - -void SnappyFile::flushWriteCache() -{ - size_t inputLength = usedCacheSize(); - - if (inputLength) { - size_t compressedLength; - - ::snappy::RawCompress(m_cache, inputLength, - m_compressedCache, &compressedLength); - - writeCompressedLength(compressedLength); - m_stream.write(m_compressedCache, compressedLength); - m_cachePtr = m_cache; - } - assert(m_cachePtr == m_cache); -} - -void SnappyFile::flushReadCache(size_t skipLength) -{ - //assert(m_cachePtr == m_cache + m_cacheSize); - m_currentOffset.chunk = m_stream.tellg(); - size_t compressedLength; - compressedLength = readCompressedLength(); - - if (compressedLength) { - m_stream.read((char*)m_compressedCache, compressedLength); - ::snappy::GetUncompressedLength(m_compressedCache, compressedLength, - &m_cacheSize); - createCache(m_cacheSize); - if (skipLength < m_cacheSize) { - ::snappy::RawUncompress(m_compressedCache, compressedLength, - m_cache); - } - } else { - createCache(0); - } -} - -void SnappyFile::createCache(size_t size) -{ - // TODO: only re-allocate if the current buffer is not big enough - - if (m_cache) { - delete [] m_cache; - } - - if (size) { - m_cache = new char[size]; - } else { - m_cache = NULL; - } - - m_cachePtr = m_cache; - m_cacheSize = size; -} - -void SnappyFile::writeCompressedLength(size_t length) -{ - unsigned char buf[4]; - buf[0] = length & 0xff; length >>= 8; - buf[1] = length & 0xff; length >>= 8; - buf[2] = length & 0xff; length >>= 8; - buf[3] = length & 0xff; length >>= 8; - assert(length == 0); - m_stream.write((const char *)buf, sizeof buf); -} - -size_t SnappyFile::readCompressedLength() -{ - unsigned char buf[4]; - size_t length; - m_stream.read((char *)buf, sizeof buf); - if (m_stream.fail()) { - length = 0; - } else { - length = (size_t)buf[0]; - length |= ((size_t)buf[1] << 8); - length |= ((size_t)buf[2] << 16); - length |= ((size_t)buf[3] << 24); - } - return length; -} - -bool SnappyFile::supportsOffsets() const -{ - return true; -} - -File::Offset SnappyFile::currentOffset() -{ - m_currentOffset.offsetInChunk = m_cachePtr - m_cache; - return m_currentOffset; -} - -void SnappyFile::setCurrentOffset(const File::Offset &offset) -{ - // to remove eof bit - m_stream.clear(); - // seek to the start of a chunk - m_stream.seekg(offset.chunk, std::ios::beg); - // load the chunk - flushReadCache(); - assert(m_cacheSize >= offset.offsetInChunk); - // seek within our cache to the correct location within the chunk - m_cachePtr = m_cache + offset.offsetInChunk; - -} - -bool SnappyFile::rawSkip(size_t length) -{ - if (endOfData()) { - return false; - } - - if (freeCacheSize() >= length) { - m_cachePtr += length; - } else { - size_t sizeToRead = length; - while (sizeToRead) { - size_t chunkSize = std::min(freeCacheSize(), sizeToRead); - m_cachePtr += chunkSize; - sizeToRead -= chunkSize; - if (sizeToRead > 0) { - flushReadCache(sizeToRead); - } - if (!m_cacheSize) { - break; - } - } - } - - return true; -} - -int SnappyFile::rawPercentRead() -{ - return 100 * (double(m_stream.tellg()) / double(m_endPos)); -} diff --git a/common/trace_snappyfile.hpp b/common/trace_snappyfile.hpp deleted file mode 100644 index 33159ec..0000000 --- a/common/trace_snappyfile.hpp +++ /dev/null @@ -1,106 +0,0 @@ -/************************************************************************** - * - * Copyright 2011 Zack Rusin - * All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - **************************************************************************/ - - -#ifndef TRACE_SNAPPYFILE_HPP -#define TRACE_SNAPPYFILE_HPP - -#include - -#include "trace_file.hpp" - -#include -#include - -namespace snappy { - class File; -} - -namespace Trace { - -#define SNAPPY_CHUNK_SIZE (1 * 1024 * 1024) - -#define SNAPPY_BYTE1 'a' -#define SNAPPY_BYTE2 't' - - -class SnappyFile : public File { -public: - SnappyFile(const std::string &filename = std::string(), - File::Mode mode = File::Read); - virtual ~SnappyFile(); - - virtual bool supportsOffsets() const; - virtual File::Offset currentOffset(); - virtual void setCurrentOffset(const File::Offset &offset); -protected: - virtual bool rawOpen(const std::string &filename, File::Mode mode); - virtual bool rawWrite(const void *buffer, size_t length); - virtual bool rawRead(void *buffer, size_t length); - virtual int rawGetc(); - virtual void rawClose(); - virtual void rawFlush(); - virtual bool rawSkip(size_t length); - virtual int rawPercentRead(); - -private: - inline size_t usedCacheSize() const - { - assert(m_cachePtr >= m_cache); - return m_cachePtr - m_cache; - } - inline size_t freeCacheSize() const - { - assert(m_cacheSize >= usedCacheSize()); - if (m_cacheSize > 0) { - return m_cacheSize - usedCacheSize(); - } else { - return 0; - } - } - inline bool endOfData() const - { - return m_stream.eof() && freeCacheSize() == 0; - } - void flushWriteCache(); - void flushReadCache(size_t skipLength = 0); - void createCache(size_t size); - void writeCompressedLength(size_t length); - size_t readCompressedLength(); -private: - std::fstream m_stream; - char *m_cache; - char *m_cachePtr; - size_t m_cacheSize; - - char *m_compressedCache; - - File::Offset m_currentOffset; - std::streampos m_endPos; -}; - -} - -#endif // TRACE_SNAPPYFILE_HPP diff --git a/common/trace_writer.cpp b/common/trace_writer.cpp index 5a5f1f7..41f5e63 100644 --- a/common/trace_writer.cpp +++ b/common/trace_writer.cpp @@ -31,8 +31,8 @@ #include #include "os.hpp" +#include "trace_file.hpp" #include "trace_writer.hpp" -#include "trace_snappyfile.hpp" #include "trace_format.hpp" @@ -42,7 +42,7 @@ namespace Trace { Writer::Writer() : call_no(0) { - m_file = new Trace::SnappyFile; + m_file = File::createSnappy(); close(); }