From b5f2ee344ef2914ca141608107c571ec0c28c6a6 Mon Sep 17 00:00:00 2001 From: Zack Rusin Date: Wed, 24 Aug 2011 21:21:38 -0400 Subject: [PATCH] Export SnappyFile class to its own file. --- CMakeLists.txt | 1 + trace_file.cpp | 170 +----------------------------------------- trace_file.hpp | 39 ---------- trace_parser.cpp | 1 + trace_snappyfile.cpp | 173 +++++++++++++++++++++++++++++++++++++++++++ trace_snappyfile.hpp | 56 ++++++++++++++ trace_writer.cpp | 2 +- 7 files changed, 234 insertions(+), 208 deletions(-) create mode 100644 trace_snappyfile.cpp create mode 100644 trace_snappyfile.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 39ffde5..e840f21 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -181,6 +181,7 @@ endif (WIN32) add_library (common trace_file.cpp + trace_snappyfile.cpp trace_model.cpp trace_parser.cpp trace_writer.cpp diff --git a/trace_file.cpp b/trace_file.cpp index 2a9705c..758ecd7 100644 --- a/trace_file.cpp +++ b/trace_file.cpp @@ -1,10 +1,11 @@ #include "trace_file.hpp" +#include "trace_snappyfile.hpp" + #include #include #include -#include #include "os.hpp" @@ -12,8 +13,6 @@ using namespace Trace; -#define SNAPPY_BYTE1 'a' -#define SNAPPY_BYTE2 't' File::File(const std::string &filename, File::Mode mode) @@ -172,168 +171,3 @@ void ZLibFile::rawFlush(FlushType type) { gzflush(m_gzFile, Z_SYNC_FLUSH); } - -SnappyFile::SnappyFile(const std::string &filename, - File::Mode mode) - : File(), - m_cache(0), - m_cachePtr(0), - m_cacheSize(0) -{ - m_compressedCache = new char[SNAPPY_CHUNK_SIZE]; -} - -SnappyFile::~SnappyFile() -{ - delete [] m_compressedCache; -} - -bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode) -{ - std::ios_base::openmode fmode = std::fstream::binary; - if (mode == File::Write) { - fmode |= (std::fstream::out | std::fstream::trunc); - createCache(SNAPPY_CHUNK_SIZE); - } else if (mode == File::Read) { - fmode |= std::fstream::in; - } - - m_stream.open(filename.c_str(), fmode); - - //read in the initial buffer if we're reading - if (m_stream.is_open() && mode == File::Read) { - // read the snappy file identifier - unsigned char byte1, byte2; - m_stream >> byte1; - m_stream >> byte2; - assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2); - - flushCache(); - } else if (m_stream.is_open() && mode == File::Write) { - // write the snappy file identifier - m_stream << SNAPPY_BYTE1; - m_stream << SNAPPY_BYTE2; - } - return m_stream.is_open(); -} - -bool SnappyFile::rawWrite(const void *buffer, int length) -{ - if (freeCacheSize() > length) { - memcpy(m_cachePtr, buffer, length); - m_cachePtr += length; - } else if (freeCacheSize() == length) { - memcpy(m_cachePtr, buffer, length); - m_cachePtr += length; - flushCache(); - } else { - int sizeToWrite = length; - - while (sizeToWrite >= freeCacheSize()) { - int endSize = freeCacheSize(); - int offset = length - sizeToWrite; - memcpy(m_cachePtr, (char*)buffer + offset, endSize); - sizeToWrite -= endSize; - m_cachePtr += endSize; - flushCache(); - } - if (sizeToWrite) { - int offset = length - sizeToWrite; - memcpy(m_cachePtr, (char*)buffer + offset, sizeToWrite); - m_cachePtr += sizeToWrite; - } - } - - return true; -} - -bool SnappyFile::rawRead(void *buffer, int length) -{ - if (m_stream.eof()) { - return false; - } - if (freeCacheSize() > length) { - memcpy(buffer, m_cachePtr, length); - m_cachePtr += length; - } else if (freeCacheSize() == length) { - memcpy(buffer, m_cachePtr, length); - m_cachePtr += length; - flushCache(); - } else { - int sizeToRead = length; - int offset = 0; - while (sizeToRead) { - int chunkSize = std::min(freeCacheSize(), sizeToRead); - offset = length - sizeToRead; - memcpy((char*)buffer + offset, m_cachePtr, chunkSize); - m_cachePtr += chunkSize; - sizeToRead -= chunkSize; - if (sizeToRead > 0) - flushCache(); - if (!m_cacheSize) - break; - } - } - - return true; -} - -int SnappyFile::rawGetc() -{ - int c = 0; - if (!rawRead(&c, 1)) - return -1; - return c; -} - -void SnappyFile::rawClose() -{ - flushCache(); - m_stream.close(); - delete [] m_cache; - m_cache = NULL; - m_cachePtr = NULL; -} - -void SnappyFile::rawFlush(FlushType type) -{ - if (type == FlushDeep) { - flushCache(); - } - m_stream.flush(); -} - -void SnappyFile::flushCache() -{ - if (m_mode == File::Write) { - size_t compressedLength; - - ::snappy::RawCompress(m_cache, SNAPPY_CHUNK_SIZE - freeCacheSize(), - m_compressedCache, &compressedLength); - - m_stream << compressedLength; - m_stream.write(m_compressedCache, compressedLength); - m_cachePtr = m_cache; - } else if (m_mode == File::Read) { - if (m_stream.eof()) - return; - //assert(m_cachePtr == m_cache + m_cacheSize); - size_t compressedLength; - m_stream >> compressedLength; - m_stream.read((char*)m_compressedCache, compressedLength); - ::snappy::GetUncompressedLength(m_compressedCache, compressedLength, - &m_cacheSize); - if (m_cache) - delete [] m_cache; - createCache(m_cacheSize); - ::snappy::RawUncompress(m_compressedCache, compressedLength, - m_cache); - } -} - -void SnappyFile::createCache(size_t size) -{ - m_cache = new char[size]; - m_cachePtr = m_cache; - m_cacheSize = size; -} diff --git a/trace_file.hpp b/trace_file.hpp index fb61b98..f1a63f2 100644 --- a/trace_file.hpp +++ b/trace_file.hpp @@ -66,45 +66,6 @@ private: void *m_gzFile; }; -namespace snappy { - class File; -} - -#define SNAPPY_CHUNK_SIZE (1 * 1024 * 1024) -class SnappyFile : public File { -public: - SnappyFile(const std::string &filename = std::string(), - File::Mode mode = File::Read); - virtual ~SnappyFile(); - -protected: - virtual bool rawOpen(const std::string &filename, File::Mode mode); - virtual bool rawWrite(const void *buffer, int length); - virtual bool rawRead(void *buffer, int length); - virtual int rawGetc(); - virtual void rawClose(); - virtual void rawFlush(FlushType type); - -private: - inline int freeCacheSize() const - { - if (m_cacheSize > 0) - return m_cacheSize - (m_cachePtr - m_cache); - else - return 0; - } - void flushCache(); - void createCache(size_t size); -private: - std::fstream m_stream; - char *m_cache; - char *m_cachePtr; - size_t m_cacheSize; - - char *m_compressedCache; -}; - - } #endif diff --git a/trace_parser.cpp b/trace_parser.cpp index f4073e7..47d5a7a 100644 --- a/trace_parser.cpp +++ b/trace_parser.cpp @@ -29,6 +29,7 @@ #include #include "trace_file.hpp" +#include "trace_snappyfile.hpp" #include "trace_parser.hpp" diff --git a/trace_snappyfile.cpp b/trace_snappyfile.cpp new file mode 100644 index 0000000..457f7e9 --- /dev/null +++ b/trace_snappyfile.cpp @@ -0,0 +1,173 @@ +#include "trace_snappyfile.hpp" + +#include + +#include +#include + +using namespace Trace; + +SnappyFile::SnappyFile(const std::string &filename, + File::Mode mode) + : File(), + m_cache(0), + m_cachePtr(0), + m_cacheSize(0) +{ + m_compressedCache = new char[SNAPPY_CHUNK_SIZE]; +} + +SnappyFile::~SnappyFile() +{ + delete [] m_compressedCache; +} + +bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode) +{ + std::ios_base::openmode fmode = std::fstream::binary; + if (mode == File::Write) { + fmode |= (std::fstream::out | std::fstream::trunc); + createCache(SNAPPY_CHUNK_SIZE); + } else if (mode == File::Read) { + fmode |= std::fstream::in; + } + + m_stream.open(filename.c_str(), fmode); + + //read in the initial buffer if we're reading + if (m_stream.is_open() && mode == File::Read) { + // read the snappy file identifier + unsigned char byte1, byte2; + m_stream >> byte1; + m_stream >> byte2; + assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2); + + flushCache(); + } else if (m_stream.is_open() && mode == File::Write) { + // write the snappy file identifier + m_stream << SNAPPY_BYTE1; + m_stream << SNAPPY_BYTE2; + } + return m_stream.is_open(); +} + +bool SnappyFile::rawWrite(const void *buffer, int length) +{ + if (freeCacheSize() > length) { + memcpy(m_cachePtr, buffer, length); + m_cachePtr += length; + } else if (freeCacheSize() == length) { + memcpy(m_cachePtr, buffer, length); + m_cachePtr += length; + flushCache(); + } else { + int sizeToWrite = length; + + while (sizeToWrite >= freeCacheSize()) { + int endSize = freeCacheSize(); + int offset = length - sizeToWrite; + memcpy(m_cachePtr, (char*)buffer + offset, endSize); + sizeToWrite -= endSize; + m_cachePtr += endSize; + flushCache(); + } + if (sizeToWrite) { + int offset = length - sizeToWrite; + memcpy(m_cachePtr, (char*)buffer + offset, sizeToWrite); + m_cachePtr += sizeToWrite; + } + } + + return true; +} + +bool SnappyFile::rawRead(void *buffer, int length) +{ + if (m_stream.eof()) { + return false; + } + if (freeCacheSize() > length) { + memcpy(buffer, m_cachePtr, length); + m_cachePtr += length; + } else if (freeCacheSize() == length) { + memcpy(buffer, m_cachePtr, length); + m_cachePtr += length; + flushCache(); + } else { + int sizeToRead = length; + int offset = 0; + while (sizeToRead) { + int chunkSize = std::min(freeCacheSize(), sizeToRead); + offset = length - sizeToRead; + memcpy((char*)buffer + offset, m_cachePtr, chunkSize); + m_cachePtr += chunkSize; + sizeToRead -= chunkSize; + if (sizeToRead > 0) + flushCache(); + if (!m_cacheSize) + break; + } + } + + return true; +} + +int SnappyFile::rawGetc() +{ + int c = 0; + if (!rawRead(&c, 1)) + return -1; + return c; +} + +void SnappyFile::rawClose() +{ + flushCache(); + m_stream.close(); + delete [] m_cache; + m_cache = NULL; + m_cachePtr = NULL; +} + +void SnappyFile::rawFlush(FlushType type) +{ + if (type == FlushDeep) { + flushCache(); + } + m_stream.flush(); +} + +void SnappyFile::flushCache() +{ + if (m_mode == File::Write) { + size_t compressedLength; + + ::snappy::RawCompress(m_cache, SNAPPY_CHUNK_SIZE - freeCacheSize(), + m_compressedCache, &compressedLength); + + m_stream << compressedLength; + m_stream.write(m_compressedCache, compressedLength); + m_cachePtr = m_cache; + } else if (m_mode == File::Read) { + if (m_stream.eof()) + return; + //assert(m_cachePtr == m_cache + m_cacheSize); + size_t compressedLength; + m_stream >> compressedLength; + m_stream.read((char*)m_compressedCache, compressedLength); + ::snappy::GetUncompressedLength(m_compressedCache, compressedLength, + &m_cacheSize); + if (m_cache) + delete [] m_cache; + createCache(m_cacheSize); + ::snappy::RawUncompress(m_compressedCache, compressedLength, + m_cache); + } +} + +void SnappyFile::createCache(size_t size) +{ + m_cache = new char[size]; + m_cachePtr = m_cache; + m_cacheSize = size; +} diff --git a/trace_snappyfile.hpp b/trace_snappyfile.hpp new file mode 100644 index 0000000..5489069 --- /dev/null +++ b/trace_snappyfile.hpp @@ -0,0 +1,56 @@ +#ifndef TRACE_SNAPPYFILE_HPP +#define TRACE_SNAPPYFILE_HPP + +#include "trace_file.hpp" + +#include +#include + +namespace snappy { + class File; +} + +namespace Trace { + +#define SNAPPY_CHUNK_SIZE (1 * 1024 * 1024) + +#define SNAPPY_BYTE1 'a' +#define SNAPPY_BYTE2 't' + + +class SnappyFile : public File { +public: + SnappyFile(const std::string &filename = std::string(), + File::Mode mode = File::Read); + virtual ~SnappyFile(); + +protected: + virtual bool rawOpen(const std::string &filename, File::Mode mode); + virtual bool rawWrite(const void *buffer, int length); + virtual bool rawRead(void *buffer, int length); + virtual int rawGetc(); + virtual void rawClose(); + virtual void rawFlush(FlushType type); + +private: + inline int freeCacheSize() const + { + if (m_cacheSize > 0) + return m_cacheSize - (m_cachePtr - m_cache); + else + return 0; + } + void flushCache(); + void createCache(size_t size); +private: + std::fstream m_stream; + char *m_cache; + char *m_cachePtr; + size_t m_cacheSize; + + char *m_compressedCache; +}; + +} + +#endif // TRACE_SNAPPYFILE_HPP diff --git a/trace_writer.cpp b/trace_writer.cpp index ce5df5c..5a5f1f7 100644 --- a/trace_writer.cpp +++ b/trace_writer.cpp @@ -32,7 +32,7 @@ #include "os.hpp" #include "trace_writer.hpp" -#include "trace_file.hpp" +#include "trace_snappyfile.hpp" #include "trace_format.hpp" -- 2.43.0