1 /**************************************************************************
3 * Copyright 2011 Zack Rusin
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 **************************************************************************/
27 #include "trace_snappyfile.hpp"
35 using namespace Trace;
41 * Snappy at its core is just a compressoin algorithm so we're
42 * creating a new file format which uses snappy compression
43 * to hold the trace data.
45 * The file is composed of a number of chunks, they are:
47 * uint32 - specifying the length of the compressed data
50 * File can contain any number of such chunks.
51 * The default size of an uncompressed chunk is specified in
55 * Currently the default size for a a to-be-compressed data is
56 * 1mb, meaning that the compressed data will be <= 1mb.
57 * The reason it's 1mb is because it seems
58 * to offer a pretty good compression/disk io speed ratio
59 * but that might change.
63 SnappyFile::SnappyFile(const std::string &filename,
70 size_t maxCompressedLength =
71 snappy::MaxCompressedLength(SNAPPY_CHUNK_SIZE);
72 m_compressedCache = new char[maxCompressedLength];
75 SnappyFile::~SnappyFile()
77 delete [] m_compressedCache;
80 bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode)
82 std::ios_base::openmode fmode = std::fstream::binary;
83 if (mode == File::Write) {
84 fmode |= (std::fstream::out | std::fstream::trunc);
85 createCache(SNAPPY_CHUNK_SIZE);
86 } else if (mode == File::Read) {
87 fmode |= std::fstream::in;
90 m_stream.open(filename.c_str(), fmode);
92 //read in the initial buffer if we're reading
93 if (m_stream.is_open() && mode == File::Read) {
94 // read the snappy file identifier
95 unsigned char byte1, byte2;
98 assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2);
101 } else if (m_stream.is_open() && mode == File::Write) {
102 // write the snappy file identifier
103 m_stream << SNAPPY_BYTE1;
104 m_stream << SNAPPY_BYTE2;
106 return m_stream.is_open();
109 bool SnappyFile::rawWrite(const void *buffer, size_t length)
111 if (freeCacheSize() > length) {
112 memcpy(m_cachePtr, buffer, length);
113 m_cachePtr += length;
114 } else if (freeCacheSize() == length) {
115 memcpy(m_cachePtr, buffer, length);
116 m_cachePtr += length;
119 int sizeToWrite = length;
121 while (sizeToWrite >= freeCacheSize()) {
122 int endSize = freeCacheSize();
123 int offset = length - sizeToWrite;
124 memcpy(m_cachePtr, (const char*)buffer + offset, endSize);
125 sizeToWrite -= endSize;
126 m_cachePtr += endSize;
130 int offset = length - sizeToWrite;
131 memcpy(m_cachePtr, (const char*)buffer + offset, sizeToWrite);
132 m_cachePtr += sizeToWrite;
139 bool SnappyFile::rawRead(void *buffer, size_t length)
145 if (freeCacheSize() >= length) {
146 memcpy(buffer, m_cachePtr, length);
147 m_cachePtr += length;
149 size_t sizeToRead = length;
152 size_t chunkSize = std::min(freeCacheSize(), sizeToRead);
153 offset = length - sizeToRead;
154 memcpy((char*)buffer + offset, m_cachePtr, chunkSize);
155 m_cachePtr += chunkSize;
156 sizeToRead -= chunkSize;
167 int SnappyFile::rawGetc()
175 void SnappyFile::rawClose()
184 void SnappyFile::rawFlush()
190 void SnappyFile::flushCache()
192 if (m_mode == File::Write) {
193 size_t compressedLength;
195 ::snappy::RawCompress(m_cache, SNAPPY_CHUNK_SIZE - freeCacheSize(),
196 m_compressedCache, &compressedLength);
198 writeCompressedLength(compressedLength);
199 m_stream.write(m_compressedCache, compressedLength);
200 m_cachePtr = m_cache;
201 } else if (m_mode == File::Read) {
204 //assert(m_cachePtr == m_cache + m_cacheSize);
205 size_t compressedLength;
206 compressedLength = readCompressedLength();
207 m_stream.read((char*)m_compressedCache, compressedLength);
209 * The reason we peek here is because the last read will
210 * read all the way until the last character, but that will not
211 * trigger m_stream.eof() to be set, so by calling peek
212 * we assure that if we in fact have read the entire stream
213 * then the m_stream.eof() is always set.
216 ::snappy::GetUncompressedLength(m_compressedCache, compressedLength,
220 createCache(m_cacheSize);
221 ::snappy::RawUncompress(m_compressedCache, compressedLength,
226 void SnappyFile::createCache(size_t size)
228 m_cache = new char[size];
229 m_cachePtr = m_cache;
233 void SnappyFile::writeCompressedLength(size_t length)
235 uint32_t value = length;
236 assert(value == length);
237 m_stream.write((const char*)&value, sizeof value);
240 size_t SnappyFile::readCompressedLength()
243 m_stream.read((char*)&length, sizeof length);