1 /**************************************************************************
3 * Copyright 2011 Zack Rusin
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 **************************************************************************/
27 #include "trace_snappyfile.hpp"
36 using namespace Trace;
42 * Snappy at its core is just a compressoin algorithm so we're
43 * creating a new file format which uses snappy compression
44 * to hold the trace data.
46 * The file is composed of a number of chunks, they are:
48 * uint32 - specifying the length of the compressed data
49 * compressed data, in little endian
51 * File can contain any number of such chunks.
52 * The default size of an uncompressed chunk is specified in
56 * Currently the default size for a a to-be-compressed data is
57 * 1mb, meaning that the compressed data will be <= 1mb.
58 * The reason it's 1mb is because it seems
59 * to offer a pretty good compression/disk io speed ratio
60 * but that might change.
64 SnappyFile::SnappyFile(const std::string &filename,
71 size_t maxCompressedLength =
72 snappy::MaxCompressedLength(SNAPPY_CHUNK_SIZE);
73 m_compressedCache = new char[maxCompressedLength];
76 SnappyFile::~SnappyFile()
78 delete [] m_compressedCache;
82 bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode)
84 std::ios_base::openmode fmode = std::fstream::binary;
85 if (mode == File::Write) {
86 fmode |= (std::fstream::out | std::fstream::trunc);
87 createCache(SNAPPY_CHUNK_SIZE);
88 } else if (mode == File::Read) {
89 fmode |= std::fstream::in;
92 m_stream.open(filename.c_str(), fmode);
94 //read in the initial buffer if we're reading
95 if (m_stream.is_open() && mode == File::Read) {
96 m_stream.seekg(0, std::ios::end);
97 m_endPos = m_stream.tellg();
98 m_stream.seekg(0, std::ios::beg);
100 // read the snappy file identifier
101 unsigned char byte1, byte2;
104 assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2);
107 } else if (m_stream.is_open() && mode == File::Write) {
108 // write the snappy file identifier
109 m_stream << SNAPPY_BYTE1;
110 m_stream << SNAPPY_BYTE2;
112 return m_stream.is_open();
115 bool SnappyFile::rawWrite(const void *buffer, size_t length)
117 if (freeCacheSize() > length) {
118 memcpy(m_cachePtr, buffer, length);
119 m_cachePtr += length;
120 } else if (freeCacheSize() == length) {
121 memcpy(m_cachePtr, buffer, length);
122 m_cachePtr += length;
125 size_t sizeToWrite = length;
127 while (sizeToWrite >= freeCacheSize()) {
128 size_t endSize = freeCacheSize();
129 size_t offset = length - sizeToWrite;
130 memcpy(m_cachePtr, (const char*)buffer + offset, endSize);
131 sizeToWrite -= endSize;
132 m_cachePtr += endSize;
136 size_t offset = length - sizeToWrite;
137 memcpy(m_cachePtr, (const char*)buffer + offset, sizeToWrite);
138 m_cachePtr += sizeToWrite;
145 bool SnappyFile::rawRead(void *buffer, size_t length)
151 if (freeCacheSize() >= length) {
152 memcpy(buffer, m_cachePtr, length);
153 m_cachePtr += length;
155 size_t sizeToRead = length;
158 size_t chunkSize = std::min(freeCacheSize(), sizeToRead);
159 offset = length - sizeToRead;
160 memcpy((char*)buffer + offset, m_cachePtr, chunkSize);
161 m_cachePtr += chunkSize;
162 sizeToRead -= chunkSize;
163 if (sizeToRead > 0) {
175 int SnappyFile::rawGetc()
183 void SnappyFile::rawClose()
185 if (m_mode == File::Write) {
194 void SnappyFile::rawFlush()
196 assert(m_mode == File::Write);
201 void SnappyFile::flushWriteCache()
203 size_t inputLength = usedCacheSize();
206 size_t compressedLength;
208 ::snappy::RawCompress(m_cache, inputLength,
209 m_compressedCache, &compressedLength);
211 writeCompressedLength(compressedLength);
212 m_stream.write(m_compressedCache, compressedLength);
213 m_cachePtr = m_cache;
215 assert(m_cachePtr == m_cache);
218 void SnappyFile::flushReadCache(size_t skipLength)
220 //assert(m_cachePtr == m_cache + m_cacheSize);
221 m_currentOffset.chunk = m_stream.tellg();
222 size_t compressedLength;
223 compressedLength = readCompressedLength();
225 if (compressedLength) {
226 m_stream.read((char*)m_compressedCache, compressedLength);
227 ::snappy::GetUncompressedLength(m_compressedCache, compressedLength,
229 createCache(m_cacheSize);
230 if (skipLength < m_cacheSize) {
231 ::snappy::RawUncompress(m_compressedCache, compressedLength,
239 void SnappyFile::createCache(size_t size)
241 // TODO: only re-allocate if the current buffer is not big enough
248 m_cache = new char[size];
253 m_cachePtr = m_cache;
257 void SnappyFile::writeCompressedLength(size_t length)
259 unsigned char buf[4];
260 buf[0] = length & 0xff; length >>= 8;
261 buf[1] = length & 0xff; length >>= 8;
262 buf[2] = length & 0xff; length >>= 8;
263 buf[3] = length & 0xff; length >>= 8;
265 m_stream.write((const char *)buf, sizeof buf);
268 size_t SnappyFile::readCompressedLength()
270 unsigned char buf[4];
272 m_stream.read((char *)buf, sizeof buf);
273 if (m_stream.fail()) {
276 length = (size_t)buf[0];
277 length |= ((size_t)buf[1] << 8);
278 length |= ((size_t)buf[2] << 16);
279 length |= ((size_t)buf[3] << 24);
284 bool SnappyFile::supportsOffsets() const
289 File::Offset SnappyFile::currentOffset()
291 m_currentOffset.offsetInChunk = m_cachePtr - m_cache;
292 return m_currentOffset;
295 void SnappyFile::setCurrentOffset(const File::Offset &offset)
299 // seek to the start of a chunk
300 m_stream.seekg(offset.chunk, std::ios::beg);
303 assert(m_cacheSize >= offset.offsetInChunk);
304 // seek within our cache to the correct location within the chunk
305 m_cachePtr = m_cache + offset.offsetInChunk;
309 bool SnappyFile::rawSkip(size_t length)
315 if (freeCacheSize() >= length) {
316 m_cachePtr += length;
318 size_t sizeToRead = length;
320 size_t chunkSize = std::min(freeCacheSize(), sizeToRead);
321 m_cachePtr += chunkSize;
322 sizeToRead -= chunkSize;
323 if (sizeToRead > 0) {
324 flushReadCache(sizeToRead);
335 int SnappyFile::rawPercentRead()
337 return 100 * (double(m_stream.tellg()) / double(m_endPos));