1 /**************************************************************************
3 * Copyright 2011 Zack Rusin
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 **************************************************************************/
31 * Snappy at its core is just a compressoin algorithm so we're
32 * creating a new file format which uses snappy compression
33 * to hold the trace data.
35 * The file is composed of a number of chunks, they are:
37 * uint32 - specifying the length of the compressed data
38 * compressed data, in little endian
40 * File can contain any number of such chunks.
41 * The default size of an uncompressed chunk is specified in
45 * Currently the default size for a a to-be-compressed data is
46 * 1mb, meaning that the compressed data will be <= 1mb.
47 * The reason it's 1mb is because it seems
48 * to offer a pretty good compression/disk io speed ratio
49 * but that might change.
61 #include "trace_file.hpp"
64 #define SNAPPY_CHUNK_SIZE (1 * 1024 * 1024)
66 #define SNAPPY_BYTE1 'a'
67 #define SNAPPY_BYTE2 't'
70 using namespace trace;
73 class SnappyFile : public File {
75 SnappyFile(const std::string &filename = std::string(),
76 File::Mode mode = File::Read);
77 virtual ~SnappyFile();
79 virtual bool supportsOffsets() const;
80 virtual File::Offset currentOffset();
81 virtual void setCurrentOffset(const File::Offset &offset);
83 virtual bool rawOpen(const std::string &filename, File::Mode mode);
84 virtual bool rawWrite(const void *buffer, size_t length);
85 virtual size_t rawRead(void *buffer, size_t length);
86 virtual int rawGetc();
87 virtual void rawClose();
88 virtual void rawFlush();
89 virtual bool rawSkip(size_t length);
90 virtual int rawPercentRead();
93 inline size_t usedCacheSize() const
95 assert(m_cachePtr >= m_cache);
96 return m_cachePtr - m_cache;
98 inline size_t freeCacheSize() const
100 assert(m_cacheSize >= usedCacheSize());
101 if (m_cacheSize > 0) {
102 return m_cacheSize - usedCacheSize();
107 inline bool endOfData() const
109 return m_stream.eof() && freeCacheSize() == 0;
111 void flushWriteCache();
112 void flushReadCache(size_t skipLength = 0);
113 void createCache(size_t size);
114 void writeCompressedLength(size_t length);
115 size_t readCompressedLength();
117 std::fstream m_stream;
118 size_t m_cacheMaxSize;
123 char *m_compressedCache;
125 File::Offset m_currentOffset;
126 std::streampos m_endPos;
129 SnappyFile::SnappyFile(const std::string &filename,
132 m_cacheMaxSize(SNAPPY_CHUNK_SIZE),
133 m_cacheSize(m_cacheMaxSize),
134 m_cache(new char [m_cacheMaxSize]),
137 size_t maxCompressedLength =
138 snappy::MaxCompressedLength(SNAPPY_CHUNK_SIZE);
139 m_compressedCache = new char[maxCompressedLength];
142 SnappyFile::~SnappyFile()
145 delete [] m_compressedCache;
149 bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode)
151 std::ios_base::openmode fmode = std::fstream::binary;
152 if (mode == File::Write) {
153 fmode |= (std::fstream::out | std::fstream::trunc);
154 createCache(SNAPPY_CHUNK_SIZE);
155 } else if (mode == File::Read) {
156 fmode |= std::fstream::in;
159 m_stream.open(filename.c_str(), fmode);
161 //read in the initial buffer if we're reading
162 if (m_stream.is_open() && mode == File::Read) {
163 m_stream.seekg(0, std::ios::end);
164 m_endPos = m_stream.tellg();
165 m_stream.seekg(0, std::ios::beg);
167 // read the snappy file identifier
168 unsigned char byte1, byte2;
171 assert(byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2);
174 } else if (m_stream.is_open() && mode == File::Write) {
175 // write the snappy file identifier
176 m_stream << SNAPPY_BYTE1;
177 m_stream << SNAPPY_BYTE2;
179 return m_stream.is_open();
182 bool SnappyFile::rawWrite(const void *buffer, size_t length)
184 if (freeCacheSize() > length) {
185 memcpy(m_cachePtr, buffer, length);
186 m_cachePtr += length;
187 } else if (freeCacheSize() == length) {
188 memcpy(m_cachePtr, buffer, length);
189 m_cachePtr += length;
192 size_t sizeToWrite = length;
194 while (sizeToWrite >= freeCacheSize()) {
195 size_t endSize = freeCacheSize();
196 size_t offset = length - sizeToWrite;
197 memcpy(m_cachePtr, (const char*)buffer + offset, endSize);
198 sizeToWrite -= endSize;
199 m_cachePtr += endSize;
203 size_t offset = length - sizeToWrite;
204 memcpy(m_cachePtr, (const char*)buffer + offset, sizeToWrite);
205 m_cachePtr += sizeToWrite;
212 size_t SnappyFile::rawRead(void *buffer, size_t length)
218 if (freeCacheSize() >= length) {
219 memcpy(buffer, m_cachePtr, length);
220 m_cachePtr += length;
222 size_t sizeToRead = length;
225 size_t chunkSize = std::min(freeCacheSize(), sizeToRead);
226 offset = length - sizeToRead;
227 memcpy((char*)buffer + offset, m_cachePtr, chunkSize);
228 m_cachePtr += chunkSize;
229 sizeToRead -= chunkSize;
230 if (sizeToRead > 0) {
234 return length - sizeToRead;
242 int SnappyFile::rawGetc()
245 if (rawRead(&c, 1) != 1)
250 void SnappyFile::rawClose()
252 if (m_mode == File::Write) {
261 void SnappyFile::rawFlush()
263 assert(m_mode == File::Write);
268 void SnappyFile::flushWriteCache()
270 size_t inputLength = usedCacheSize();
273 size_t compressedLength;
275 ::snappy::RawCompress(m_cache, inputLength,
276 m_compressedCache, &compressedLength);
278 writeCompressedLength(compressedLength);
279 m_stream.write(m_compressedCache, compressedLength);
280 m_cachePtr = m_cache;
282 assert(m_cachePtr == m_cache);
285 void SnappyFile::flushReadCache(size_t skipLength)
287 //assert(m_cachePtr == m_cache + m_cacheSize);
288 m_currentOffset.chunk = m_stream.tellg();
289 size_t compressedLength;
290 compressedLength = readCompressedLength();
292 if (compressedLength) {
293 m_stream.read((char*)m_compressedCache, compressedLength);
294 ::snappy::GetUncompressedLength(m_compressedCache, compressedLength,
296 createCache(m_cacheSize);
297 if (skipLength < m_cacheSize) {
298 ::snappy::RawUncompress(m_compressedCache, compressedLength,
306 void SnappyFile::createCache(size_t size)
308 if (size > m_cacheMaxSize) {
310 m_cacheMaxSize <<= 1;
311 } while (size > m_cacheMaxSize);
314 m_cache = new char[size];
315 m_cacheMaxSize = size;
318 m_cachePtr = m_cache;
322 void SnappyFile::writeCompressedLength(size_t length)
324 unsigned char buf[4];
325 buf[0] = length & 0xff; length >>= 8;
326 buf[1] = length & 0xff; length >>= 8;
327 buf[2] = length & 0xff; length >>= 8;
328 buf[3] = length & 0xff; length >>= 8;
330 m_stream.write((const char *)buf, sizeof buf);
333 size_t SnappyFile::readCompressedLength()
335 unsigned char buf[4];
337 m_stream.read((char *)buf, sizeof buf);
338 if (m_stream.fail()) {
341 length = (size_t)buf[0];
342 length |= ((size_t)buf[1] << 8);
343 length |= ((size_t)buf[2] << 16);
344 length |= ((size_t)buf[3] << 24);
349 bool SnappyFile::supportsOffsets() const
354 File::Offset SnappyFile::currentOffset()
356 m_currentOffset.offsetInChunk = m_cachePtr - m_cache;
357 return m_currentOffset;
360 void SnappyFile::setCurrentOffset(const File::Offset &offset)
364 // seek to the start of a chunk
365 m_stream.seekg(offset.chunk, std::ios::beg);
368 assert(m_cacheSize >= offset.offsetInChunk);
369 // seek within our cache to the correct location within the chunk
370 m_cachePtr = m_cache + offset.offsetInChunk;
374 bool SnappyFile::rawSkip(size_t length)
380 if (freeCacheSize() >= length) {
381 m_cachePtr += length;
383 size_t sizeToRead = length;
385 size_t chunkSize = std::min(freeCacheSize(), sizeToRead);
386 m_cachePtr += chunkSize;
387 sizeToRead -= chunkSize;
388 if (sizeToRead > 0) {
389 flushReadCache(sizeToRead);
400 int SnappyFile::rawPercentRead()
402 return 100 * (double(m_stream.tellg()) / double(m_endPos));
406 File* File::createSnappy(void) {
407 return new SnappyFile;
410 bool File::isSnappyCompressed(const std::string &filename)
412 std::fstream stream(filename.c_str(),
413 std::fstream::binary | std::fstream::in);
414 if (!stream.is_open())
417 unsigned char byte1, byte2;
422 return (byte1 == SNAPPY_BYTE1 && byte2 == SNAPPY_BYTE2);