Merge remote-tracking branch 'origin/master' into on-demand-loading

diff --combined trace_file.cpp
index 652cd0f3f70bb0b14216aa2a48919b11b74c4767,81b5d03c1f3f273f972f7b8b3d9db8be22f53336..2d002868920feda1d7c2d48438bd8af5e88b8351
--- 1/trace_file.cpp
--- 2/trace_file.cpp
+++ b/trace_file.cpp
@@@ -56,16 -56,6 +56,16 @@@ File::~File(
      close();
  }
  
 +
 +File::Offset File::currentOffset()
 +{
 +    return File::Offset();
 +}
 +
 +void File::setCurrentOffset(const File::Offset &offset)
 +{
 +}
 +
  bool File::isZLibCompressed(const std::string &filename)
  {
      std::fstream stream(filename.c_str(),
@@@ -116,12 -106,12 +116,12 @@@ bool ZLibFile::rawOpen(const std::strin
      return m_gzFile != NULL;
  }
  
- bool ZLibFile::rawWrite(const void *buffer, int length)
+ bool ZLibFile::rawWrite(const void *buffer, size_t length)
  {
      return gzwrite(m_gzFile, buffer, length) != -1;
  }
  
- bool ZLibFile::rawRead(void *buffer, int length)
+ bool ZLibFile::rawRead(void *buffer, size_t length)
  {
      return gzread(m_gzFile, buffer, length) != -1;
  }
@@@ -143,14 -133,3 +143,14 @@@ void ZLibFile::rawFlush(
  {
      gzflush(m_gzFile, Z_SYNC_FLUSH);
  }
 +
 +
 +bool ZLibFile::supportsOffsets() const
 +{
 +    return false;
 +}
 +
 +bool ZLibFile::rawSkip(unsigned)
 +{
 +    return false;
 +}

diff --combined trace_file.hpp
index 13b392cdaa8876729791fb641a2e1f747f2dca8b,c709050d6ab96371a957a818bddffc52389951d4..487121187fdd9dad368fe631445b3ffea218a80e
--- 1/trace_file.hpp
--- 2/trace_file.hpp
+++ b/trace_file.hpp
@@@ -29,7 -29,6 +29,7 @@@
  
  #include <string>
  #include <fstream>
 +#include <stdint.h>
  
  namespace Trace {
  
@@@ -39,15 -38,6 +39,15 @@@ public
          Read,
          Write
      };
 +    struct Offset {
 +        Offset()
 +            : chunk(0),
 +              offsetInChunk(0)
 +        {}
 +        uint64_t chunk;
 +        uint32_t offsetInChunk;
 +    };
 +
  public:
      static bool isZLibCompressed(const std::string &filename);
      static bool isSnappyCompressed(const std::string &filename);
@@@ -62,24 -52,19 +62,24 @@@ public
      std::string filename() const;
  
      bool open(const std::string &filename, File::Mode mode);
-     bool write(const void *buffer, int length);
-     bool read(void *buffer, int length);
+     bool write(const void *buffer, size_t length);
+     bool read(void *buffer, size_t length);
      void close();
      void flush(void);
      int getc();
 +    bool skip(unsigned length);
  
 +    virtual bool supportsOffsets() const = 0;
 +    virtual File::Offset currentOffset();
 +    virtual void setCurrentOffset(const File::Offset &offset);
  protected:
      virtual bool rawOpen(const std::string &filename, File::Mode mode) = 0;
-     virtual bool rawWrite(const void *buffer, int length) = 0;
-     virtual bool rawRead(void *buffer, int length) = 0;
+     virtual bool rawWrite(const void *buffer, size_t length) = 0;
+     virtual bool rawRead(void *buffer, size_t length) = 0;
      virtual int rawGetc() = 0;
      virtual void rawClose() = 0;
      virtual void rawFlush() = 0;
 +    virtual bool rawSkip(unsigned length) = 0;
  
  protected:
      std::string m_filename;
@@@ -113,7 -98,7 +113,7 @@@ inline bool File::open(const std::strin
      return m_isOpened;
  }
  
- inline bool File::write(const void *buffer, int length)
+ inline bool File::write(const void *buffer, size_t length)
  {
      if (!m_isOpened || m_mode != File::Write) {
          return false;
@@@ -121,7 -106,7 +121,7 @@@
      return rawWrite(buffer, length);
  }
  
- inline bool File::read(void *buffer, int length)
+ inline bool File::read(void *buffer, size_t length)
  {
      if (!m_isOpened || m_mode != File::Read) {
          return false;
@@@ -150,68 -135,23 +150,68 @@@ inline int File::getc(
      return rawGetc();
  }
  
 +inline bool File::skip(unsigned length)
 +{
 +    if (!m_isOpened || m_mode != File::Read) {
 +        return false;
 +    }
 +    return rawSkip(length);
 +}
 +
  class ZLibFile : public File {
  public:
      ZLibFile(const std::string &filename = std::string(),
               File::Mode mode = File::Read);
      virtual ~ZLibFile();
  
 +
 +    virtual bool supportsOffsets() const;
  protected:
      virtual bool rawOpen(const std::string &filename, File::Mode mode);
-     virtual bool rawWrite(const void *buffer, int length);
-     virtual bool rawRead(void *buffer, int length);
+     virtual bool rawWrite(const void *buffer, size_t length);
+     virtual bool rawRead(void *buffer, size_t length);
      virtual int rawGetc();
      virtual void rawClose();
      virtual void rawFlush();
 +    virtual bool rawSkip(unsigned length);
  private:
      void *m_gzFile;
  };
  
 +inline bool
 +operator<(const File::Offset &one, const File::Offset &two)
 +{
 +    return one.chunk < two.chunk ||
 +            (one.chunk == two.chunk && one.offsetInChunk < two.offsetInChunk);
 +}
 +
 +inline bool
 +operator==(const File::Offset &one, const File::Offset &two)
 +{
 +    return one.chunk == two.chunk &&
 +            one.offsetInChunk == two.offsetInChunk;
 +}
 +
 +inline bool
 +operator>=(const File::Offset &one, const File::Offset &two)
 +{
 +    return one.chunk > two.chunk ||
 +            (one.chunk == two.chunk && one.offsetInChunk >= two.offsetInChunk);
 +}
 +
 +inline bool
 +operator>(const File::Offset &one, const File::Offset &two)
 +{
 +    return two < one;
 +}
 +
 +inline bool
 +operator<=(const File::Offset &one, const File::Offset &two)
 +{
 +    return two >= one;
 +}
 +
 +
  }
  
  #endif

diff --combined trace_snappyfile.cpp
index 443ebe7c5040c131bf7eef72daeb84ace273b98b,7e7072543426851c125257d2c9d0b5e0367e3e86..c350013d7f1acc7e7ff1546fa860019f94d47762
--- 1/trace_snappyfile.cpp
--- 2/trace_snappyfile.cpp
+++ b/trace_snappyfile.cpp
@@@ -28,8 -28,6 +28,8 @@@
  
  #include <snappy.h>
  
 +#include <iostream>
 +
  #include <assert.h>
  #include <string.h>
  
@@@ -46,7 -44,7 +46,7 @@@ using namespace Trace
   * The file is composed of a number of chunks, they are:
   * chunk {
   *     uint32 - specifying the length of the compressed data
-  *     compressed data
+  *     compressed data, in little endian
   * }
   * File can contain any number of such chunks.
   * The default size of an uncompressed chunk is specified in
@@@ -76,6 -74,7 +76,7 @@@ SnappyFile::SnappyFile(const std::strin
  SnappyFile::~SnappyFile()
  {
      delete [] m_compressedCache;
+     delete [] m_cache;
  }
  
  bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode)
@@@ -107,7 -106,7 +108,7 @@@
      return m_stream.is_open();
  }
  
- bool SnappyFile::rawWrite(const void *buffer, int length)
+ bool SnappyFile::rawWrite(const void *buffer, size_t length)
  {
      if (freeCacheSize() > length) {
          memcpy(m_cachePtr, buffer, length);
@@@ -122,14 -121,14 +123,14 @@@
          while (sizeToWrite >= freeCacheSize()) {
              int endSize = freeCacheSize();
              int offset = length - sizeToWrite;
-             memcpy(m_cachePtr, (char*)buffer + offset, endSize);
+             memcpy(m_cachePtr, (const char*)buffer + offset, endSize);
              sizeToWrite -= endSize;
              m_cachePtr += endSize;
              flushCache();
          }
          if (sizeToWrite) {
              int offset = length - sizeToWrite;
-             memcpy(m_cachePtr, (char*)buffer + offset, sizeToWrite);
+             memcpy(m_cachePtr, (const char*)buffer + offset, sizeToWrite);
              m_cachePtr += sizeToWrite;
          }
      }
@@@ -137,7 -136,7 +138,7 @@@
      return true;
  }
  
- bool SnappyFile::rawRead(void *buffer, int length)
+ bool SnappyFile::rawRead(void *buffer, size_t length)
  {
      if (endOfData()) {
          return false;
@@@ -147,10 -146,10 +148,10 @@@
          memcpy(buffer, m_cachePtr, length);
          m_cachePtr += length;
      } else {
-         int sizeToRead = length;
-         int offset = 0;
+         size_t sizeToRead = length;
+         size_t offset = 0;
          while (sizeToRead) {
-             int chunkSize = std::min(freeCacheSize(), sizeToRead);
+             size_t chunkSize = std::min(freeCacheSize(), sizeToRead);
              offset = length - sizeToRead;
              memcpy((char*)buffer + offset, m_cachePtr, chunkSize);
              m_cachePtr += chunkSize;
@@@ -191,104 -190,78 +192,128 @@@ void SnappyFile::rawFlush(
  void SnappyFile::flushCache()
  {
      if (m_mode == File::Write) {
-         size_t compressedLength;
+         size_t inputLength = usedCacheSize();
  
-         ::snappy::RawCompress(m_cache, SNAPPY_CHUNK_SIZE - freeCacheSize(),
-                               m_compressedCache, &compressedLength);
+         if (inputLength) {
+             size_t compressedLength;
  
-         writeCompressedLength(compressedLength);
-         m_stream.write(m_compressedCache, compressedLength);
-         m_cachePtr = m_cache;
+             ::snappy::RawCompress(m_cache, inputLength,
+                                   m_compressedCache, &compressedLength);
+ 
+             writeCompressedLength(compressedLength);
+             m_stream.write(m_compressedCache, compressedLength);
+             m_cachePtr = m_cache;
+         }
+         assert(m_cachePtr == m_cache);
      } else if (m_mode == File::Read) {
-         if (m_stream.eof())
-             return;
          //assert(m_cachePtr == m_cache + m_cacheSize);
 +        m_currentOffset.chunk = m_stream.tellg();
          size_t compressedLength;
          compressedLength = readCompressedLength();
-         m_stream.read((char*)m_compressedCache, compressedLength);
-         /*
-          * The reason we peek here is because the last read will
-          * read all the way until the last character, but that will not
-          * trigger m_stream.eof() to be set, so by calling peek
-          * we assure that if we in fact have read the entire stream
-          * then the m_stream.eof() is always set.
-          */
-         m_stream.peek();
-         ::snappy::GetUncompressedLength(m_compressedCache, compressedLength,
-                                         &m_cacheSize);
-         if (m_cache)
-             delete [] m_cache;
-         createCache(m_cacheSize);
-         ::snappy::RawUncompress(m_compressedCache, compressedLength,
-                                 m_cache);
+ 
+         if (compressedLength) {
+             m_stream.read((char*)m_compressedCache, compressedLength);
+             ::snappy::GetUncompressedLength(m_compressedCache, compressedLength,
+                                             &m_cacheSize);
+             createCache(m_cacheSize);
+             ::snappy::RawUncompress(m_compressedCache, compressedLength,
+                                     m_cache);
+         } else {
+             createCache(0);
+         }
      }
  }
  
  void SnappyFile::createCache(size_t size)
  {
-     m_cache = new char[size];
+     // TODO: only re-allocate if the current buffer is not big enough
+ 
+     if (m_cache) {
+         delete [] m_cache;
+     }
+ 
+     if (size) {
+         m_cache = new char[size];
+     } else {
+         m_cache = NULL;
+     }
+ 
      m_cachePtr = m_cache;
      m_cacheSize = size;
  }
  
- void SnappyFile::writeCompressedLength(uint32_t value)
+ void SnappyFile::writeCompressedLength(size_t length)
  {
-     m_stream.write((const char*)&value, sizeof value);
+     unsigned char buf[4];
+     buf[0] = length & 0xff; length >>= 8;
+     buf[1] = length & 0xff; length >>= 8;
+     buf[2] = length & 0xff; length >>= 8;
+     buf[3] = length & 0xff; length >>= 8;
+     assert(length == 0);
+     m_stream.write((const char *)buf, sizeof buf);
  }
  
- uint32_t SnappyFile::readCompressedLength()
+ size_t SnappyFile::readCompressedLength()
  {
-     uint32_t len;
-     m_stream.read((char*)&len, sizeof len);
-     return len;
+     unsigned char buf[4];
+     size_t length;
+     m_stream.read((char *)buf, sizeof buf);
+     if (m_stream.fail()) {
+         length = 0;
+     } else {
+         length  =  (size_t)buf[0];
+         length |= ((size_t)buf[1] <<  8);
+         length |= ((size_t)buf[2] << 16);
+         length |= ((size_t)buf[3] << 24);
+     }
+     return length;
  }
 +
 +bool SnappyFile::supportsOffsets() const
 +{
 +    return true;
 +}
 +
 +File::Offset SnappyFile::currentOffset()
 +{
 +    m_currentOffset.offsetInChunk = m_cachePtr - m_cache;
 +    return m_currentOffset;
 +}
 +
 +void SnappyFile::setCurrentOffset(const File::Offset &offset)
 +{
 +    // to remove eof bit
 +    m_stream.clear();
 +    // seek to the start of a chunk
 +    m_stream.seekg(offset.chunk, std::ios::beg);
 +    // load the chunk
 +    flushCache();
 +    assert(m_cacheSize >= offset.offsetInChunk);
 +    // seek within our cache to the correct location within the chunk
 +    m_cachePtr = m_cache + offset.offsetInChunk;
 +
 +}
 +
 +bool SnappyFile::rawSkip(unsigned length)
 +{
 +    if (endOfData()) {
 +        return false;
 +    }
 +
 +    if (freeCacheSize() >= length) {
 +        m_cachePtr += length;
 +    } else {
 +        int sizeToRead = length;
 +        while (sizeToRead) {
 +            int chunkSize = std::min(freeCacheSize(), sizeToRead);
 +            m_cachePtr += chunkSize;
 +            sizeToRead -= chunkSize;
 +            if (sizeToRead > 0)
 +                flushCache();
 +            if (!m_cacheSize)
 +                break;
 +        }
 +    }
 +
 +    return true;
 +}

diff --combined trace_snappyfile.hpp
index ecf39dedbe649a470f531aaba1bf56918812e797,61b7ab1b29435aeeb51c833a05958c82f3bcee32..398ff7900c335a4c7f3b55d8afe0e643c5651e49
--- 1/trace_snappyfile.hpp
--- 2/trace_snappyfile.hpp
+++ b/trace_snappyfile.hpp
@@@ -27,13 -27,13 +27,13 @@@
  #ifndef TRACE_SNAPPYFILE_HPP
  #define TRACE_SNAPPYFILE_HPP
  
+ #include <assert.h>
+ 
  #include "trace_file.hpp"
  
  #include <string>
  #include <fstream>
  
- #include <stdint.h>
- 
  namespace snappy {
      class File;
  }
@@@ -52,25 -52,28 +52,32 @@@ public
                 File::Mode mode = File::Read);
      virtual ~SnappyFile();
  
 +    virtual bool supportsOffsets() const;
 +    virtual File::Offset currentOffset();
 +    virtual void setCurrentOffset(const File::Offset &offset);
  protected:
      virtual bool rawOpen(const std::string &filename, File::Mode mode);
-     virtual bool rawWrite(const void *buffer, int length);
-     virtual bool rawRead(void *buffer, int length);
+     virtual bool rawWrite(const void *buffer, size_t length);
+     virtual bool rawRead(void *buffer, size_t length);
      virtual int rawGetc();
      virtual void rawClose();
      virtual void rawFlush();
 +    virtual bool rawSkip(unsigned length);
  
  private:
-     inline int freeCacheSize() const
+     inline size_t usedCacheSize() const
+     {
+         assert(m_cachePtr >= m_cache);
+         return m_cachePtr - m_cache;
+     }
+     inline size_t freeCacheSize() const
      {
-         if (m_cacheSize > 0)
-             return m_cacheSize - (m_cachePtr - m_cache);
-         else
+         assert(m_cacheSize >= usedCacheSize());
+         if (m_cacheSize > 0) {
+             return m_cacheSize - usedCacheSize();
+         } else {
              return 0;
+         }
      }
      inline bool endOfData() const
      {
@@@ -78,8 -81,8 +85,8 @@@
      }
      void flushCache();
      void createCache(size_t size);
-     void writeCompressedLength(uint32_t  num);
-     uint32_t readCompressedLength();
+     void writeCompressedLength(size_t length);
+     size_t readCompressedLength();
  private:
      std::fstream m_stream;
      char *m_cache;
@@@ -87,8 -90,6 +94,8 @@@
      size_t m_cacheSize;
  
      char *m_compressedCache;
 +
 +    File::Offset m_currentOffset;
  };
  
  }