Add snappy compression/decompression code.

diff --git a/CMakeLists.txt b/CMakeLists.txt
index a543107de15ab4857c3053d31900f541c3d58265..64e63bf1d9192df95a8eeb700b20d61e6ba957b0 100755 (executable)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -23,6 +23,7 @@ set (CMAKE_USE_PYTHON_VERSION 2.7 2.6)
 
 find_package (PythonInterp REQUIRED)
 find_package (OpenGL REQUIRED)
+find_package (Snappy REQUIRED)
 
 if (ENABLE_GUI)
     if (NOT (ENABLE_GUI STREQUAL "AUTO"))
@@ -140,6 +141,8 @@ add_subdirectory (thirdparty/zlib EXCLUDE_FROM_ALL)
 include_directories (${ZLIB_INCLUDE_DIRS})
 link_libraries (${ZLIB_LIBRARIES})
 
+link_libraries(${SNAPPY_LIBRARIES})
+
 set (PNG_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/libpng)
 set (PNG_DEFINITIONS "")
 set (PNG_LIBRARIES png_bundled)
@@ -339,7 +342,7 @@ else ()
     )
 
     target_link_libraries (glxtrace dl ${X11_X11_LIB})
-    
+
     install (TARGETS glxtrace LIBRARY DESTINATION lib)
 endif ()
 

diff --git a/cmake/FindSnappy.cmake b/cmake/FindSnappy.cmake
new file mode 100644 (file)
index 0000000..4ddcd54
--- /dev/null
+++ b/cmake/FindSnappy.cmake
@@ -0,0 +1,28 @@
+# - Try to find Snappy
+# Once done this will define
+#  SNAPPY_FOUND - System has Snappy
+#  SNAPPY_INCLUDE_DIRS - The Snappy include directories
+#  SNAPPY_LIBRARIES - The libraries needed to use Snappy
+#  SNAPPY_DEFINITIONS - Compiler switches required for using Snappy
+
+find_package(PkgConfig)
+pkg_check_modules(PC_SNAPPY QUIET snappy)
+set(SNAPPY_DEFINITIONS ${PC_SNAPPY_CFLAGS_OTHER})
+
+find_path(SNAPPY_INCLUDE_DIR snappy.h
+          HINTS ${PC_SNAPPY_INCLUDEDIR} ${PC_SNAPPY_INCLUDE_DIRS}
+          PATH_SUFFIXES snappy )
+
+find_library(SNAPPY_LIBRARY NAMES snappy libsnappy
+             HINTS ${PC_SNAPPY_LIBDIR} ${PC_SNAPPY_LIBRARY_DIRS} )
+
+set(SNAPPY_LIBRARIES ${SNAPPY_LIBRARY} )
+set(SNAPPY_INCLUDE_DIRS ${SNAPPY_INCLUDE_DIR} )
+
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set SNAPPY_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(snappy  DEFAULT_MSG
+                                  SNAPPY_LIBRARY SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(SNAPPY_INCLUDE_DIR SNAPPY_LIBRARY )

diff --git a/trace_file.cpp b/trace_file.cpp
index 11499783a71ab03e34c9382a6e0b21686cf19ba3..e92eb754898e4569e6a98c750b941ea0235a4aa2 100644 (file)
--- a/trace_file.cpp
+++ b/trace_file.cpp
@@ -1,6 +1,12 @@
 #include "trace_file.hpp"
 
+#include <assert.h>
+#include <string.h>
+
 #include <zlib.h>
+#include <snappy.h>
+
+#include <iostream>
 
 using namespace Trace;
 
@@ -15,8 +21,11 @@ File::File(const std::string &filename,
     }
 }
 
+static size_t writtenRef = 0;
+
 File::~File()
 {
+    std::cerr << "written ref = "<<writtenRef << std::endl;
     close();
 }
 
@@ -41,6 +50,7 @@ bool File::open(const std::string &filename, File::Mode mode)
         close();
     }
     m_isOpened = rawOpen(filename, mode);
+    m_mode = mode;
     return m_isOpened;
 }
 
@@ -49,6 +59,7 @@ bool File::write(const void *buffer, int length)
     if (!m_isOpened || m_mode != File::Write) {
         return false;
     }
+    writtenRef += length;
     return rawWrite(buffer, length);
 }
 
@@ -126,3 +137,196 @@ void ZLibFile::rawFlush()
 {
     gzflush(m_gzFile, Z_SYNC_FLUSH);
 }
+
+SnappyFile::SnappyFile(const std::string &filename,
+                              File::Mode mode)
+    : File(),
+      m_cache(0),
+      m_cachePtr(0),
+      m_cacheSize(0)
+{
+    m_compressedCache = new char[SNAPPY_CHUNK_SIZE];
+}
+
+SnappyFile::~SnappyFile()
+{
+    delete [] m_compressedCache;
+}
+
+static size_t written = 0;
+static size_t writtenComp = 0;
+
+bool SnappyFile::rawOpen(const std::string &filename, File::Mode mode)
+{
+    std::ios_base::openmode fmode = std::fstream::binary;
+    if (mode == File::Write) {
+        fmode |= (std::fstream::out | std::fstream::trunc);
+        createCache(SNAPPY_CHUNK_SIZE);
+    } else if (mode == File::Read) {
+        fmode |= std::fstream::in;
+    }
+
+    m_stream.open(filename.c_str(), fmode);
+
+    //read in the initial buffer if we're reading
+    if (m_stream.is_open() && mode == File::Read) {
+        flushCache();
+    }
+    return m_stream.is_open();
+}
+
+bool SnappyFile::rawWrite(const void *buffer, int length)
+{
+    static int bufWritten = 0;
+    if (bufWritten < 500) {
+        const char *cbuffer = (const char*)buffer;
+        for (int i = 0; i < length; ++i) {
+            std::cerr << "--- "<<bufWritten << ") "<< (int) cbuffer[i]
+                      << std::endl;
+            ++ bufWritten;
+        }
+    }
+    if (freeCacheSize() > length) {
+        memcpy(m_cachePtr, buffer, length);
+        m_cachePtr += length;
+    } else if (freeCacheSize() == length) {
+        memcpy(m_cachePtr, buffer, length);
+        m_cachePtr += length;
+        flushCache();
+    } else {
+        int sizeToWrite = length;
+
+        while (sizeToWrite >= freeCacheSize()) {
+            int endSize = freeCacheSize();
+            int offset = length - sizeToWrite;
+            memcpy(m_cachePtr, (char*)buffer + offset, endSize);
+            sizeToWrite -= endSize;
+            m_cachePtr += endSize;
+            flushCache();
+        }
+        if (sizeToWrite) {
+            int offset = length - sizeToWrite;
+            memcpy(m_cachePtr, (char*)buffer + offset, sizeToWrite);
+            m_cachePtr += sizeToWrite;
+        }
+    }
+
+    return true;
+}
+
+bool SnappyFile::rawRead(void *buffer, int length)
+{
+    std::cerr << "Reading byte = "<< (m_cachePtr - m_cache) << std::endl;
+    if (freeCacheSize() > length) {
+        memcpy(buffer, m_cachePtr, length);
+        m_cachePtr += length;
+    } else if (freeCacheSize() == length) {
+        memcpy(buffer, m_cachePtr, length);
+        m_cachePtr += length;
+        assert(0);
+        flushCache();
+    } else {
+        int sizeToRead = length;
+        int offset = 0;
+        assert(0);
+        while (sizeToRead) {
+            int chunkSize = std::min(freeCacheSize(), sizeToRead);
+            offset = length - sizeToRead;
+            memcpy((char*)buffer + offset, m_cachePtr, chunkSize);
+            m_cachePtr += chunkSize;
+            sizeToRead -= chunkSize;
+            if (sizeToRead > 0)
+                flushCache();
+        }
+    }
+
+    return true;
+}
+
+char SnappyFile::rawGetc()
+{
+    char c;
+    rawRead(&c, 1);
+    return c;
+}
+
+void SnappyFile::rawClose()
+{
+    flushCache();
+    std::cerr << "written = "<< written
+              <<", comp = "<< writtenComp
+              << std::endl;
+    m_stream.close();
+    delete [] m_cache;
+    m_cache = NULL;
+    m_cachePtr = NULL;
+}
+
+void SnappyFile::rawFlush()
+{
+    m_stream.flush();
+}
+
+void SnappyFile::flushCache()
+{
+    if (m_mode == File::Write) {
+        size_t compressedLength;
+
+        static bool first = true;
+        if (first) {
+            std::cerr << "Buffer = [";
+            for (int i = 0; i < 512; ++i) {
+                std::cerr << i << " ) "<< (int)m_cache[i] << std::endl;
+            }
+            std::cerr << "]"<<std::endl;
+            first = false;
+        }
+
+        ::snappy::RawCompress(m_cache, SNAPPY_CHUNK_SIZE - freeCacheSize(),
+                              m_compressedCache, &compressedLength);
+
+        m_stream << compressedLength;
+        m_stream.write(m_compressedCache, compressedLength);
+        std::cerr << "compressed length = "<<compressedLength
+                  <<" cache size = "<<(SNAPPY_CHUNK_SIZE - freeCacheSize())
+                  <<" (ref = " << SNAPPY_CHUNK_SIZE <<")"
+                  << std::endl;
+        written += SNAPPY_CHUNK_SIZE - freeCacheSize();
+        writtenComp += compressedLength;
+        m_cachePtr = m_cache;
+    } else if (m_mode == File::Read) {
+        if (m_stream.eof())
+            return;
+        //assert(m_cachePtr == m_cache + m_cacheSize);
+        size_t compressedLength;
+        m_stream >> compressedLength;
+        m_stream.read((char*)m_compressedCache, compressedLength);
+        ::snappy::GetUncompressedLength(m_compressedCache, compressedLength,
+                                        &m_cacheSize);
+        std::cerr << "compressed length = "<<compressedLength
+                  <<" cache size = "<<m_cacheSize
+                  <<" (ref = " << SNAPPY_CHUNK_SIZE <<")"
+                  << std::endl;
+        if (m_cache)
+            delete [] m_cache;
+        createCache(m_cacheSize);
+        ::snappy::RawUncompress(m_compressedCache, compressedLength,
+                                m_cache);
+        static bool first = true;
+        if (first) {
+            std::cerr << "Buffer = [";
+            for (int i = 0; i < 512; ++i) {
+                std::cerr << i << " ) "<< (int)m_cache[i] << std::endl;
+            }
+            std::cerr << "]"<<std::endl;
+            first = false;
+        }
+    }
+}
+
+void SnappyFile::createCache(size_t size)
+{
+    m_cache = new char[size];
+    m_cachePtr = m_cache;
+    m_cacheSize = size;
+}

diff --git a/trace_file.hpp b/trace_file.hpp
index 9ea6bb8dd893b067754af82070f29cb1047787bd..5a460f79b0c13e1027970d89c6b281757878f389 100644 (file)
--- a/trace_file.hpp
+++ b/trace_file.hpp
@@ -2,6 +2,7 @@
 #define TRACE_FILE_HPP
 
 #include <string>
+#include <fstream>
 
 namespace Trace {
 
@@ -58,6 +59,45 @@ private:
     void *m_gzFile;
 };
 
+namespace snappy {
+    class File;
+}
+
+#define SNAPPY_CHUNK_SIZE (1 * 1024 * 1024)
+class SnappyFile : public File {
+public:
+    SnappyFile(const std::string &filename = std::string(),
+               File::Mode mode = File::Read);
+    virtual ~SnappyFile();
+
+protected:
+    virtual bool rawOpen(const std::string &filename, File::Mode mode);
+    virtual bool rawWrite(const void *buffer, int length);
+    virtual bool rawRead(void *buffer, int length);
+    virtual char rawGetc();
+    virtual void rawClose();
+    virtual void rawFlush();
+
+private:
+    inline int freeCacheSize() const
+    {
+        if (m_cacheSize > 0)
+            return m_cacheSize - (m_cachePtr - m_cache);
+        else
+            return 0;
+    }
+    void flushCache();
+    void createCache(size_t size);
+private:
+    std::fstream m_stream;
+    char *m_cache;
+    char *m_cachePtr;
+    size_t m_cacheSize;
+
+    char *m_compressedCache;
+};
+
+
 }
 
 #endif