From 712429a22a8cc9d51b620d433ade6d327fbabbf0 Mon Sep 17 00:00:00 2001 From: Zack Rusin Date: Thu, 25 Aug 2011 23:22:30 -0400 Subject: [PATCH] Some initial thoughts on the on-demand loading api. --- CMakeLists.txt | 3 ++ loadertest.cpp | 41 ++++++++++++++ trace_file.cpp | 16 ++++++ trace_file.hpp | 15 ++++++ trace_loader.cpp | 126 +++++++++++++++++++++++++++++++++++++++++++ trace_loader.hpp | 72 +++++++++++++++++++++++++ trace_parser.cpp | 3 +- trace_parser.hpp | 18 ++++++- trace_snappyfile.cpp | 45 +++++++++++++--- trace_snappyfile.hpp | 10 ++++ 10 files changed, 340 insertions(+), 9 deletions(-) create mode 100644 loadertest.cpp create mode 100644 trace_loader.cpp create mode 100644 trace_loader.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e840f21..2ee433a 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,6 +187,7 @@ add_library (common trace_writer.cpp trace_local_writer.cpp trace_model_writer.cpp + trace_loader.cpp image.cpp image_bmp.cpp image_pnm.cpp @@ -204,6 +205,8 @@ link_libraries (common) add_executable (tracedump tracedump.cpp) install (TARGETS tracedump RUNTIME DESTINATION bin) +add_executable (loadertest loadertest.cpp) +install (TARGETS loadertest RUNTIME DESTINATION bin) ############################################################################## # API tracers diff --git a/loadertest.cpp b/loadertest.cpp new file mode 100644 index 0000000..48bdac9 --- /dev/null +++ b/loadertest.cpp @@ -0,0 +1,41 @@ +#include "trace_loader.hpp" + +#include + +int main(int argc, char **argv) +{ + int i; + + for (i = 1; i < argc; ++i) { + Trace::Loader loader; + + if (!loader.open(argv[i])) { + std::cerr << "error: failed to open " << argv[i] << "\n"; + return 1; + } + + std::cout << "Number of frames = " + << loader.numberOfFrames() + << std::endl; + std::cout << "Number of calls in frame 0 = " + << loader.numberOfCallsInFrame(0) + << std::endl; + int lastFrame = loader.numberOfFrames() - 1; + std::cout << "Number of calls in frame " + << lastFrame << " = " + << loader.numberOfCallsInFrame(lastFrame) + << std::endl; + + + std::vector frame = loader.frame( + 0); + std::vector::const_iterator itr; + for (itr = frame.begin(); itr != frame.end(); ++itr) { + (*itr)->dump(std::cout, true); + } + } + + + return 0; +} + diff --git a/trace_file.cpp b/trace_file.cpp index e206858..d4e74b9 100644 --- a/trace_file.cpp +++ b/trace_file.cpp @@ -56,6 +56,16 @@ File::~File() close(); } + +File::Offset File::currentOffset() +{ + return File::Offset(); +} + +void File::setCurrentOffset(const File::Offset &offset) +{ +} + bool File::isZLibCompressed(const std::string &filename) { std::fstream stream(filename.c_str(), @@ -133,3 +143,9 @@ void ZLibFile::rawFlush() { gzflush(m_gzFile, Z_SYNC_FLUSH); } + + +bool ZLibFile::supportsOffsets() const +{ + return false; +} diff --git a/trace_file.hpp b/trace_file.hpp index a8b3345..fed30a0 100644 --- a/trace_file.hpp +++ b/trace_file.hpp @@ -29,6 +29,7 @@ #include #include +#include namespace Trace { @@ -38,6 +39,15 @@ public: Read, Write }; + struct Offset { + Offset() + : chunk(0), + offsetInChunk(0) + {} + uint64_t chunk; + uint32_t offsetInChunk; + }; + public: static bool isZLibCompressed(const std::string &filename); static bool isSnappyCompressed(const std::string &filename); @@ -58,6 +68,9 @@ public: void flush(void); int getc(); + virtual bool supportsOffsets() const = 0; + virtual File::Offset currentOffset(); + virtual void setCurrentOffset(const File::Offset &offset); protected: virtual bool rawOpen(const std::string &filename, File::Mode mode) = 0; virtual bool rawWrite(const void *buffer, int length) = 0; @@ -141,6 +154,8 @@ public: File::Mode mode = File::Read); virtual ~ZLibFile(); + + virtual bool supportsOffsets() const; protected: virtual bool rawOpen(const std::string &filename, File::Mode mode); virtual bool rawWrite(const void *buffer, int length); diff --git a/trace_loader.cpp b/trace_loader.cpp new file mode 100644 index 0000000..f2e6c6e --- /dev/null +++ b/trace_loader.cpp @@ -0,0 +1,126 @@ +#include "trace_loader.hpp" + +#include "trace_snappyfile.hpp" + +using namespace Trace; + +Loader::Loader() + : m_frameMarker(FrameMarker_SwapBuffers), + file(0) +{ +} + +Loader::~Loader() +{ + close(); +} + +Loader::FrameMarker Loader::frameMarker() const +{ + return m_frameMarker; +} + +void Loader::setFrameMarker(Loader::FrameMarker marker) +{ + m_frameMarker = marker; +} + +int Loader::numberOfFrames() const +{ + return m_frameOffsets.size(); +} + +int Loader::numberOfCallsInFrame(int frameIdx) +{ + if (frameIdx > m_frameOffsets.size()) { + return 0; + } + + return m_frameOffsets[frameIdx].numberOfCalls; +} + +bool Loader::open(const char *filename) +{ + if (!m_parser.open(filename)) { + std::cerr << "error: failed to open " << filename << "\n"; + return false; + } + + Trace::Call *call; + File::Offset startOffset; + int numOfFrames = 0; + int numOfCalls = 0; + + startOffset = m_parser.currentOffset(); + + while ((call = m_parser.parse_call())) { + + ++numOfCalls; + + if (isCallAFrameMarker(call)) { + File::Offset endOffset = m_parser.currentOffset(); + FrameOffset frameOffset(startOffset, endOffset); + + frameOffset.numberOfCalls = numOfCalls; + m_frameOffsets[numOfFrames] = frameOffset; + ++numOfFrames; + + startOffset = endOffset; + numOfCalls = 0; + } + //call->dump(std::cout, color); + delete call; + } + return true; +} + +void Loader::close() +{ + m_parser.close(); +} + +bool Loader::isCallAFrameMarker(const Trace::Call *call) const +{ + std::string name = call->name(); + + switch (m_frameMarker) { + case FrameMarker_SwapBuffers: + return name.find("SwapBuffers") != std::string::npos || + name == "CGLFlushDrawable" || + name == "glFrameTerminatorGREMEDY"; + break; + case FrameMarker_Flush: + return name == "glFlush"; + break; + case FrameMarker_Finish: + return name == "glFinish"; + break; + case FrameMarker_Clear: + return name == "glClear"; + break; + } + return false; +} + +std::vector Trace::Loader::frame(int idx) +{ + int numOfCalls = numberOfCallsInFrame(idx); + if (numOfCalls) { + std::vector calls(numOfCalls); + m_parser.setCurrentOffset(m_frameOffsets[idx].start); + + Trace::Call *call; + int parsedCalls = 0; + while ((call = m_parser.parse_call())) { + + if (isCallAFrameMarker(call)) { + break; + } + + calls[parsedCalls] = call; + ++parsedCalls; + } + return calls; + } + return std::vector(); +} diff --git a/trace_loader.hpp b/trace_loader.hpp new file mode 100644 index 0000000..7b746b9 --- /dev/null +++ b/trace_loader.hpp @@ -0,0 +1,72 @@ +#ifndef TRACE_LOADER_HPP +#define TRACE_LOADER_HPP + +#include "trace_file.hpp" +#include "trace_parser.hpp" + +#include +#include +#include +#include + +namespace Trace { + +class Frame; + +class Loader +{ +public: + enum FrameMarker { + FrameMarker_SwapBuffers, + FrameMarker_Flush, + FrameMarker_Finish, + FrameMarker_Clear + }; +public: + Loader(); + ~Loader(); + + Loader::FrameMarker frameMarker() const; + void setFrameMarker(Loader::FrameMarker marker); + + int numberOfFrames() const; + int numberOfCallsInFrame(int frameIdx); + + bool open(const char *filename); + void close(); + + std::vector frame(int idx); + +private: + struct FrameOffset { + FrameOffset() + : numberOfCalls(0) + {} + FrameOffset(const File::Offset &s, + const File::Offset &e) + : start(s), + end(e), + numberOfCalls(0) + {} + + File::Offset start; + File::Offset end; + int numberOfCalls; + }; + bool isCallAFrameMarker(const Trace::Call *call) const; + +private: + Trace::Parser m_parser; + FrameMarker m_frameMarker; + + std::map m_frameCache; + std::queue m_loadedFrames; + + std::map m_frameOffsets; + + Trace::File *file; +}; + +} + +#endif // TRACE_LOADER_HPP diff --git a/trace_parser.cpp b/trace_parser.cpp index 47d5a7a..44d1786 100644 --- a/trace_parser.cpp +++ b/trace_parser.cpp @@ -84,9 +84,10 @@ deleteAll(Iter begin, Iter end) template inline void -deleteAll(const Container &c) +deleteAll(Container &c) { deleteAll(c.begin(), c.end()); + c.clear(); } void Parser::close(void) { diff --git a/trace_parser.hpp b/trace_parser.hpp index 4fff9ad..a000986 100644 --- a/trace_parser.hpp +++ b/trace_parser.hpp @@ -30,14 +30,13 @@ #include #include +#include "trace_file.hpp" #include "trace_format.hpp" #include "trace_model.hpp" namespace Trace { -class File; - class Parser { protected: @@ -73,6 +72,21 @@ public: Call *parse_call(void); + bool supportsOffsets() const + { + return file-supportsOffsets(); + } + + File::Offset currentOffset() + { + return file->currentOffset(); + } + + void setCurrentOffset(const File::Offset &offset) + { + file->setCurrentOffset(offset); + } + protected: void parse_enter(void); diff --git a/trace_snappyfile.cpp b/trace_snappyfile.cpp index a348183..c13a979 100644 --- a/trace_snappyfile.cpp +++ b/trace_snappyfile.cpp @@ -28,6 +28,8 @@ #include +#include + #include #include @@ -135,16 +137,13 @@ bool SnappyFile::rawWrite(const void *buffer, int length) bool SnappyFile::rawRead(void *buffer, int length) { - if (m_stream.eof()) { + if (endOfData()) { return false; } - if (freeCacheSize() > length) { - memcpy(buffer, m_cachePtr, length); - m_cachePtr += length; - } else if (freeCacheSize() == length) { + + if (freeCacheSize() >= length) { memcpy(buffer, m_cachePtr, length); m_cachePtr += length; - flushCache(); } else { int sizeToRead = length; int offset = 0; @@ -202,9 +201,18 @@ void SnappyFile::flushCache() if (m_stream.eof()) return; //assert(m_cachePtr == m_cache + m_cacheSize); + m_currentOffset.chunk = m_stream.tellg(); size_t compressedLength; compressedLength = readCompressedLength(); m_stream.read((char*)m_compressedCache, compressedLength); + /* + * The reason we peek here is because the last read will + * read all the way until the last character, but that will not + * trigger m_stream.eof() to be set, so by calling peek + * we assure that if we in fact have read the entire stream + * then the m_stream.eof() is always set. + */ + m_stream.peek(); ::snappy::GetUncompressedLength(m_compressedCache, compressedLength, &m_cacheSize); if (m_cache) @@ -233,3 +241,28 @@ uint32_t SnappyFile::readCompressedLength() m_stream.read((char*)&len, sizeof len); return len; } + +bool SnappyFile::supportsOffsets() const +{ + return true; +} + +File::Offset SnappyFile::currentOffset() +{ + m_currentOffset.offsetInChunk = m_cachePtr - m_cache; + return m_currentOffset; +} + +void SnappyFile::setCurrentOffset(const File::Offset &offset) +{ + // to remove eof bit + m_stream.clear(); + // seek to the start of a chunk + m_stream.seekg(offset.chunk, std::ios::beg); + // load the chunk + flushCache(); + assert(m_cacheSize >= offset.offsetInChunk); + // seek within our cache to the correct location within the chunk + m_cachePtr = m_cache + offset.offsetInChunk; + +} diff --git a/trace_snappyfile.hpp b/trace_snappyfile.hpp index 7fab835..6c0c9ad 100644 --- a/trace_snappyfile.hpp +++ b/trace_snappyfile.hpp @@ -52,6 +52,9 @@ public: File::Mode mode = File::Read); virtual ~SnappyFile(); + virtual bool supportsOffsets() const; + virtual File::Offset currentOffset(); + virtual void setCurrentOffset(const File::Offset &offset); protected: virtual bool rawOpen(const std::string &filename, File::Mode mode); virtual bool rawWrite(const void *buffer, int length); @@ -68,6 +71,11 @@ private: else return 0; } + inline bool endOfData() const + { + return m_stream.eof() && freeCacheSize() == 0; + } + void flushCache(); void createCache(size_t size); void writeCompressedLength(uint32_t num); @@ -79,6 +87,8 @@ private: size_t m_cacheSize; char *m_compressedCache; + + File::Offset m_currentOffset; }; } -- 2.43.0