From 299a1b3adc806ef33234683bae4b9477ba5cc378 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Jos=C3=A9=20Fonseca?= Date: Thu, 26 Jan 2012 20:32:59 +0000 Subject: [PATCH] Python pickle output. Proof of concept. This allows to process traces w/ python extremely fast, so hopefully this should enable allow to quickly implement complex analysis in Python, with usable performance. --- cli/CMakeLists.txt | 1 + cli/cli.hpp | 1 + cli/cli_main.cpp | 1 + cli/cli_pickle.cpp | 216 +++++++++++++++++++++++++++++++++++++++++++ common/pickle.hpp | 221 ++++++++++++++++++++++++++++++++++++++++++++ scripts/unpickle.py | 60 ++++++++++++ 6 files changed, 500 insertions(+) create mode 100644 cli/cli_pickle.cpp create mode 100644 common/pickle.hpp create mode 100755 scripts/unpickle.py diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt index 3cdf52c..17d4ffd 100644 --- a/cli/CMakeLists.txt +++ b/cli/CMakeLists.txt @@ -5,6 +5,7 @@ add_executable (apitrace cli_diff_images.cpp cli_dump.cpp cli_pager.cpp + cli_pickle.cpp cli_repack.cpp cli_trace.cpp ) diff --git a/cli/cli.hpp b/cli/cli.hpp index ba2148b..fa72fc7 100644 --- a/cli/cli.hpp +++ b/cli/cli.hpp @@ -44,6 +44,7 @@ extern const Command diff_command; extern const Command diff_state_command; extern const Command diff_images_command; extern const Command dump_command; +extern const Command pickle_command; extern const Command repack_command; extern const Command trace_command; diff --git a/cli/cli_main.cpp b/cli/cli_main.cpp index 8f9392e..9080a4c 100644 --- a/cli/cli_main.cpp +++ b/cli/cli_main.cpp @@ -70,6 +70,7 @@ static const Command * commands[] = { &diff_state_command, &diff_images_command, &dump_command, + &pickle_command, &repack_command, &trace_command, &help_command diff --git a/cli/cli_pickle.cpp b/cli/cli_pickle.cpp new file mode 100644 index 0000000..460a3c3 --- /dev/null +++ b/cli/cli_pickle.cpp @@ -0,0 +1,216 @@ +/************************************************************************** + * + * Copyright 2012 Jose Fonseca + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + **************************************************************************/ + + +#include + +#include "pickle.hpp" + +#include "cli.hpp" +#include "cli_pager.hpp" + +#include "trace_parser.hpp" +#include "trace_model.hpp" +#include "trace_callset.hpp" + + +using namespace trace; + + +class PickleVisitor : public trace::Visitor +{ +protected: + PickleWriter &writer; + +public: + PickleVisitor(PickleWriter &_writer) : + writer(_writer) { + } + + void visit(Null *node) { + writer.writeInt(0); + } + + void visit(Bool *node) { + writer.writeBool(node->value); + } + + void visit(SInt *node) { + writer.writeInt(node->value); + } + + void visit(UInt *node) { + writer.writeInt(node->value); + } + + void visit(Float *node) { + writer.writeFloat(node->value); + } + + void visit(Double *node) { + writer.writeFloat(node->value); + } + + void visit(String *node) { + writer.writeString(node->value); + } + + void visit(Enum *node) { + // TODO: keep symbolic name + writer.writeInt(node->value); + } + + void visit(Bitmask *node) { + // TODO: keep symbolic name + writer.writeInt(node->value); + } + + void visit(Struct *node) { + writer.beginDict(); + for (unsigned i = 0; i < node->sig->num_members; ++i) { + writer.beginItem(node->sig->member_names[i]); + _visit(node->members[i]); + writer.endItem(); + } + writer.endDict(); + } + + void visit(Array *node) { + writer.beginList(); + for (std::vector::iterator it = node->values.begin(); it != node->values.end(); ++it) { + _visit(*it); + } + writer.endList(); + } + + void visit(Blob *node) { + writer.writeString((const char *)node->buf, node->size); + } + + void visit(Pointer *node) { + writer.writeInt(node->value); + } + + void visit(Call *call) { + writer.beginTuple(); + + writer.writeInt(call->no); + + writer.writeString(call->name()); + + writer.beginList(); + for (unsigned i = 0; i < call->args.size(); ++i) { + if (call->args[i]) { + _visit(call->args[i]); + } else { + writer.writeNone(); + } + } + writer.endList(); + + if (call->ret) { + _visit(call->ret); + } else { + writer.writeNone(); + } + + writer.endTuple(); + } +}; + + +static trace::CallSet calls(trace::FREQUENCY_ALL); + +static const char *synopsis = "Pickle given trace(s) to standard output."; + +static void +usage(void) +{ + std::cout + << "usage: apitrace pickle [OPTIONS] ...\n" + << synopsis << "\n" + "\n" + " --calls Only pickle specified calls\n" + ; +} + +static int +command(int argc, char *argv[]) +{ + int i; + + for (i = 0; i < argc;) { + const char *arg = argv[i]; + + if (arg[0] != '-') { + break; + } + + ++i; + + if (!strcmp(arg, "--")) { + break; + } else if (!strcmp(arg, "--help")) { + usage(); + return 0; + } else if (!strcmp(arg, "--calls")) { + calls = trace::CallSet(argv[i++]); + } else { + std::cerr << "error: unknown option " << arg << "\n"; + usage(); + return 1; + } + } + + + for (; i < argc; ++i) { + trace::Parser parser; + + if (!parser.open(argv[i])) { + std::cerr << "error: failed to open " << argv[i] << "\n"; + return 1; + } + + trace::Call *call; + while ((call = parser.parse_call())) { + if (calls.contains(*call)) { + PickleWriter writer(std::cout); + PickleVisitor visitor(writer); + + visitor.visit(call); + } + delete call; + } + } + + return 0; +} + +const Command pickle_command = { + "pickle", + synopsis, + usage, + command +}; diff --git a/common/pickle.hpp b/common/pickle.hpp new file mode 100644 index 0000000..c8f90ae --- /dev/null +++ b/common/pickle.hpp @@ -0,0 +1,221 @@ +/************************************************************************** + * + * Copyright 2012 Jose Fonseca + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + **************************************************************************/ + +/* + * Python pickle writer + */ + +#ifndef _PICKLE_HPP_ +#define _PICKLE_HPP_ + +#include +#include + +#include +#include + + +class PickleWriter +{ +private: + std::ostream &os; + + /* + * Python pickle opcodes. See pickle.py and pickletools.py from Python + * standard library for details. + */ + enum Opcode { + MARK = '(', + STOP = '.', + POP = '0', + POP_MARK = '1', + DUP = '2', + FLOAT = 'F', + INT = 'I', + BININT = 'J', + BININT1 = 'K', + LONG = 'L', + BININT2 = 'M', + NONE = 'N', + PERSID = 'P', + BINPERSID = 'Q', + REDUCE = 'R', + STRING = 'S', + BINSTRING = 'T', + SHORT_BINSTRING = 'U', + UNICODE = 'V', + BINUNICODE = 'X', + APPEND = 'a', + BUILD = 'b', + GLOBAL = 'c', + DICT = 'd', + EMPTY_DICT = '}', + APPENDS = 'e', + GET = 'g', + BINGET = 'h', + INST = 'i', + LONG_BINGET = 'j', + LIST = 'l', + EMPTY_LIST = ']', + OBJ = 'o', + PUT = 'p', + BINPUT = 'q', + LONG_BINPUT = 'r', + SETITEM = 's', + TUPLE = 't', + EMPTY_TUPLE = ')', + SETITEMS = 'u', + BINFLOAT = 'G', + + PROTO = 0x80, + NEWOBJ = 0x81, + EXT1 = 0x82, + EXT2 = 0x83, + EXT4 = 0x84, + TUPLE1 = 0x85, + TUPLE2 = 0x86, + TUPLE3 = 0x87, + NEWTRUE = 0x88, + NEWFALSE = 0x89, + LONG1 = 0x8a, + LONG4 = 0x8b, + }; + +public: + PickleWriter(std::ostream &_os) : + os(_os) + { + os.put(PROTO); + os.put(2); + } + + ~PickleWriter() { + os.put(STOP); + } + + inline void beginDict() { + os.put(EMPTY_DICT); + os.put(BINPUT); + os.put(1); + } + + inline void endDict() { + } + + inline void beginItem() { + } + + inline void beginItem(const char * name) { + writeString(name); + } + + inline void beginItem(const std::string &name) { + beginItem(name.c_str()); + } + + inline void endItem(void) { + os.put(SETITEM); + } + + inline void beginList() { + os.put(EMPTY_LIST); + os.put(BINPUT); + os.put(1); + os.put(MARK); + } + + inline void endList(void) { + os.put(APPENDS); + } + + inline void beginTuple() { + os.put(MARK); + } + + inline void endTuple(void) { + os.put(TUPLE); + } + + inline void putInt(int i) { + os.put( i & 0xff); + os.put((i >> 8) & 0xff); + os.put((i >> 16) & 0xff); + os.put((i >> 24) & 0xff); + } + + inline void writeString(const char *s, size_t length) { + if (!s) { + writeNone(); + return; + } + + if (length < 256) { + os.put(SHORT_BINSTRING); + os.put(length); + } else { + os.put(BINSTRING); + putInt(length); + } + os.write(s, length); + + os.put(BINPUT); + os.put(1); + } + + inline void writeString(const char *s) { + if (!s) { + writeNone(); + return; + } + + writeString(s, strlen(s)); + } + + inline void writeString(const std::string &s) { + writeString(s.c_str(), s.size()); + } + + inline void writeNone(void) { + os.put(NONE); + } + + inline void writeBool(bool b) { + os.put(b ? NEWTRUE : NEWFALSE); + } + + inline void writeInt(long i) { + // TODO: binary + os.put(INT); + os << i << '\n'; + } + + inline void writeFloat(float f) { + // TODO: binary + os.put(FLOAT); + os << f << '\n'; + } +}; + +#endif /* _Pickle_HPP_ */ diff --git a/scripts/unpickle.py b/scripts/unpickle.py new file mode 100755 index 0000000..efda4a6 --- /dev/null +++ b/scripts/unpickle.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +########################################################################## +# +# Copyright 2012 Jose Fonseca +# All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##########################################################################/ + +'''Sample program for apitrace pickle command. + +Run as: + + apitrace pickle foo.trace | python unpickle.py + +''' + + +import cPickle as pickle +import sys +import time + + +def main(): + calls = 0 + startTime = time.time() + while True: + try: + call = pickle.load(sys.stdin) + except EOFError: + break + else: + callNo, functionName, args, ret = call + if False: + sys.stdout.write('%u %s%r = %r\n' % (callNo, functionName, tuple(args), ret)) + calls += 1 + stopTime = time.time() + duration = stopTime - startTime + sys.stderr.write('%u calls, %.03f secs, %u class/sec\n' % (calls, duration, calls/duration)) + + +if __name__ == '__main__': + main() -- 2.43.0