Split the driver code.
authorJosé Fonseca <jose.r.fonseca@gmail.com>
Sat, 17 Mar 2012 14:13:33 +0000 (14:13 +0000)
committerJosé Fonseca <jose.r.fonseca@gmail.com>
Sat, 17 Mar 2012 14:13:33 +0000 (14:13 +0000)
app_driver.py [new file with mode: 0755]
apps/CMakeLists.txt
base_driver.py [new file with mode: 0644]
driver.py [deleted file]
tool_driver.py [new file with mode: 0755]
traces/CMakeLists.txt
traces/zlib-no-eof.ref.txt

diff --git a/app_driver.py b/app_driver.py
new file mode 100755 (executable)
index 0000000..a44bade
--- /dev/null
@@ -0,0 +1,458 @@
+#!/usr/bin/env python
+##########################################################################
+#
+# Copyright 2011 Jose Fonseca
+# All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+##########################################################################/
+
+'''Application test driver.'''
+
+
+import os.path
+import platform
+import re
+import subprocess
+import sys
+import time
+import json
+import base64
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+
+from base_driver import *
+
+
+class TraceChecker:
+
+    def __init__(self, srcStream, refFileName, verbose=False):
+        self.srcStream = srcStream
+        self.refFileName = refFileName
+        if refFileName:
+            self.refStream = open(refFileName, 'rt')
+        else:
+            self.refStream = None
+        self.verbose = verbose
+        self.doubleBuffer = False
+        self.callNo = 0
+        self.refLine = ''
+        self.images = []
+        self.states = []
+
+    call_re = re.compile(r'^([0-9]+) (\w+)\(')
+
+    def check(self):
+
+        swapbuffers = 0
+        flushes = 0
+
+        srcLines = []
+        self.consumeRefLine()
+        for line in self.srcStream:
+            line = line.rstrip()
+            if self.verbose:
+                sys.stdout.write(line + '\n')
+            mo = self.call_re.match(line)
+            if mo:
+                self.callNo = int(mo.group(1))
+                function_name = mo.group(2)
+                if function_name.find('SwapBuffers') != -1 or \
+                   line.find('kCGLPFADoubleBuffer') != -1:
+                    swapbuffers += 1
+                if function_name in ('glFlush', 'glFinish'):
+                    flushes += 1
+                srcLine = line[mo.start(2):]
+            else:
+                srcLine = line
+            if self.refLine:
+                if srcLine == self.refLine:
+                    self.consumeRefLine()
+                    srcLines = []
+                else:
+                    srcLines.append(srcLine)
+
+        if self.refLine:
+            if srcLines:
+                fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0]))
+            else:
+                fail('missing call %s' % self.refLine)
+
+        if swapbuffers:
+            self.doubleBuffer = True
+        else:
+            self.doubleBuffer = False
+
+    def consumeRefLine(self):
+        if not self.refStream:
+            self.refLine = ''
+            return
+
+        while True:
+            line = self.refStream.readline()
+            if not line:
+                break
+            line = line.rstrip()
+            if line.startswith('#'):
+                self.handlePragma(line)
+            else:
+                break
+        self.refLine = line
+
+    def handlePragma(self, line):
+        pragma, rest = line.split(None, 1)
+        if pragma == '#image':
+            imageFileName = self.getAbsPath(rest)
+            self.images.append((self.callNo, imageFileName))
+        elif pragma == '#state':
+            stateFileName = self.getAbsPath(rest)
+            self.states.append((self.callNo, stateFileName))
+        else:
+            assert False
+
+    def getAbsPath(self, path):
+        '''Get the absolute from a path relative to the reference filename'''
+        return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path))
+
+
+
+class TestCase:
+
+    cmd = None
+    cwd = None
+
+    api = 'gl'
+    max_frames = None
+    trace_file = None
+
+    ref_dump = None
+
+    doubleBuffer = True
+
+    verbose = False
+
+    threshold_precision = 12.0
+
+    def __init__(self):
+        self.stateCache = {}
+    
+    def runApp(self):
+        '''Run the application standalone, skipping this test if it fails by
+        some reason.'''
+
+        if not self.cmd:
+            return
+
+        p = popen(self.cmd, cwd=self.cwd)
+        p.wait()
+        if p.returncode:
+            skip('application returned code %i' % p.returncode)
+
+    api_map = {
+        'gl': 'gl',
+        'egl_gl': 'egl',
+        'egl_gles1': 'egl',
+        'egl_gles2': 'egl',
+    }
+
+    def traceApp(self):
+        if not self.cmd:
+            return
+
+        if self.trace_file is None:
+            if self.ref_dump is not None:
+                name = self.ref_dump
+            else:
+                name = self.cmd[0]
+            name, ext = os.path.splitext(os.path.basename(name))
+            while ext:
+                name, ext = os.path.splitext(os.path.basename(name))
+            self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
+        if os.path.exists(self.trace_file):
+            os.remove(self.trace_file)
+        else:
+            trace_dir = os.path.dirname(self.trace_file)
+            if not os.path.exists(trace_dir):
+                os.makedirs(trace_dir)
+
+        cmd = self.cmd
+        env = os.environ.copy()
+        
+        cmd = [
+            options.apitrace, 'trace', 
+            '--api', self.api_map[self.api],
+            '--output', self.trace_file,
+            '--'
+        ] + cmd
+        if self.max_frames is not None:
+            env['TRACE_FRAMES'] = str(self.max_frames)
+
+        p = popen(cmd, env=env, cwd=self.cwd)
+        p.wait()
+
+        if not os.path.exists(self.trace_file):
+            fail('no trace file generated\n')
+    
+    def checkTrace(self):
+        cmd = [options.apitrace, 'dump', '--color=never', self.trace_file]
+        p = popen(cmd, stdout=subprocess.PIPE)
+
+        checker = TraceChecker(p.stdout, self.ref_dump, self.verbose)
+        checker.check()
+        p.wait()
+        if p.returncode != 0:
+            fail('`apitrace dump` returned code %i' % p.returncode)
+
+        self.doubleBuffer = checker.doubleBuffer
+
+        for callNo, refImageFileName in checker.images:
+            self.checkImage(callNo, refImageFileName)
+        for callNo, refStateFileName in checker.states:
+            self.checkState(callNo, refStateFileName)
+
+    def checkImage(self, callNo, refImageFileName):
+        try:
+            from PIL import Image
+        except ImportError:
+            return
+
+        srcImage = self.getImage(callNo)
+        refImage = Image.open(refImageFileName)
+
+        from snapdiff import Comparer
+        comparer = Comparer(refImage, srcImage)
+        precision = comparer.precision(filter=True)
+        sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
+        if precision < self.threshold_precision:
+            prefix = '%s.%u' % (self.getNamePrefix(), callNo)
+            srcImageFileName = prefix + '.src.png'
+            srcImage.save(srcImageFileName)
+            diffImageFileName = prefix + '.diff.png'
+            comparer.write_diff(diffImageFileName)
+            fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
+
+    def checkState(self, callNo, refStateFileName):
+        srcState = self.getState(callNo)
+        refState = self.getRefState(refStateFileName)
+
+        from jsondiff import Comparer, Differ
+        comparer = Comparer(ignore_added = True)
+        match = comparer.visit(refState, srcState)
+        if not match:
+            prefix = '%s.%u' % (self.getNamePrefix(), callNo)
+            srcStateFileName = prefix + '.src.json'
+            diffStateFileName = prefix + '.diff.json'
+            self.saveState(srcState, srcStateFileName)
+            #diffStateFile = open(diffStateFileName, 'wt')
+            diffStateFile = sys.stdout
+            differ = Differ(diffStateFile, ignore_added = True)
+            differ.visit(refState, srcState)
+            fail('state from call %u does not match %s' % (callNo, refStateFileName))
+
+    def getRefState(self, refStateFileName):
+        stream = open(refStateFileName, 'rt')
+        from jsondiff import load
+        state = load(stream)
+        self.adjustRefState(state)
+        return state
+
+    def getNamePrefix(self):
+        name = os.path.basename(self.ref_dump)
+        try:
+            index = name.index('.')
+        except ValueError:
+            pass
+        else:
+            name = name[:index]
+        return name
+
+    def saveState(self, state, filename):
+        s = json.dumps(state, sort_keys=True, indent=2)
+        open(filename, 'wt').write(s)
+
+    def retrace(self):
+        p = self._retrace()
+        p.wait()
+        if p.returncode != 0:
+            fail('retrace failed with code %i' % (p.returncode))
+
+    def getImage(self, callNo):
+        from PIL import Image
+        state = self.getState(callNo)
+        if self.doubleBuffer:
+            attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0']
+        else:
+            attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0']
+        imageObj = self.getFramebufferAttachment(state, attachments)
+        data = imageObj['__data__']
+        stream = StringIO(base64.b64decode(data))
+        im = Image.open(stream)
+        im.save('test.png')
+        return im
+
+    def getFramebufferAttachment(self, state, attachments):
+        framebufferObj = state['framebuffer']
+        for attachment in attachments:
+            try:
+                attachmentObj = framebufferObj[attachment]
+            except KeyError:
+                pass
+            else:
+                return attachmentObj
+        raise Exception("no attachment found")
+
+    def getState(self, callNo):
+        try:
+            state = self.stateCache[callNo]
+        except KeyError:
+            pass
+        else:
+            return state
+
+        p = self._retrace(['-D', str(callNo)])
+        state = json.load(p.stdout, strict=False)
+        p.wait()
+        if p.returncode != 0:
+            fail('retrace returned code %i' % (p.returncode))
+
+        self.adjustSrcState(state)
+
+        self.stateCache[callNo] = state
+
+        return state
+
+    def adjustSrcState(self, state):
+        # Do some adjustments on the obtained state to eliminate failures from
+        # bugs/issues outside of apitrace
+
+        try:
+            parameters = state['parameters']
+        except KeyError:
+            return
+
+        # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1
+        self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
+
+        # On Gallium 
+        if 'Gallium' in parameters['GL_RENDERER'].split():
+            # Gallium drivers have wrong defaults for draw/read buffer state
+            self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
+            self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK')
+            self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
+            self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
+            self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT')
+            self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
+
+    def adjustRefState(self, state):
+        # Do some adjustments on reference state to eliminate failures from
+        # bugs/issues outside of apitrace
+
+        try:
+            parameters = state['parameters']
+        except KeyError:
+            return
+
+        if platform.system() == 'Darwin':
+            # Mac OS X drivers fail on GL_COLOR_SUM
+            # XXX: investigate this
+            self.removeState(parameters, 'GL_COLOR_SUM')
+
+    def replaceState(self, obj, key, srcValue, dstValue):
+        try:
+            value = obj[key]
+        except KeyError:
+            pass
+        else:
+            if value == srcValue:
+                obj[key] = dstValue
+
+    def removeState(self, obj, key):
+        try:
+            del obj[key]
+        except KeyError:
+            pass
+
+    def _retrace(self, args = None, stdout=subprocess.PIPE):
+        retrace = self.api_map[self.api] + 'retrace'
+        cmd = [get_build_program(retrace)]
+        if self.doubleBuffer:
+            cmd += ['-db']
+        else:
+            cmd += ['-sb']
+        if args:
+            cmd += args
+        cmd += [self.trace_file]
+        return popen(cmd, stdout=stdout)
+
+    def run(self):
+        self.runApp()
+        self.traceApp()
+        self.checkTrace()
+        self.retrace()
+
+        pass_()
+
+
+class AppMain(Main):
+
+    def createOptParser(self):
+        optparser = Main.createOptParser(self)
+
+        optparser.add_option(
+            '-a', '--api', metavar='API',
+            type='string', dest='api', default='gl',
+            help='api to trace')
+        optparser.add_option(
+            '-R', '--results', metavar='PATH',
+            type='string', dest='results', default='.',
+            help='results directory [default=%default]')
+        optparser.add_option(
+            '--ref-dump', metavar='PATH',
+            type='string', dest='ref_dump', default=None,
+            help='reference dump')
+
+        return optparser
+
+    def main(self):
+        global options
+
+        (options, args) = self.parseOptions()
+
+        if not os.path.exists(options.results):
+            os.makedirs(options.results)
+
+        test = TestCase()
+        test.verbose = options.verbose
+
+        test.cmd = args
+        test.cwd = options.cwd
+        test.api = options.api
+        test.ref_dump = options.ref_dump
+        test.results = options.results
+
+        test.run()
+
+
+if __name__ == '__main__':
+    AppMain().main()
index 5736c21f65af7d1cea1e97208f4ef9da5ddd09f0..179370948b714a967c1df95b2dc768dc11c8e4d3 100644 (file)
@@ -13,7 +13,7 @@ function (ADD_APP_TEST)
     add_test(
         NAME app_${TEST_NAME}
         COMMAND
-        python ${CMAKE_SOURCE_DIR}/driver.py
+        python ${CMAKE_SOURCE_DIR}/app_driver.py
             --apitrace ${APITRACE_EXECUTABLE}
             --api ${api}
             --ref-dump ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_REF}
diff --git a/base_driver.py b/base_driver.py
new file mode 100644 (file)
index 0000000..51032bd
--- /dev/null
@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+##########################################################################
+#
+# Copyright 2011-2012 Jose Fonseca
+# All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+##########################################################################/
+
+'''Common test driver code.'''
+
+
+import optparse
+import os.path
+import platform
+import subprocess
+import sys
+
+
+def _exit(status, code, reason=None):
+    if reason is None:
+        reason = ''
+    else:
+        reason = ' (%s)' % reason
+    sys.stdout.write('%s%s\n' % (status, reason))
+    sys.exit(code)
+
+def fail(reason=None):
+    _exit('FAIL', 1, reason)
+
+def skip(reason=None):
+    _exit('SKIP', 0, reason)
+
+def pass_(reason=None):
+    _exit('PASS', 0, reason)
+
+
+def popen(command, *args, **kwargs):
+    if kwargs.get('cwd', None) is not None:
+        sys.stdout.write('cd %s && ' % kwargs['cwd'])
+
+    try:
+        env = kwargs.pop('env')
+    except KeyError:
+        env = None
+    else:
+        names = env.keys()
+        names.sort()
+        for name in names:
+            value = env[name]
+            if value != os.environ.get(name, None):
+                sys.stdout.write('%s=%s ' % (name, value))
+            env[name] = str(value)
+
+    sys.stdout.write(' '.join(command) + '\n')
+    sys.stdout.flush()
+
+    return subprocess.Popen(command, *args, env=env, **kwargs)
+
+
+def which(executable):
+    dirs = os.environ['PATH'].split(os.path.pathsep)
+    for dir in dirs:
+        path = os.path.join(dir, executable)
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def get_bin_path():
+    if os.path.exists(options.apitrace):
+        apitrace_abspath = os.path.abspath(options.apitrace)
+    else:
+        apitrace_abspath = which(options.apitrace)
+        if apitrace_abspath is None:
+            sys.stderr.write('error: could not determine the absolute path of\n' % options.apitrace)
+            sys.exit(1)
+    return os.path.dirname(apitrace_abspath)
+
+
+def get_build_program(program):
+    bin_path = get_bin_path()
+    if platform.system() == 'Windows':
+        program += '.exe'
+    path = os.path.join(bin_path, program)
+    if not os.path.exists(path):
+        sys.stderr.write('error: %s does not exist\n' % path)
+        sys.exit(1)
+    return path
+
+
+def get_scripts_path():
+    bin_path = get_bin_path()
+
+    try_paths = [
+        'scripts',
+        '../lib/scripts',
+        '../lib/apitrace/scripts',
+    ]
+
+    for try_path in try_paths:
+        path = os.path.join(bin_path, try_path)
+        if os.path.exists(path):
+            return os.path.abspath(path)
+
+    sys.stderr.write('error: could not find scripts directory\n')
+    sys.exit(1)
+
+
+class Main:
+    global options
+
+    def createOptParser(self):
+        default_apitrace = 'apitrace'
+        if platform.system() == 'Windows':
+            default_apitrace += '.exe'
+
+        # Parse command line options
+        optparser = optparse.OptionParser(
+            usage='\n\t%prog [OPTIONS] -- [ARGS] ...',
+            version='%%prog')
+        optparser.add_option(
+            '-v', '--verbose',
+            action="store_true",
+            dest="verbose", default=False,
+            help="verbose output")
+        optparser.add_option(
+            '--apitrace', metavar='PROGRAM',
+            type='string', dest='apitrace', default=default_apitrace,
+            help='path to apitrace executable')
+        optparser.add_option(
+            '-C', '--directory', metavar='PATH',
+            type='string', dest='cwd', default=None,
+            help='change to directory')
+
+        return optparser
+
+    def parseOptions(self):
+        global options
+
+        optparser = self.createOptParser()
+        (options, args) = optparser.parse_args(sys.argv[1:])
+        if not args:
+            optparser.error('an argument must be specified')
+        
+        print get_scripts_path()
+
+        sys.path.insert(0, get_scripts_path())
+
+        return options, args
+
+    def main(self):
+        raise NotImplementedError
+
diff --git a/driver.py b/driver.py
deleted file mode 100755 (executable)
index d5789e3..0000000
--- a/driver.py
+++ /dev/null
@@ -1,568 +0,0 @@
-#!/usr/bin/env python
-##########################################################################
-#
-# Copyright 2011 Jose Fonseca
-# All Rights Reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-##########################################################################/
-
-'''Main test driver.'''
-
-
-import optparse
-import os.path
-import platform
-import re
-import subprocess
-import sys
-import time
-import json
-import base64
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
-
-def _exit(status, code, reason=None):
-    if reason is None:
-        reason = ''
-    else:
-        reason = ' (%s)' % reason
-    sys.stdout.write('%s%s\n' % (status, reason))
-    sys.exit(code)
-
-def fail(reason=None):
-    _exit('FAIL', 1, reason)
-
-def skip(reason=None):
-    _exit('SKIP', 0, reason)
-
-def pass_(reason=None):
-    _exit('PASS', 0, reason)
-
-
-def popen(command, *args, **kwargs):
-    if kwargs.get('cwd', None) is not None:
-        sys.stdout.write('cd %s && ' % kwargs['cwd'])
-
-    try:
-        env = kwargs.pop('env')
-    except KeyError:
-        env = None
-    else:
-        names = env.keys()
-        names.sort()
-        for name in names:
-            value = env[name]
-            if value != os.environ.get(name, None):
-                sys.stdout.write('%s=%s ' % (name, value))
-            env[name] = str(value)
-
-    sys.stdout.write(' '.join(command) + '\n')
-    sys.stdout.flush()
-
-    return subprocess.Popen(command, *args, env=env, **kwargs)
-
-
-def which(executable):
-    dirs = os.environ['PATH'].split(os.path.pathsep)
-    for dir in dirs:
-        path = os.path.join(dir, executable)
-        if os.path.exists(path):
-            return path
-    return None
-
-
-def _get_bin_path():
-    if os.path.exists(options.apitrace):
-        apitrace_abspath = os.path.abspath(options.apitrace)
-    else:
-        apitrace_abspath = which(options.apitrace)
-        if apitrace_abspath is None:
-            sys.stderr.write('error: could not determine the absolute path of\n' % options.apitrace)
-            sys.exit(1)
-    return os.path.dirname(apitrace_abspath)
-
-
-def _get_build_program(program):
-    bin_path = _get_bin_path()
-    if platform.system() == 'Windows':
-        program += '.exe'
-    path = os.path.join(bin_path, program)
-    if not os.path.exists(path):
-        sys.stderr.write('error: %s does not exist\n' % path)
-        sys.exit(1)
-    return path
-
-def _get_scripts_path():
-    bin_path = _get_bin_path()
-
-    try_paths = [
-        'scripts',
-        '../lib/scripts',
-        '../lib/apitrace/scripts',
-    ]
-
-    for try_path in try_paths:
-        path = os.path.join(bin_path, try_path)
-        if os.path.exists(path):
-            return os.path.abspath(path)
-
-    sys.stderr.write('error: could not find scripts directory\n')
-    sys.exit(1)
-
-
-class TraceChecker:
-
-    def __init__(self, srcStream, refFileName, verbose=False):
-        self.srcStream = srcStream
-        self.refFileName = refFileName
-        if refFileName:
-            self.refStream = open(refFileName, 'rt')
-        else:
-            self.refStream = None
-        self.verbose = verbose
-        self.doubleBuffer = False
-        self.callNo = 0
-        self.refLine = ''
-        self.images = []
-        self.states = []
-
-    call_re = re.compile(r'^([0-9]+) (\w+)\(')
-
-    def check(self):
-
-        swapbuffers = 0
-        flushes = 0
-
-        srcLines = []
-        self.consumeRefLine()
-        for line in self.srcStream:
-            line = line.rstrip()
-            if self.verbose:
-                sys.stdout.write(line + '\n')
-            mo = self.call_re.match(line)
-            if mo:
-                self.callNo = int(mo.group(1))
-                function_name = mo.group(2)
-                if function_name.find('SwapBuffers') != -1 or \
-                   line.find('kCGLPFADoubleBuffer') != -1:
-                    swapbuffers += 1
-                if function_name in ('glFlush', 'glFinish'):
-                    flushes += 1
-                srcLine = line[mo.start(2):]
-            else:
-                srcLine = line
-            if self.refLine:
-                if srcLine == self.refLine:
-                    self.consumeRefLine()
-                    srcLines = []
-                else:
-                    srcLines.append(srcLine)
-
-        if self.refLine:
-            if srcLines:
-                fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0]))
-            else:
-                fail('missing call %s' % self.refLine)
-
-        if swapbuffers:
-            self.doubleBuffer = True
-        else:
-            self.doubleBuffer = False
-
-    def consumeRefLine(self):
-        if not self.refStream:
-            self.refLine = ''
-            return
-
-        while True:
-            line = self.refStream.readline()
-            if not line:
-                break
-            line = line.rstrip()
-            if line.startswith('#'):
-                self.handlePragma(line)
-            else:
-                break
-        self.refLine = line
-
-    def handlePragma(self, line):
-        pragma, rest = line.split(None, 1)
-        if pragma == '#image':
-            imageFileName = self.getAbsPath(rest)
-            self.images.append((self.callNo, imageFileName))
-        elif pragma == '#state':
-            stateFileName = self.getAbsPath(rest)
-            self.states.append((self.callNo, stateFileName))
-        else:
-            assert False
-
-    def getAbsPath(self, path):
-        '''Get the absolute from a path relative to the reference filename'''
-        return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path))
-
-
-
-class TestCase:
-
-    cmd = None
-    cwd = None
-
-    api = 'gl'
-    max_frames = None
-    trace_file = None
-
-    ref_dump = None
-
-    doubleBuffer = True
-
-    verbose = False
-
-    threshold_precision = 12.0
-
-    def __init__(self):
-        self.stateCache = {}
-    
-    def runApp(self):
-        '''Run the application standalone, skipping this test if it fails by
-        some reason.'''
-
-        if not self.cmd:
-            return
-
-        p = popen(self.cmd, cwd=self.cwd)
-        p.wait()
-        if p.returncode:
-            skip('application returned code %i' % p.returncode)
-
-    api_map = {
-        'gl': 'gl',
-        'egl_gl': 'egl',
-        'egl_gles1': 'egl',
-        'egl_gles2': 'egl',
-    }
-
-    def traceApp(self):
-        if not self.cmd:
-            return
-
-        if self.trace_file is None:
-            if self.ref_dump is not None:
-                name = self.ref_dump
-            else:
-                name = self.cmd[0]
-            name, ext = os.path.splitext(os.path.basename(name))
-            while ext:
-                name, ext = os.path.splitext(os.path.basename(name))
-            self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
-        if os.path.exists(self.trace_file):
-            os.remove(self.trace_file)
-        else:
-            trace_dir = os.path.dirname(self.trace_file)
-            if not os.path.exists(trace_dir):
-                os.makedirs(trace_dir)
-
-        cmd = self.cmd
-        env = os.environ.copy()
-        
-        cmd = [
-            options.apitrace, 'trace', 
-            '--api', self.api_map[self.api],
-            '--output', self.trace_file,
-            '--'
-        ] + cmd
-        if self.max_frames is not None:
-            env['TRACE_FRAMES'] = str(self.max_frames)
-
-        p = popen(cmd, env=env, cwd=self.cwd)
-        p.wait()
-
-        if not os.path.exists(self.trace_file):
-            fail('no trace file generated\n')
-    
-    def checkTrace(self):
-        cmd = [options.apitrace, 'dump', '--color=never', self.trace_file]
-        p = popen(cmd, stdout=subprocess.PIPE)
-
-        checker = TraceChecker(p.stdout, self.ref_dump, self.verbose)
-        checker.check()
-        p.wait()
-        if p.returncode != 0:
-            fail('`apitrace dump` returned code %i' % p.returncode)
-
-        self.doubleBuffer = checker.doubleBuffer
-
-        for callNo, refImageFileName in checker.images:
-            self.checkImage(callNo, refImageFileName)
-        for callNo, refStateFileName in checker.states:
-            self.checkState(callNo, refStateFileName)
-
-    def checkImage(self, callNo, refImageFileName):
-        try:
-            from PIL import Image
-        except ImportError:
-            return
-
-        srcImage = self.getImage(callNo)
-        refImage = Image.open(refImageFileName)
-
-        from snapdiff import Comparer
-        comparer = Comparer(refImage, srcImage)
-        precision = comparer.precision(filter=True)
-        sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
-        if precision < self.threshold_precision:
-            prefix = '%s.%u' % (self.getNamePrefix(), callNo)
-            srcImageFileName = prefix + '.src.png'
-            srcImage.save(srcImageFileName)
-            diffImageFileName = prefix + '.diff.png'
-            comparer.write_diff(diffImageFileName)
-            fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
-
-    def checkState(self, callNo, refStateFileName):
-        srcState = self.getState(callNo)
-        refState = self.getRefState(refStateFileName)
-
-        from jsondiff import Comparer, Differ
-        comparer = Comparer(ignore_added = True)
-        match = comparer.visit(refState, srcState)
-        if not match:
-            prefix = '%s.%u' % (self.getNamePrefix(), callNo)
-            srcStateFileName = prefix + '.src.json'
-            diffStateFileName = prefix + '.diff.json'
-            self.saveState(srcState, srcStateFileName)
-            #diffStateFile = open(diffStateFileName, 'wt')
-            diffStateFile = sys.stdout
-            differ = Differ(diffStateFile, ignore_added = True)
-            differ.visit(refState, srcState)
-            fail('state from call %u does not match %s' % (callNo, refStateFileName))
-
-    def getRefState(self, refStateFileName):
-        stream = open(refStateFileName, 'rt')
-        from jsondiff import load
-        state = load(stream)
-        self.adjustRefState(state)
-        return state
-
-    def getNamePrefix(self):
-        name = os.path.basename(self.ref_dump)
-        try:
-            index = name.index('.')
-        except ValueError:
-            pass
-        else:
-            name = name[:index]
-        return name
-
-    def saveState(self, state, filename):
-        s = json.dumps(state, sort_keys=True, indent=2)
-        open(filename, 'wt').write(s)
-
-    def retrace(self):
-        p = self._retrace()
-        p.wait()
-        if p.returncode != 0:
-            fail('retrace failed with code %i' % (p.returncode))
-
-    def getImage(self, callNo):
-        from PIL import Image
-        state = self.getState(callNo)
-        if self.doubleBuffer:
-            attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0']
-        else:
-            attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0']
-        imageObj = self.getFramebufferAttachment(state, attachments)
-        data = imageObj['__data__']
-        stream = StringIO(base64.b64decode(data))
-        im = Image.open(stream)
-        im.save('test.png')
-        return im
-
-    def getFramebufferAttachment(self, state, attachments):
-        framebufferObj = state['framebuffer']
-        for attachment in attachments:
-            try:
-                attachmentObj = framebufferObj[attachment]
-            except KeyError:
-                pass
-            else:
-                return attachmentObj
-        raise Exception("no attachment found")
-
-    def getState(self, callNo):
-        try:
-            state = self.stateCache[callNo]
-        except KeyError:
-            pass
-        else:
-            return state
-
-        p = self._retrace(['-D', str(callNo)])
-        state = json.load(p.stdout, strict=False)
-        p.wait()
-        if p.returncode != 0:
-            fail('retrace returned code %i' % (p.returncode))
-
-        self.adjustSrcState(state)
-
-        self.stateCache[callNo] = state
-
-        return state
-
-    def adjustSrcState(self, state):
-        # Do some adjustments on the obtained state to eliminate failures from
-        # bugs/issues outside of apitrace
-
-        try:
-            parameters = state['parameters']
-        except KeyError:
-            return
-
-        # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1
-        self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
-
-        # On Gallium 
-        if 'Gallium' in parameters['GL_RENDERER'].split():
-            # Gallium drivers have wrong defaults for draw/read buffer state
-            self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
-            self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK')
-            self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
-            self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
-            self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT')
-            self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
-
-    def adjustRefState(self, state):
-        # Do some adjustments on reference state to eliminate failures from
-        # bugs/issues outside of apitrace
-
-        try:
-            parameters = state['parameters']
-        except KeyError:
-            return
-
-        if platform.system() == 'Darwin':
-            # Mac OS X drivers fail on GL_COLOR_SUM
-            # XXX: investigate this
-            self.removeState(parameters, 'GL_COLOR_SUM')
-
-    def replaceState(self, obj, key, srcValue, dstValue):
-        try:
-            value = obj[key]
-        except KeyError:
-            pass
-        else:
-            if value == srcValue:
-                obj[key] = dstValue
-
-    def removeState(self, obj, key):
-        try:
-            del obj[key]
-        except KeyError:
-            pass
-
-    def _retrace(self, args = None, stdout=subprocess.PIPE):
-        retrace = self.api_map[self.api] + 'retrace'
-        cmd = [_get_build_program(retrace)]
-        if self.doubleBuffer:
-            cmd += ['-db']
-        else:
-            cmd += ['-sb']
-        if args:
-            cmd += args
-        cmd += [self.trace_file]
-        return popen(cmd, stdout=stdout)
-
-    def run(self):
-        self.runApp()
-        self.traceApp()
-        self.checkTrace()
-        self.retrace()
-
-        pass_()
-
-
-def main():
-    global options
-
-    default_apitrace = 'apitrace'
-    if platform.system() == 'Windows':
-        default_apitrace += '.exe'
-
-    # Parse command line options
-    optparser = optparse.OptionParser(
-        usage='\n\t%prog [options] -- [TRACE|PROGRAM] ...',
-        version='%%prog')
-    optparser.add_option(
-        '-v', '--verbose',
-        action="store_true",
-        dest="verbose", default=False,
-        help="verbose output")
-    optparser.add_option(
-        '-a', '--api', metavar='API',
-        type='string', dest='api', default='gl',
-        help='api to trace')
-    optparser.add_option(
-        '--apitrace', metavar='PROGRAM',
-        type='string', dest='apitrace', default=default_apitrace,
-        help='path to apitrace executable')
-    optparser.add_option(
-        '-C', '--directory', metavar='PATH',
-        type='string', dest='cwd', default=None,
-        help='change to directory')
-    optparser.add_option(
-        '-R', '--results', metavar='PATH',
-        type='string', dest='results', default='.',
-        help='results directory [default=%default]')
-    optparser.add_option(
-        '--ref-dump', metavar='PATH',
-        type='string', dest='ref_dump', default=None,
-        help='reference dump')
-
-    (options, args) = optparser.parse_args(sys.argv[1:])
-    if not args:
-        optparser.error('an argument must be specified')
-
-    if not os.path.exists(options.results):
-        os.makedirs(options.results)
-
-    print _get_scripts_path()
-
-    sys.path.insert(0, _get_scripts_path())
-
-    test = TestCase()
-    test.verbose = options.verbose
-
-    if args[0].endswith('.trace'):
-        test.trace_file = args[0]
-    else:
-        test.cmd = args
-    test.cwd = options.cwd
-    test.api = options.api
-    test.ref_dump = options.ref_dump
-    test.results = options.results
-
-    test.run()
-
-
-if __name__ == '__main__':
-    main()
diff --git a/tool_driver.py b/tool_driver.py
new file mode 100755 (executable)
index 0000000..63c10af
--- /dev/null
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+##########################################################################
+#
+# Copyright 2011 Jose Fonseca
+# All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+##########################################################################/
+
+'''Tool test driver.'''
+
+
+import os.path
+import subprocess
+import sys
+
+
+from base_driver import *
+
+
+class AsciiComparer:
+
+    def __init__(self, srcStream, refFileName, verbose=False):
+        self.srcStream = srcStream
+        self.refFileName = refFileName
+        if refFileName:
+            self.refStream = open(refFileName, 'rt')
+        else:
+            self.refStream = None
+    
+    def readLines(self, stream):
+        lines = []
+        for line in stream:
+            line = line[:-1]
+            lines.append(line)
+        return lines
+
+    def compare(self):
+        refLines = self.readLines(self.refStream)
+        srcLines = self.readLines(self.srcStream)
+
+        numLines = max(len(refLines), len(srcLines))
+
+        for lineNo in xrange(numLines):
+            try:
+                refLine = refLines[lineNo]
+            except IndexError:
+                fail('unexpected junk: %r' % self.srcLines[lineNo])
+
+            try:
+                srcLine = srcLines[lineNo]
+            except IndexError:
+                fail('unexpected EOF: %r expected' % refLine)
+
+            if refLine != srcLine:
+                fail('mismatch: expected %r but got %r' % (refLine ,srcLine))
+
+
+
+class TestCase:
+
+    cmd = None
+    cwd = None
+    ref_dump = None
+
+    verbose = False
+
+    def __init__(self):
+        pass
+    
+    def runTool(self):
+        '''Run the application standalone, skipping this test if it fails by
+        some reason.'''
+
+        if not self.cmd:
+            return
+
+        if self.ref_dump:
+            stdout = subprocess.PIPE
+        else:
+            stdout = None
+
+        cmd = [options.apitrace] + self.cmd
+        p = popen(cmd, cwd=self.cwd, stdout=stdout)
+
+        if self.ref_dump:
+            comparer = AsciiComparer(p.stdout, self.ref_dump, self.verbose)
+            comparer.compare()
+
+        p.wait()
+        if p.returncode != 0:
+            fail('tool returned code %i' % p.returncode)
+
+    def run(self):
+        self.runTool()
+
+        pass_()
+
+
+class ToolMain(Main):
+
+    def createOptParser(self):
+        optparser = Main.createOptParser(self)
+
+        optparser.add_option(
+            '--ref-dump', metavar='PATH',
+            type='string', dest='ref_dump', default=None,
+            help='reference dump')
+
+        return optparser
+
+    def main(self):
+        global options
+
+        (options, args) = self.parseOptions()
+
+        test = TestCase()
+        test.verbose = options.verbose
+
+        test.cmd = args
+        test.cwd = options.cwd
+        test.ref_dump = options.ref_dump
+
+        test.run()
+
+
+if __name__ == '__main__':
+    ToolMain().main()
index e25bc9fc5508a404e5b56081cee0fb916f65d349..ab18a523a64275ef42d09489c81be99e1f0a593b 100644 (file)
@@ -7,9 +7,10 @@ foreach (trace ${traces})
     add_test(
         NAME trace_${trace}
         COMMAND
-        python ${CMAKE_SOURCE_DIR}/driver.py
+        python ${CMAKE_SOURCE_DIR}/tool_driver.py
             --apitrace ${APITRACE_EXECUTABLE}
             --ref-dump ${CMAKE_CURRENT_SOURCE_DIR}/${trace}.ref.txt
-            ${CMAKE_CURRENT_SOURCE_DIR}/${trace}.trace
+            --
+            dump --call-nos=no ${CMAKE_CURRENT_SOURCE_DIR}/${trace}.trace
     )
 endforeach (trace)
index a7cf0a9356c3a9fee3f324c321674b2d03e2648c..acd3858995a1d2d3e64c27fb7d7200f67421e4db 100644 (file)
@@ -24,4 +24,5 @@ glEnd()
 glFlush()
 glFlush()
 glXSwapBuffers(dpy = 0x1da4360, drawable = 69206018)
+
 glXMakeContextCurrent(dpy = 0x1da4360, draw = 69206018, read = 69206018, ctx = 0x1e1cad0) = True