From: José Fonseca Date: Sat, 17 Mar 2012 14:13:33 +0000 (+0000) Subject: Split the driver code. X-Git-Url: https://git.cworth.org/git?p=apitrace-tests;a=commitdiff_plain;h=b8296ac85e510854e7ad57b02ddba25114b92775 Split the driver code. --- diff --git a/app_driver.py b/app_driver.py new file mode 100755 index 0000000..a44bade --- /dev/null +++ b/app_driver.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python +########################################################################## +# +# Copyright 2011 Jose Fonseca +# All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##########################################################################/ + +'''Application test driver.''' + + +import os.path +import platform +import re +import subprocess +import sys +import time +import json +import base64 + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + + +from base_driver import * + + +class TraceChecker: + + def __init__(self, srcStream, refFileName, verbose=False): + self.srcStream = srcStream + self.refFileName = refFileName + if refFileName: + self.refStream = open(refFileName, 'rt') + else: + self.refStream = None + self.verbose = verbose + self.doubleBuffer = False + self.callNo = 0 + self.refLine = '' + self.images = [] + self.states = [] + + call_re = re.compile(r'^([0-9]+) (\w+)\(') + + def check(self): + + swapbuffers = 0 + flushes = 0 + + srcLines = [] + self.consumeRefLine() + for line in self.srcStream: + line = line.rstrip() + if self.verbose: + sys.stdout.write(line + '\n') + mo = self.call_re.match(line) + if mo: + self.callNo = int(mo.group(1)) + function_name = mo.group(2) + if function_name.find('SwapBuffers') != -1 or \ + line.find('kCGLPFADoubleBuffer') != -1: + swapbuffers += 1 + if function_name in ('glFlush', 'glFinish'): + flushes += 1 + srcLine = line[mo.start(2):] + else: + srcLine = line + if self.refLine: + if srcLine == self.refLine: + self.consumeRefLine() + srcLines = [] + else: + srcLines.append(srcLine) + + if self.refLine: + if srcLines: + fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0])) + else: + fail('missing call %s' % self.refLine) + + if swapbuffers: + self.doubleBuffer = True + else: + self.doubleBuffer = False + + def consumeRefLine(self): + if not self.refStream: + self.refLine = '' + return + + while True: + line = self.refStream.readline() + if not line: + break + line = line.rstrip() + if line.startswith('#'): + self.handlePragma(line) + else: + break + self.refLine = line + + def handlePragma(self, line): + pragma, rest = line.split(None, 1) + if pragma == '#image': + imageFileName = self.getAbsPath(rest) + self.images.append((self.callNo, imageFileName)) + elif pragma == '#state': + stateFileName = self.getAbsPath(rest) + self.states.append((self.callNo, stateFileName)) + else: + assert False + + def getAbsPath(self, path): + '''Get the absolute from a path relative to the reference filename''' + return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path)) + + + +class TestCase: + + cmd = None + cwd = None + + api = 'gl' + max_frames = None + trace_file = None + + ref_dump = None + + doubleBuffer = True + + verbose = False + + threshold_precision = 12.0 + + def __init__(self): + self.stateCache = {} + + def runApp(self): + '''Run the application standalone, skipping this test if it fails by + some reason.''' + + if not self.cmd: + return + + p = popen(self.cmd, cwd=self.cwd) + p.wait() + if p.returncode: + skip('application returned code %i' % p.returncode) + + api_map = { + 'gl': 'gl', + 'egl_gl': 'egl', + 'egl_gles1': 'egl', + 'egl_gles2': 'egl', + } + + def traceApp(self): + if not self.cmd: + return + + if self.trace_file is None: + if self.ref_dump is not None: + name = self.ref_dump + else: + name = self.cmd[0] + name, ext = os.path.splitext(os.path.basename(name)) + while ext: + name, ext = os.path.splitext(os.path.basename(name)) + self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace')) + if os.path.exists(self.trace_file): + os.remove(self.trace_file) + else: + trace_dir = os.path.dirname(self.trace_file) + if not os.path.exists(trace_dir): + os.makedirs(trace_dir) + + cmd = self.cmd + env = os.environ.copy() + + cmd = [ + options.apitrace, 'trace', + '--api', self.api_map[self.api], + '--output', self.trace_file, + '--' + ] + cmd + if self.max_frames is not None: + env['TRACE_FRAMES'] = str(self.max_frames) + + p = popen(cmd, env=env, cwd=self.cwd) + p.wait() + + if not os.path.exists(self.trace_file): + fail('no trace file generated\n') + + def checkTrace(self): + cmd = [options.apitrace, 'dump', '--color=never', self.trace_file] + p = popen(cmd, stdout=subprocess.PIPE) + + checker = TraceChecker(p.stdout, self.ref_dump, self.verbose) + checker.check() + p.wait() + if p.returncode != 0: + fail('`apitrace dump` returned code %i' % p.returncode) + + self.doubleBuffer = checker.doubleBuffer + + for callNo, refImageFileName in checker.images: + self.checkImage(callNo, refImageFileName) + for callNo, refStateFileName in checker.states: + self.checkState(callNo, refStateFileName) + + def checkImage(self, callNo, refImageFileName): + try: + from PIL import Image + except ImportError: + return + + srcImage = self.getImage(callNo) + refImage = Image.open(refImageFileName) + + from snapdiff import Comparer + comparer = Comparer(refImage, srcImage) + precision = comparer.precision(filter=True) + sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName)) + if precision < self.threshold_precision: + prefix = '%s.%u' % (self.getNamePrefix(), callNo) + srcImageFileName = prefix + '.src.png' + srcImage.save(srcImageFileName) + diffImageFileName = prefix + '.diff.png' + comparer.write_diff(diffImageFileName) + fail('snapshot from call %u does not match %s' % (callNo, refImageFileName)) + + def checkState(self, callNo, refStateFileName): + srcState = self.getState(callNo) + refState = self.getRefState(refStateFileName) + + from jsondiff import Comparer, Differ + comparer = Comparer(ignore_added = True) + match = comparer.visit(refState, srcState) + if not match: + prefix = '%s.%u' % (self.getNamePrefix(), callNo) + srcStateFileName = prefix + '.src.json' + diffStateFileName = prefix + '.diff.json' + self.saveState(srcState, srcStateFileName) + #diffStateFile = open(diffStateFileName, 'wt') + diffStateFile = sys.stdout + differ = Differ(diffStateFile, ignore_added = True) + differ.visit(refState, srcState) + fail('state from call %u does not match %s' % (callNo, refStateFileName)) + + def getRefState(self, refStateFileName): + stream = open(refStateFileName, 'rt') + from jsondiff import load + state = load(stream) + self.adjustRefState(state) + return state + + def getNamePrefix(self): + name = os.path.basename(self.ref_dump) + try: + index = name.index('.') + except ValueError: + pass + else: + name = name[:index] + return name + + def saveState(self, state, filename): + s = json.dumps(state, sort_keys=True, indent=2) + open(filename, 'wt').write(s) + + def retrace(self): + p = self._retrace() + p.wait() + if p.returncode != 0: + fail('retrace failed with code %i' % (p.returncode)) + + def getImage(self, callNo): + from PIL import Image + state = self.getState(callNo) + if self.doubleBuffer: + attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0'] + else: + attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0'] + imageObj = self.getFramebufferAttachment(state, attachments) + data = imageObj['__data__'] + stream = StringIO(base64.b64decode(data)) + im = Image.open(stream) + im.save('test.png') + return im + + def getFramebufferAttachment(self, state, attachments): + framebufferObj = state['framebuffer'] + for attachment in attachments: + try: + attachmentObj = framebufferObj[attachment] + except KeyError: + pass + else: + return attachmentObj + raise Exception("no attachment found") + + def getState(self, callNo): + try: + state = self.stateCache[callNo] + except KeyError: + pass + else: + return state + + p = self._retrace(['-D', str(callNo)]) + state = json.load(p.stdout, strict=False) + p.wait() + if p.returncode != 0: + fail('retrace returned code %i' % (p.returncode)) + + self.adjustSrcState(state) + + self.stateCache[callNo] = state + + return state + + def adjustSrcState(self, state): + # Do some adjustments on the obtained state to eliminate failures from + # bugs/issues outside of apitrace + + try: + parameters = state['parameters'] + except KeyError: + return + + # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1 + self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1) + + # On Gallium + if 'Gallium' in parameters['GL_RENDERER'].split(): + # Gallium drivers have wrong defaults for draw/read buffer state + self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') + self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT') + self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') + + def adjustRefState(self, state): + # Do some adjustments on reference state to eliminate failures from + # bugs/issues outside of apitrace + + try: + parameters = state['parameters'] + except KeyError: + return + + if platform.system() == 'Darwin': + # Mac OS X drivers fail on GL_COLOR_SUM + # XXX: investigate this + self.removeState(parameters, 'GL_COLOR_SUM') + + def replaceState(self, obj, key, srcValue, dstValue): + try: + value = obj[key] + except KeyError: + pass + else: + if value == srcValue: + obj[key] = dstValue + + def removeState(self, obj, key): + try: + del obj[key] + except KeyError: + pass + + def _retrace(self, args = None, stdout=subprocess.PIPE): + retrace = self.api_map[self.api] + 'retrace' + cmd = [get_build_program(retrace)] + if self.doubleBuffer: + cmd += ['-db'] + else: + cmd += ['-sb'] + if args: + cmd += args + cmd += [self.trace_file] + return popen(cmd, stdout=stdout) + + def run(self): + self.runApp() + self.traceApp() + self.checkTrace() + self.retrace() + + pass_() + + +class AppMain(Main): + + def createOptParser(self): + optparser = Main.createOptParser(self) + + optparser.add_option( + '-a', '--api', metavar='API', + type='string', dest='api', default='gl', + help='api to trace') + optparser.add_option( + '-R', '--results', metavar='PATH', + type='string', dest='results', default='.', + help='results directory [default=%default]') + optparser.add_option( + '--ref-dump', metavar='PATH', + type='string', dest='ref_dump', default=None, + help='reference dump') + + return optparser + + def main(self): + global options + + (options, args) = self.parseOptions() + + if not os.path.exists(options.results): + os.makedirs(options.results) + + test = TestCase() + test.verbose = options.verbose + + test.cmd = args + test.cwd = options.cwd + test.api = options.api + test.ref_dump = options.ref_dump + test.results = options.results + + test.run() + + +if __name__ == '__main__': + AppMain().main() diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 5736c21..1793709 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -13,7 +13,7 @@ function (ADD_APP_TEST) add_test( NAME app_${TEST_NAME} COMMAND - python ${CMAKE_SOURCE_DIR}/driver.py + python ${CMAKE_SOURCE_DIR}/app_driver.py --apitrace ${APITRACE_EXECUTABLE} --api ${api} --ref-dump ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_REF} diff --git a/base_driver.py b/base_driver.py new file mode 100644 index 0000000..51032bd --- /dev/null +++ b/base_driver.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python +########################################################################## +# +# Copyright 2011-2012 Jose Fonseca +# All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##########################################################################/ + +'''Common test driver code.''' + + +import optparse +import os.path +import platform +import subprocess +import sys + + +def _exit(status, code, reason=None): + if reason is None: + reason = '' + else: + reason = ' (%s)' % reason + sys.stdout.write('%s%s\n' % (status, reason)) + sys.exit(code) + +def fail(reason=None): + _exit('FAIL', 1, reason) + +def skip(reason=None): + _exit('SKIP', 0, reason) + +def pass_(reason=None): + _exit('PASS', 0, reason) + + +def popen(command, *args, **kwargs): + if kwargs.get('cwd', None) is not None: + sys.stdout.write('cd %s && ' % kwargs['cwd']) + + try: + env = kwargs.pop('env') + except KeyError: + env = None + else: + names = env.keys() + names.sort() + for name in names: + value = env[name] + if value != os.environ.get(name, None): + sys.stdout.write('%s=%s ' % (name, value)) + env[name] = str(value) + + sys.stdout.write(' '.join(command) + '\n') + sys.stdout.flush() + + return subprocess.Popen(command, *args, env=env, **kwargs) + + +def which(executable): + dirs = os.environ['PATH'].split(os.path.pathsep) + for dir in dirs: + path = os.path.join(dir, executable) + if os.path.exists(path): + return path + return None + + +def get_bin_path(): + if os.path.exists(options.apitrace): + apitrace_abspath = os.path.abspath(options.apitrace) + else: + apitrace_abspath = which(options.apitrace) + if apitrace_abspath is None: + sys.stderr.write('error: could not determine the absolute path of\n' % options.apitrace) + sys.exit(1) + return os.path.dirname(apitrace_abspath) + + +def get_build_program(program): + bin_path = get_bin_path() + if platform.system() == 'Windows': + program += '.exe' + path = os.path.join(bin_path, program) + if not os.path.exists(path): + sys.stderr.write('error: %s does not exist\n' % path) + sys.exit(1) + return path + + +def get_scripts_path(): + bin_path = get_bin_path() + + try_paths = [ + 'scripts', + '../lib/scripts', + '../lib/apitrace/scripts', + ] + + for try_path in try_paths: + path = os.path.join(bin_path, try_path) + if os.path.exists(path): + return os.path.abspath(path) + + sys.stderr.write('error: could not find scripts directory\n') + sys.exit(1) + + +class Main: + global options + + def createOptParser(self): + default_apitrace = 'apitrace' + if platform.system() == 'Windows': + default_apitrace += '.exe' + + # Parse command line options + optparser = optparse.OptionParser( + usage='\n\t%prog [OPTIONS] -- [ARGS] ...', + version='%%prog') + optparser.add_option( + '-v', '--verbose', + action="store_true", + dest="verbose", default=False, + help="verbose output") + optparser.add_option( + '--apitrace', metavar='PROGRAM', + type='string', dest='apitrace', default=default_apitrace, + help='path to apitrace executable') + optparser.add_option( + '-C', '--directory', metavar='PATH', + type='string', dest='cwd', default=None, + help='change to directory') + + return optparser + + def parseOptions(self): + global options + + optparser = self.createOptParser() + (options, args) = optparser.parse_args(sys.argv[1:]) + if not args: + optparser.error('an argument must be specified') + + print get_scripts_path() + + sys.path.insert(0, get_scripts_path()) + + return options, args + + def main(self): + raise NotImplementedError + diff --git a/driver.py b/driver.py deleted file mode 100755 index d5789e3..0000000 --- a/driver.py +++ /dev/null @@ -1,568 +0,0 @@ -#!/usr/bin/env python -########################################################################## -# -# Copyright 2011 Jose Fonseca -# All Rights Reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# -##########################################################################/ - -'''Main test driver.''' - - -import optparse -import os.path -import platform -import re -import subprocess -import sys -import time -import json -import base64 - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO - - -def _exit(status, code, reason=None): - if reason is None: - reason = '' - else: - reason = ' (%s)' % reason - sys.stdout.write('%s%s\n' % (status, reason)) - sys.exit(code) - -def fail(reason=None): - _exit('FAIL', 1, reason) - -def skip(reason=None): - _exit('SKIP', 0, reason) - -def pass_(reason=None): - _exit('PASS', 0, reason) - - -def popen(command, *args, **kwargs): - if kwargs.get('cwd', None) is not None: - sys.stdout.write('cd %s && ' % kwargs['cwd']) - - try: - env = kwargs.pop('env') - except KeyError: - env = None - else: - names = env.keys() - names.sort() - for name in names: - value = env[name] - if value != os.environ.get(name, None): - sys.stdout.write('%s=%s ' % (name, value)) - env[name] = str(value) - - sys.stdout.write(' '.join(command) + '\n') - sys.stdout.flush() - - return subprocess.Popen(command, *args, env=env, **kwargs) - - -def which(executable): - dirs = os.environ['PATH'].split(os.path.pathsep) - for dir in dirs: - path = os.path.join(dir, executable) - if os.path.exists(path): - return path - return None - - -def _get_bin_path(): - if os.path.exists(options.apitrace): - apitrace_abspath = os.path.abspath(options.apitrace) - else: - apitrace_abspath = which(options.apitrace) - if apitrace_abspath is None: - sys.stderr.write('error: could not determine the absolute path of\n' % options.apitrace) - sys.exit(1) - return os.path.dirname(apitrace_abspath) - - -def _get_build_program(program): - bin_path = _get_bin_path() - if platform.system() == 'Windows': - program += '.exe' - path = os.path.join(bin_path, program) - if not os.path.exists(path): - sys.stderr.write('error: %s does not exist\n' % path) - sys.exit(1) - return path - -def _get_scripts_path(): - bin_path = _get_bin_path() - - try_paths = [ - 'scripts', - '../lib/scripts', - '../lib/apitrace/scripts', - ] - - for try_path in try_paths: - path = os.path.join(bin_path, try_path) - if os.path.exists(path): - return os.path.abspath(path) - - sys.stderr.write('error: could not find scripts directory\n') - sys.exit(1) - - -class TraceChecker: - - def __init__(self, srcStream, refFileName, verbose=False): - self.srcStream = srcStream - self.refFileName = refFileName - if refFileName: - self.refStream = open(refFileName, 'rt') - else: - self.refStream = None - self.verbose = verbose - self.doubleBuffer = False - self.callNo = 0 - self.refLine = '' - self.images = [] - self.states = [] - - call_re = re.compile(r'^([0-9]+) (\w+)\(') - - def check(self): - - swapbuffers = 0 - flushes = 0 - - srcLines = [] - self.consumeRefLine() - for line in self.srcStream: - line = line.rstrip() - if self.verbose: - sys.stdout.write(line + '\n') - mo = self.call_re.match(line) - if mo: - self.callNo = int(mo.group(1)) - function_name = mo.group(2) - if function_name.find('SwapBuffers') != -1 or \ - line.find('kCGLPFADoubleBuffer') != -1: - swapbuffers += 1 - if function_name in ('glFlush', 'glFinish'): - flushes += 1 - srcLine = line[mo.start(2):] - else: - srcLine = line - if self.refLine: - if srcLine == self.refLine: - self.consumeRefLine() - srcLines = [] - else: - srcLines.append(srcLine) - - if self.refLine: - if srcLines: - fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0])) - else: - fail('missing call %s' % self.refLine) - - if swapbuffers: - self.doubleBuffer = True - else: - self.doubleBuffer = False - - def consumeRefLine(self): - if not self.refStream: - self.refLine = '' - return - - while True: - line = self.refStream.readline() - if not line: - break - line = line.rstrip() - if line.startswith('#'): - self.handlePragma(line) - else: - break - self.refLine = line - - def handlePragma(self, line): - pragma, rest = line.split(None, 1) - if pragma == '#image': - imageFileName = self.getAbsPath(rest) - self.images.append((self.callNo, imageFileName)) - elif pragma == '#state': - stateFileName = self.getAbsPath(rest) - self.states.append((self.callNo, stateFileName)) - else: - assert False - - def getAbsPath(self, path): - '''Get the absolute from a path relative to the reference filename''' - return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path)) - - - -class TestCase: - - cmd = None - cwd = None - - api = 'gl' - max_frames = None - trace_file = None - - ref_dump = None - - doubleBuffer = True - - verbose = False - - threshold_precision = 12.0 - - def __init__(self): - self.stateCache = {} - - def runApp(self): - '''Run the application standalone, skipping this test if it fails by - some reason.''' - - if not self.cmd: - return - - p = popen(self.cmd, cwd=self.cwd) - p.wait() - if p.returncode: - skip('application returned code %i' % p.returncode) - - api_map = { - 'gl': 'gl', - 'egl_gl': 'egl', - 'egl_gles1': 'egl', - 'egl_gles2': 'egl', - } - - def traceApp(self): - if not self.cmd: - return - - if self.trace_file is None: - if self.ref_dump is not None: - name = self.ref_dump - else: - name = self.cmd[0] - name, ext = os.path.splitext(os.path.basename(name)) - while ext: - name, ext = os.path.splitext(os.path.basename(name)) - self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace')) - if os.path.exists(self.trace_file): - os.remove(self.trace_file) - else: - trace_dir = os.path.dirname(self.trace_file) - if not os.path.exists(trace_dir): - os.makedirs(trace_dir) - - cmd = self.cmd - env = os.environ.copy() - - cmd = [ - options.apitrace, 'trace', - '--api', self.api_map[self.api], - '--output', self.trace_file, - '--' - ] + cmd - if self.max_frames is not None: - env['TRACE_FRAMES'] = str(self.max_frames) - - p = popen(cmd, env=env, cwd=self.cwd) - p.wait() - - if not os.path.exists(self.trace_file): - fail('no trace file generated\n') - - def checkTrace(self): - cmd = [options.apitrace, 'dump', '--color=never', self.trace_file] - p = popen(cmd, stdout=subprocess.PIPE) - - checker = TraceChecker(p.stdout, self.ref_dump, self.verbose) - checker.check() - p.wait() - if p.returncode != 0: - fail('`apitrace dump` returned code %i' % p.returncode) - - self.doubleBuffer = checker.doubleBuffer - - for callNo, refImageFileName in checker.images: - self.checkImage(callNo, refImageFileName) - for callNo, refStateFileName in checker.states: - self.checkState(callNo, refStateFileName) - - def checkImage(self, callNo, refImageFileName): - try: - from PIL import Image - except ImportError: - return - - srcImage = self.getImage(callNo) - refImage = Image.open(refImageFileName) - - from snapdiff import Comparer - comparer = Comparer(refImage, srcImage) - precision = comparer.precision(filter=True) - sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName)) - if precision < self.threshold_precision: - prefix = '%s.%u' % (self.getNamePrefix(), callNo) - srcImageFileName = prefix + '.src.png' - srcImage.save(srcImageFileName) - diffImageFileName = prefix + '.diff.png' - comparer.write_diff(diffImageFileName) - fail('snapshot from call %u does not match %s' % (callNo, refImageFileName)) - - def checkState(self, callNo, refStateFileName): - srcState = self.getState(callNo) - refState = self.getRefState(refStateFileName) - - from jsondiff import Comparer, Differ - comparer = Comparer(ignore_added = True) - match = comparer.visit(refState, srcState) - if not match: - prefix = '%s.%u' % (self.getNamePrefix(), callNo) - srcStateFileName = prefix + '.src.json' - diffStateFileName = prefix + '.diff.json' - self.saveState(srcState, srcStateFileName) - #diffStateFile = open(diffStateFileName, 'wt') - diffStateFile = sys.stdout - differ = Differ(diffStateFile, ignore_added = True) - differ.visit(refState, srcState) - fail('state from call %u does not match %s' % (callNo, refStateFileName)) - - def getRefState(self, refStateFileName): - stream = open(refStateFileName, 'rt') - from jsondiff import load - state = load(stream) - self.adjustRefState(state) - return state - - def getNamePrefix(self): - name = os.path.basename(self.ref_dump) - try: - index = name.index('.') - except ValueError: - pass - else: - name = name[:index] - return name - - def saveState(self, state, filename): - s = json.dumps(state, sort_keys=True, indent=2) - open(filename, 'wt').write(s) - - def retrace(self): - p = self._retrace() - p.wait() - if p.returncode != 0: - fail('retrace failed with code %i' % (p.returncode)) - - def getImage(self, callNo): - from PIL import Image - state = self.getState(callNo) - if self.doubleBuffer: - attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0'] - else: - attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0'] - imageObj = self.getFramebufferAttachment(state, attachments) - data = imageObj['__data__'] - stream = StringIO(base64.b64decode(data)) - im = Image.open(stream) - im.save('test.png') - return im - - def getFramebufferAttachment(self, state, attachments): - framebufferObj = state['framebuffer'] - for attachment in attachments: - try: - attachmentObj = framebufferObj[attachment] - except KeyError: - pass - else: - return attachmentObj - raise Exception("no attachment found") - - def getState(self, callNo): - try: - state = self.stateCache[callNo] - except KeyError: - pass - else: - return state - - p = self._retrace(['-D', str(callNo)]) - state = json.load(p.stdout, strict=False) - p.wait() - if p.returncode != 0: - fail('retrace returned code %i' % (p.returncode)) - - self.adjustSrcState(state) - - self.stateCache[callNo] = state - - return state - - def adjustSrcState(self, state): - # Do some adjustments on the obtained state to eliminate failures from - # bugs/issues outside of apitrace - - try: - parameters = state['parameters'] - except KeyError: - return - - # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1 - self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1) - - # On Gallium - if 'Gallium' in parameters['GL_RENDERER'].split(): - # Gallium drivers have wrong defaults for draw/read buffer state - self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') - self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK') - self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') - self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') - self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT') - self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') - - def adjustRefState(self, state): - # Do some adjustments on reference state to eliminate failures from - # bugs/issues outside of apitrace - - try: - parameters = state['parameters'] - except KeyError: - return - - if platform.system() == 'Darwin': - # Mac OS X drivers fail on GL_COLOR_SUM - # XXX: investigate this - self.removeState(parameters, 'GL_COLOR_SUM') - - def replaceState(self, obj, key, srcValue, dstValue): - try: - value = obj[key] - except KeyError: - pass - else: - if value == srcValue: - obj[key] = dstValue - - def removeState(self, obj, key): - try: - del obj[key] - except KeyError: - pass - - def _retrace(self, args = None, stdout=subprocess.PIPE): - retrace = self.api_map[self.api] + 'retrace' - cmd = [_get_build_program(retrace)] - if self.doubleBuffer: - cmd += ['-db'] - else: - cmd += ['-sb'] - if args: - cmd += args - cmd += [self.trace_file] - return popen(cmd, stdout=stdout) - - def run(self): - self.runApp() - self.traceApp() - self.checkTrace() - self.retrace() - - pass_() - - -def main(): - global options - - default_apitrace = 'apitrace' - if platform.system() == 'Windows': - default_apitrace += '.exe' - - # Parse command line options - optparser = optparse.OptionParser( - usage='\n\t%prog [options] -- [TRACE|PROGRAM] ...', - version='%%prog') - optparser.add_option( - '-v', '--verbose', - action="store_true", - dest="verbose", default=False, - help="verbose output") - optparser.add_option( - '-a', '--api', metavar='API', - type='string', dest='api', default='gl', - help='api to trace') - optparser.add_option( - '--apitrace', metavar='PROGRAM', - type='string', dest='apitrace', default=default_apitrace, - help='path to apitrace executable') - optparser.add_option( - '-C', '--directory', metavar='PATH', - type='string', dest='cwd', default=None, - help='change to directory') - optparser.add_option( - '-R', '--results', metavar='PATH', - type='string', dest='results', default='.', - help='results directory [default=%default]') - optparser.add_option( - '--ref-dump', metavar='PATH', - type='string', dest='ref_dump', default=None, - help='reference dump') - - (options, args) = optparser.parse_args(sys.argv[1:]) - if not args: - optparser.error('an argument must be specified') - - if not os.path.exists(options.results): - os.makedirs(options.results) - - print _get_scripts_path() - - sys.path.insert(0, _get_scripts_path()) - - test = TestCase() - test.verbose = options.verbose - - if args[0].endswith('.trace'): - test.trace_file = args[0] - else: - test.cmd = args - test.cwd = options.cwd - test.api = options.api - test.ref_dump = options.ref_dump - test.results = options.results - - test.run() - - -if __name__ == '__main__': - main() diff --git a/tool_driver.py b/tool_driver.py new file mode 100755 index 0000000..63c10af --- /dev/null +++ b/tool_driver.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +########################################################################## +# +# Copyright 2011 Jose Fonseca +# All Rights Reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +##########################################################################/ + +'''Tool test driver.''' + + +import os.path +import subprocess +import sys + + +from base_driver import * + + +class AsciiComparer: + + def __init__(self, srcStream, refFileName, verbose=False): + self.srcStream = srcStream + self.refFileName = refFileName + if refFileName: + self.refStream = open(refFileName, 'rt') + else: + self.refStream = None + + def readLines(self, stream): + lines = [] + for line in stream: + line = line[:-1] + lines.append(line) + return lines + + def compare(self): + refLines = self.readLines(self.refStream) + srcLines = self.readLines(self.srcStream) + + numLines = max(len(refLines), len(srcLines)) + + for lineNo in xrange(numLines): + try: + refLine = refLines[lineNo] + except IndexError: + fail('unexpected junk: %r' % self.srcLines[lineNo]) + + try: + srcLine = srcLines[lineNo] + except IndexError: + fail('unexpected EOF: %r expected' % refLine) + + if refLine != srcLine: + fail('mismatch: expected %r but got %r' % (refLine ,srcLine)) + + + +class TestCase: + + cmd = None + cwd = None + ref_dump = None + + verbose = False + + def __init__(self): + pass + + def runTool(self): + '''Run the application standalone, skipping this test if it fails by + some reason.''' + + if not self.cmd: + return + + if self.ref_dump: + stdout = subprocess.PIPE + else: + stdout = None + + cmd = [options.apitrace] + self.cmd + p = popen(cmd, cwd=self.cwd, stdout=stdout) + + if self.ref_dump: + comparer = AsciiComparer(p.stdout, self.ref_dump, self.verbose) + comparer.compare() + + p.wait() + if p.returncode != 0: + fail('tool returned code %i' % p.returncode) + + def run(self): + self.runTool() + + pass_() + + +class ToolMain(Main): + + def createOptParser(self): + optparser = Main.createOptParser(self) + + optparser.add_option( + '--ref-dump', metavar='PATH', + type='string', dest='ref_dump', default=None, + help='reference dump') + + return optparser + + def main(self): + global options + + (options, args) = self.parseOptions() + + test = TestCase() + test.verbose = options.verbose + + test.cmd = args + test.cwd = options.cwd + test.ref_dump = options.ref_dump + + test.run() + + +if __name__ == '__main__': + ToolMain().main() diff --git a/traces/CMakeLists.txt b/traces/CMakeLists.txt index e25bc9f..ab18a52 100644 --- a/traces/CMakeLists.txt +++ b/traces/CMakeLists.txt @@ -7,9 +7,10 @@ foreach (trace ${traces}) add_test( NAME trace_${trace} COMMAND - python ${CMAKE_SOURCE_DIR}/driver.py + python ${CMAKE_SOURCE_DIR}/tool_driver.py --apitrace ${APITRACE_EXECUTABLE} --ref-dump ${CMAKE_CURRENT_SOURCE_DIR}/${trace}.ref.txt - ${CMAKE_CURRENT_SOURCE_DIR}/${trace}.trace + -- + dump --call-nos=no ${CMAKE_CURRENT_SOURCE_DIR}/${trace}.trace ) endforeach (trace) diff --git a/traces/zlib-no-eof.ref.txt b/traces/zlib-no-eof.ref.txt index a7cf0a9..acd3858 100644 --- a/traces/zlib-no-eof.ref.txt +++ b/traces/zlib-no-eof.ref.txt @@ -24,4 +24,5 @@ glEnd() glFlush() glFlush() glXSwapBuffers(dpy = 0x1da4360, drawable = 69206018) + glXMakeContextCurrent(dpy = 0x1da4360, draw = 69206018, read = 69206018, ctx = 0x1e1cad0) = True