X-Git-Url: https://git.cworth.org/git?p=apitrace-tests;a=blobdiff_plain;f=driver.py;h=9894c9c80447cc806401361b9bf497b8fc4267ab;hp=60699099e2b9623693131839c292f44e26df989a;hb=25dbf53f26766ce9ff3adea306cbd0996b45574a;hpb=4764b4045c5fb51d2c3ee5cabedc721b019b4f19 diff --git a/driver.py b/driver.py index 6069909..9894c9c 100755 --- a/driver.py +++ b/driver.py @@ -35,18 +35,56 @@ import shutil import subprocess import sys import time +import json +import base64 + +from PIL import Image + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + + +def _exit(status, code, reason=None): + if reason is None: + reason = '' + else: + reason = ' (%s)' % reason + sys.stdout.write('%s%s\n' % (status, reason)) + sys.exit(code) + +def fail(reason=None): + _exit('FAIL', 1, reason) + +def skip(reason=None): + _exit('SKIP', 0, reason) + +def pass_(reason=None): + _exit('PASS', 0, reason) def popen(command, *args, **kwargs): if kwargs.get('cwd', None) is not None: sys.stdout.write('cd %s && ' % kwargs['cwd']) - if 'env' in kwargs: - for name, value in kwargs['env'].iteritems(): + + try: + env = kwargs.pop('env') + except KeyError: + env = None + else: + names = env.keys() + names.sort() + for name in names: + value = env[name] if value != os.environ.get(name, None): sys.stdout.write('%s=%s ' % (name, value)) + env[name] = str(value) + sys.stdout.write(' '.join(command) + '\n') sys.stdout.flush() - return subprocess.Popen(command, *args, **kwargs) + + return subprocess.Popen(command, *args, env=env, **kwargs) def _get_build_path(path): @@ -62,6 +100,106 @@ def _get_build_program(program): program += '.exe' return _get_build_path(program) +def _get_source_path(path): + cache = _get_build_path('CMakeCache.txt') + for line in open(cache, 'rt'): + if line.startswith('CMAKE_HOME_DIRECTORY:INTERNAL='): + _, source_root = line.strip().split('=', 1) + return os.path.join(source_root, path) + return None + + +class TraceChecker: + + def __init__(self, srcStream, refFileName, verbose=False): + self.srcStream = srcStream + self.refFileName = refFileName + if refFileName: + self.refStream = open(refFileName, 'rt') + else: + self.refStream = None + self.verbose = verbose + self.doubleBuffer = False + self.callNo = 0 + self.refLine = '' + self.images = [] + self.states = [] + + call_re = re.compile(r'^([0-9]+) (\w+)\(') + + def check(self): + + swapbuffers = 0 + flushes = 0 + + srcLines = [] + self.consumeRefLine() + for line in self.srcStream: + line = line.rstrip() + if self.verbose: + sys.stdout.write(line + '\n') + mo = self.call_re.match(line) + if mo: + self.callNo = int(mo.group(1)) + function_name = mo.group(2) + if function_name.find('SwapBuffers') != -1 or \ + line.find('kCGLPFADoubleBuffer') != -1: + swapbuffers += 1 + if function_name in ('glFlush', 'glFinish'): + flushes += 1 + srcLine = line[mo.start(2):] + else: + srcLine = line + if self.refLine: + if srcLine == self.refLine: + self.consumeRefLine() + srcLines = [] + else: + srcLines.append(srcLine) + + if self.refLine: + if srcLines: + fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0])) + else: + fail('missing call %s' % self.refLine) + + if swapbuffers: + self.doubleBuffer = True + else: + self.doubleBuffer = False + + def consumeRefLine(self): + if not self.refStream: + self.refLine = '' + return + + while True: + line = self.refStream.readline() + if not line: + break + line = line.rstrip() + if line.startswith('#'): + self.handlePragma(line) + else: + break + self.refLine = line + + def handlePragma(self, line): + pragma, rest = line.split(None, 1) + if pragma == '#image': + imageFileName = self.getAbsPath(rest) + self.images.append((self.callNo, imageFileName)) + elif pragma == '#state': + stateFileName = self.getAbsPath(rest) + self.states.append((self.callNo, stateFileName)) + else: + assert False + + def getAbsPath(self, path): + '''Get the absolute from a path relative to the reference filename''' + return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path)) + + class TestCase: @@ -72,16 +210,26 @@ class TestCase: max_frames = None trace_file = None - expected_dump = None + ref_dump = None + + doubleBuffer = True + + verbose = False + + def __init__(self): + self.stateCache = {} + + def runApp(self): + '''Run the application standalone, skipping this test if it fails by + some reason.''' - def standalone(self): if not self.cmd: return p = popen(self.cmd, cwd=self.cwd) p.wait() if p.returncode: - self.skip('application returned code %i' % p.returncode) + skip('application returned code %i' % p.returncode) api_map = { 'gl': 'gl', @@ -90,12 +238,12 @@ class TestCase: 'egl_gles2': 'egl', } - def trace(self): + def traceApp(self): if not self.cmd: return if self.trace_file is None: - name = os.path.basename(self.cmd[0]) + name, ext = os.path.splitext(os.path.basename(self.cmd[0])) self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace')) if os.path.exists(self.trace_file): os.remove(self.trace_file) @@ -113,7 +261,7 @@ class TestCase: wrapper = _get_build_path('wrappers/opengl32.dll') local_wrapper = os.path.join(os.path.dirname(self.cmd[0]), os.path.basename(wrapper)) shutil.copy(wrapper, local_wrapper) - env['TRACE_FILE'] = self.trace_file + env['TRACE_FILE'] = str(self.trace_file) else: apitrace = _get_build_program('apitrace') cmd = [ @@ -133,142 +281,198 @@ class TestCase: os.remove(local_wrapper) if not os.path.exists(self.trace_file): - self.fail('no trace file generated\n') + fail('no trace file generated\n') - call_re = re.compile(r'^([0-9]+) (\w+)\(') - - def dump(self): - + def checkTrace(self): cmd = [_get_build_program('apitrace'), 'dump', '--color=never', self.trace_file] p = popen(cmd, stdout=subprocess.PIPE) - swapbuffers = 0 - flushes = 0 + checker = TraceChecker(p.stdout, self.ref_dump, self.verbose) + checker.check() + p.wait() + if p.returncode != 0: + fail('`apitrace dump` returned code %i' % p.returncode) + + self.doubleBuffer = checker.doubleBuffer + + for callNo, refImageFileName in checker.images: + self.checkImage(callNo, refImageFileName) + for callNo, refStateFileName in checker.states: + self.checkState(callNo, refStateFileName) + + def checkImage(self, callNo, refImageFileName): + srcImage = self.getImage(callNo) + refImage = Image.open(refImageFileName) + + from snapdiff import Comparer + comparer = Comparer(refImage, srcImage) + match = comparer.ae() + if not match: + prefix = '%s.%u' % (self.getNamePrefix(), callNo) + srcImageFileName = prefix + '.src.png' + srcImage.save(srcImageFileName) + diffImageFileName = prefix + '.diff.png' + comparer.write_diff(diffImageFileName) + fail('snapshot from call %u does not match %s' % (callNo, refImageFileName)) + + def checkState(self, callNo, refStateFileName): + srcState = self.getState(callNo) + refState = self.getRefState(refStateFileName) + + from jsondiff import Comparer, Differ + comparer = Comparer(ignore_added = True) + match = comparer.visit(refState, srcState) + if not match: + prefix = '%s.%u' % (self.getNamePrefix(), callNo) + srcStateFileName = prefix + '.src.json' + diffStateFileName = prefix + '.diff.json' + self.saveState(srcState, srcStateFileName) + #diffStateFile = open(diffStateFileName, 'wt') + diffStateFile = sys.stdout + differ = Differ(diffStateFile, ignore_added = True) + differ.visit(refState, srcState) + fail('state from call %u does not match %s' % (callNo, refStateFileName)) + + def getRefState(self, refStateFileName): + stream = open(refStateFileName, 'rt') + from jsondiff import load + state = load(stream) + return state + + def getNamePrefix(self): + name = os.path.basename(self.ref_dump) + try: + index = name.index('.') + except ValueError: + pass + else: + name = name[:index] + return name - ref_line = '' - src_lines = [] - if self.ref_dump is not None: - ref = open(self.ref_dump, 'rt') - ref_line = ref.readline().rstrip() - for line in p.stdout: - line = line.rstrip() - sys.stdout.write(line + '\n') - mo = self.call_re.match(line) - if mo: - call_no = int(mo.group(1)) - function_name = mo.group(2) - if function_name == 'glXSwapBuffers': - swapbuffers += 1 - if function_name in ('glFlush', 'glFinish'): - flushes += 1 - src_line = line[mo.start(2):] - else: - src_line = line - if ref_line: - if src_line == ref_line: - sys.stdout.write(src_line + '\n') - ref_line = ref.readline().rstrip() - src_lines = [] - else: - src_lines.append(src_line) + def saveState(self, state, filename): + s = json.dumps(state, sort_keys=True, indent=2) + open(filename, 'wt').write(s) + def retrace(self): + p = self._retrace() p.wait() if p.returncode != 0: - self.fail('`apitrace dump` returned code %i' % p.returncode) - if ref_line: - if src_lines: - self.fail('missing call `%s` (found `%s`)' % (ref_line, src_lines[0])) + fail('retrace failed with code %i' % (p.returncode)) + + def getImage(self, callNo): + state = self.getState(callNo) + if self.doubleBuffer: + attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0'] + else: + attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0'] + imageObj = self.getFramebufferAttachment(state, attachments) + data = imageObj['__data__'] + stream = StringIO(base64.b64decode(data)) + im = Image.open(stream) + im.save('test.png') + return im + + def getFramebufferAttachment(self, state, attachments): + framebufferObj = state['framebuffer'] + for attachment in attachments: + try: + attachmentObj = framebufferObj[attachment] + except KeyError: + pass else: - self.fail('missing call %s' % ref_line) + return attachmentObj + raise Exception("no attachment found") - def retrace(self): - retrace = self.api_map[self.api] + 'retrace' - args = [_get_build_path(retrace)] - args += [self.trace_file] - p = popen(args, stdout=subprocess.PIPE) + def getState(self, callNo): + try: + state = self.stateCache[callNo] + except KeyError: + pass + else: + return state + + p = self._retrace(['-D', str(callNo)]) + state = json.load(p.stdout, strict=False) p.wait() if p.returncode != 0: - self.fail('`%s` returned code %i' % (retrace, p.returncode)) + fail('retrace returned code %i' % (p.returncode)) - def run(self): - self.standalone() - self.trace() - self.dump() - self.retrace() + self.adjustSrcState(state) - self.pass_() - return + self.stateCache[callNo] = state - ref_prefix = os.path.abspath(os.path.join(self.results, self.name + '.ref.')) - src_prefix = os.path.join(self.results, self.name + '.src.') - diff_prefix = os.path.join(self.results, self.name + '.diff.') + return state + def adjustSrcState(self, state): + # Do some adjustments on the obtained state to eliminate failures from + # bugs/issues outside of apitrace - if not os.path.isfile(trace): - sys.stdout.write('SKIP (no trace)\n') + try: + parameters = state['parameters'] + except KeyError: return - retrace = self.api_map[self.api] + 'retrace' - args = [_get_build_path(retrace)] - if swapbuffers: - args += ['-db'] - frames = swapbuffers - else: - args += ['-sb'] - frames = flushes - args += ['-s', src_prefix] - args += [trace] - p = popen(args, stdout=subprocess.PIPE) - image_re = re.compile(r'^Wrote (.*\.png)$') - images = [] - for line in p.stdout: - line = line.rstrip() - mo = image_re.match(line) - if mo: - image = mo.group(1) - if image.startswith(src_prefix): - image = image[len(src_prefix):] - images.append(image) - p.wait() - if p.returncode != 0: - sys.stdout.write('FAIL (glretrace)\n') + # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1 + self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1) + + # On Gallium + if 'Gallium' in parameters['GL_RENDERER'].split(): + # Gallium drivers have wrong defaults for draw/read buffer state + self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') + self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT') + self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') + + def adjustRefState(self, state): + # Do some adjustments on reference state to eliminate failures from + # bugs/issues outside of apitrace + + try: + parameters = state['parameters'] + except KeyError: return - for image in images: - ref_image = ref_prefix + image - src_image = src_prefix + image - diff_image = diff_prefix + image - - if not os.path.isfile(ref_image): - continue - assert os.path.isfile(src_image) - - comparer = Comparer(ref_image, src_image) - match = comparer.ae() - sys.stdout.write('%s: %s bits\n' % (image, comparer.precision())) - if not match: - comparer.write_diff(diff_image) - #report.add_snapshot(ref_image, src_image, diff_image) - sys.stdout.write('FAIL (snapshot)\n') - return - - def fail(self, reason=None): - self._exit('FAIL', 1, reason) - - def skip(self, reason=None): - self._exit('SKIP', 0, reason) - - def pass_(self, reason=None): - self._exit('PASS', 0, reason) - - def _exit(self, status, code, reason=None): - if reason is None: - reason = '' + if platform.system() == 'Darwin' or True: + # Mac OS X drivers fail on GL_COLOR_SUM + # XXX: investigate this + self.removeState(parameters, 'GL_COLOR_SUM') + + def replaceState(self, obj, key, srcValue, dstValue): + try: + value = obj[key] + except KeyError: + pass + else: + if value == srcValue: + obj[key] = dstValue + + def removeState(self, obj, key): + try: + del obj[key] + except KeyError: + pass + + def _retrace(self, args = None, stdout=subprocess.PIPE): + retrace = self.api_map[self.api] + 'retrace' + cmd = [_get_build_program(retrace)] + if self.doubleBuffer: + cmd += ['-db'] else: - reason = ' (%s)' % reason - sys.stdout.write('%s%s\n' % (status, reason)) - sys.exit(code) + cmd += ['-sb'] + if args: + cmd += args + cmd += [self.trace_file] + return popen(cmd, stdout=stdout) + + def run(self): + self.runApp() + self.traceApp() + self.checkTrace() + self.retrace() + pass_() def main(): @@ -278,6 +482,11 @@ def main(): optparser = optparse.OptionParser( usage='\n\t%prog [options] -- [TRACE|PROGRAM] ...', version='%%prog') + optparser.add_option( + '-v', '--verbose', + action="store_true", + dest="verbose", default=False, + help="verbose output") optparser.add_option( '-a', '--api', metavar='API', type='string', dest='api', default='gl', @@ -306,7 +515,10 @@ def main(): if not os.path.exists(options.results): os.makedirs(options.results) + sys.path.insert(0, _get_source_path('scripts')) + test = TestCase() + test.verbose = options.verbose if args[0].endswith('.trace'): test.trace_file = args[0]