X-Git-Url: https://git.cworth.org/git?p=apitrace-tests;a=blobdiff_plain;f=driver.py;h=1d005b6e09df9429fb3deb99d044758476ec7a2c;hp=7799b6587693f7d1db058c83a1fcaf5bd684e286;hb=276ee5387182ca865c00500013877b6cc2d5c77e;hpb=5cde484f5bded377ca859164aa6919b0cbe3e7d8 diff --git a/driver.py b/driver.py index 7799b65..1d005b6 100755 --- a/driver.py +++ b/driver.py @@ -34,56 +34,250 @@ import re import subprocess import sys import time +import json +import base64 + +from PIL import Image + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + + +def _exit(status, code, reason=None): + if reason is None: + reason = '' + else: + reason = ' (%s)' % reason + sys.stdout.write('%s%s\n' % (status, reason)) + sys.exit(code) + +def fail(reason=None): + _exit('FAIL', 1, reason) + +def skip(reason=None): + _exit('SKIP', 0, reason) + +def pass_(reason=None): + _exit('PASS', 0, reason) def popen(command, *args, **kwargs): if kwargs.get('cwd', None) is not None: sys.stdout.write('cd %s && ' % kwargs['cwd']) - if 'env' in kwargs: - for name, value in kwargs['env'].iteritems(): + + try: + env = kwargs.pop('env') + except KeyError: + env = None + else: + names = env.keys() + names.sort() + for name in names: + value = env[name] if value != os.environ.get(name, None): sys.stdout.write('%s=%s ' % (name, value)) + env[name] = str(value) + sys.stdout.write(' '.join(command) + '\n') sys.stdout.flush() - return subprocess.Popen(command, *args, **kwargs) + return subprocess.Popen(command, *args, env=env, **kwargs) + + +def which(executable): + dirs = os.environ['PATH'].split(os.path.pathsep) + for dir in dirs: + path = os.path.join(dir, executable) + if os.path.exists(path): + return path + return None + + +def _get_bin_path(): + if os.path.exists(options.apitrace): + apitrace_abspath = os.path.abspath(options.apitrace) + else: + apitrace_abspath = which(options.apitrace) + if apitrace_abspath is None: + sys.stderr.write('error: could not determine the absolute path of\n' % options.apitrace) + sys.exit(1) + return os.path.dirname(apitrace_abspath) -def _get_build_path(path): - if options.build is not None: - path = os.path.abspath(os.path.join(options.build, path)) + +def _get_build_program(program): + bin_path = _get_bin_path() + if platform.system() == 'Windows': + program += '.exe' + path = os.path.join(bin_path, program) if not os.path.exists(path): sys.stderr.write('error: %s does not exist\n' % path) sys.exit(1) return path +def _get_scripts_path(): + bin_path = _get_bin_path() + + try_paths = [ + 'scripts', + '../lib/scripts', + '../lib/apitrace/scripts', + ] + + for try_path in try_paths: + path = os.path.join(bin_path, try_path) + if os.path.exists(path): + return os.path.abspath(path) + + sys.stderr.write('error: could not find scripts directory\n') + sys.exit(1) + + +class TraceChecker: + + def __init__(self, srcStream, refFileName, verbose=False): + self.srcStream = srcStream + self.refFileName = refFileName + if refFileName: + self.refStream = open(refFileName, 'rt') + else: + self.refStream = None + self.verbose = verbose + self.doubleBuffer = False + self.callNo = 0 + self.refLine = '' + self.images = [] + self.states = [] + + call_re = re.compile(r'^([0-9]+) (\w+)\(') + + def check(self): + + swapbuffers = 0 + flushes = 0 + + srcLines = [] + self.consumeRefLine() + for line in self.srcStream: + line = line.rstrip() + if self.verbose: + sys.stdout.write(line + '\n') + mo = self.call_re.match(line) + if mo: + self.callNo = int(mo.group(1)) + function_name = mo.group(2) + if function_name.find('SwapBuffers') != -1 or \ + line.find('kCGLPFADoubleBuffer') != -1: + swapbuffers += 1 + if function_name in ('glFlush', 'glFinish'): + flushes += 1 + srcLine = line[mo.start(2):] + else: + srcLine = line + if self.refLine: + if srcLine == self.refLine: + self.consumeRefLine() + srcLines = [] + else: + srcLines.append(srcLine) + + if self.refLine: + if srcLines: + fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0])) + else: + fail('missing call %s' % self.refLine) + + if swapbuffers: + self.doubleBuffer = True + else: + self.doubleBuffer = False + + def consumeRefLine(self): + if not self.refStream: + self.refLine = '' + return + + while True: + line = self.refStream.readline() + if not line: + break + line = line.rstrip() + if line.startswith('#'): + self.handlePragma(line) + else: + break + self.refLine = line + + def handlePragma(self, line): + pragma, rest = line.split(None, 1) + if pragma == '#image': + imageFileName = self.getAbsPath(rest) + self.images.append((self.callNo, imageFileName)) + elif pragma == '#state': + stateFileName = self.getAbsPath(rest) + self.states.append((self.callNo, stateFileName)) + else: + assert False + + def getAbsPath(self, path): + '''Get the absolute from a path relative to the reference filename''' + return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path)) + + class TestCase: - max_frames = None + cmd = None + cwd = None + api = 'gl' + max_frames = None trace_file = None - def __init__(self, name, args, cwd=None, build=None, results = '.'): - self.name = name - self.args = args - self.cwd = cwd - self.build = build - self.results = results + ref_dump = None + + doubleBuffer = True - if not os.path.exists(results): - os.makedirs(results) + verbose = False - expected_dump = None + threshold_precision = 12.0 - def standalone(self): - p = popen(self.args, cwd=self.cwd) + def __init__(self): + self.stateCache = {} + + def runApp(self): + '''Run the application standalone, skipping this test if it fails by + some reason.''' + + if not self.cmd: + return + + p = popen(self.cmd, cwd=self.cwd) p.wait() if p.returncode: - self.skip('application returned code %i' % p.returncode) + skip('application returned code %i' % p.returncode) + + api_map = { + 'gl': 'gl', + 'egl_gl': 'egl', + 'egl_gles1': 'egl', + 'egl_gles2': 'egl', + } + + def traceApp(self): + if not self.cmd: + return - def trace(self): if self.trace_file is None: - self.trace_file = os.path.abspath(os.path.join(self.results, self.name + '.trace')) + if self.ref_dump is not None: + name = self.ref_dump + else: + name = self.cmd[0] + name, ext = os.path.splitext(os.path.basename(name)) + while ext: + name, ext = os.path.splitext(os.path.basename(name)) + self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace')) if os.path.exists(self.trace_file): os.remove(self.trace_file) else: @@ -91,155 +285,242 @@ class TestCase: if not os.path.exists(trace_dir): os.makedirs(trace_dir) + cmd = self.cmd env = os.environ.copy() - system = platform.system() - if system == 'Windows': - # TODO - self.skip('tracing not supported on Windows') - wrapper = _get_build_path('wrappers/opengl32.dll') - elif system == 'Darwin': - wrapper = _get_build_path('wrappers/OpenGL') - env['DYLD_LIBRARY_PATH'] = os.path.dirname(wrapper) - else: - wrapper = _get_build_path('glxtrace.so') - env['LD_PRELOAD'] = wrapper - - env['TRACE_FILE'] = self.trace_file + cmd = [ + options.apitrace, 'trace', + '--api', self.api_map[self.api], + '--output', self.trace_file, + '--' + ] + cmd if self.max_frames is not None: env['TRACE_FRAMES'] = str(self.max_frames) - p = popen(self.args, env=env, cwd=self.cwd) + p = popen(cmd, env=env, cwd=self.cwd) p.wait() if not os.path.exists(self.trace_file): - self.fail('no trace file generated\n') + fail('no trace file generated\n') - call_re = re.compile(r'^([0-9]+) (\w+)\(') + def checkTrace(self): + cmd = [options.apitrace, 'dump', '--color=never', self.trace_file] + p = popen(cmd, stdout=subprocess.PIPE) - def dump(self): + checker = TraceChecker(p.stdout, self.ref_dump, self.verbose) + checker.check() + p.wait() + if p.returncode != 0: + fail('`apitrace dump` returned code %i' % p.returncode) + + self.doubleBuffer = checker.doubleBuffer + + for callNo, refImageFileName in checker.images: + self.checkImage(callNo, refImageFileName) + for callNo, refStateFileName in checker.states: + self.checkState(callNo, refStateFileName) + + def checkImage(self, callNo, refImageFileName): + srcImage = self.getImage(callNo) + refImage = Image.open(refImageFileName) + + from snapdiff import Comparer + comparer = Comparer(refImage, srcImage) + precision = comparer.precision(filter=True) + sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName)) + if precision < self.threshold_precision: + prefix = '%s.%u' % (self.getNamePrefix(), callNo) + srcImageFileName = prefix + '.src.png' + srcImage.save(srcImageFileName) + diffImageFileName = prefix + '.diff.png' + comparer.write_diff(diffImageFileName) + fail('snapshot from call %u does not match %s' % (callNo, refImageFileName)) + + def checkState(self, callNo, refStateFileName): + srcState = self.getState(callNo) + refState = self.getRefState(refStateFileName) + + from jsondiff import Comparer, Differ + comparer = Comparer(ignore_added = True) + match = comparer.visit(refState, srcState) + if not match: + prefix = '%s.%u' % (self.getNamePrefix(), callNo) + srcStateFileName = prefix + '.src.json' + diffStateFileName = prefix + '.diff.json' + self.saveState(srcState, srcStateFileName) + #diffStateFile = open(diffStateFileName, 'wt') + diffStateFile = sys.stdout + differ = Differ(diffStateFile, ignore_added = True) + differ.visit(refState, srcState) + fail('state from call %u does not match %s' % (callNo, refStateFileName)) + + def getRefState(self, refStateFileName): + stream = open(refStateFileName, 'rt') + from jsondiff import load + state = load(stream) + self.adjustRefState(state) + return state + + def getNamePrefix(self): + name = os.path.basename(self.ref_dump) + try: + index = name.index('.') + except ValueError: + pass + else: + name = name[:index] + return name - cmd = [_get_build_path('tracedump'), '--color=never', self.trace_file] - p = popen(cmd, stdout=subprocess.PIPE) + def saveState(self, state, filename): + s = json.dumps(state, sort_keys=True, indent=2) + open(filename, 'wt').write(s) - swapbuffers = 0 - flushes = 0 + def retrace(self): + p = self._retrace() + p.wait() + if p.returncode != 0: + fail('retrace failed with code %i' % (p.returncode)) - ref_line = '' - if self.ref_dump is not None: - ref = open(self.ref_dump, 'rt') - ref_line = ref.readline().rstrip() - for line in p.stdout: - line = line.rstrip() - mo = self.call_re.match(line) - assert mo - if mo: - call_no = int(mo.group(1)) - function_name = mo.group(2) - if function_name == 'glXSwapBuffers': - swapbuffers += 1 - if function_name in ('glFlush', 'glFinish'): - flushes += 1 - src_line = line[mo.start(2):] - sys.stdout.write(src_line + '\n') - if ref_line: - if src_line == ref_line: - ref_line = ref.readline().rstrip() + def getImage(self, callNo): + state = self.getState(callNo) + if self.doubleBuffer: + attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0'] + else: + attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0'] + imageObj = self.getFramebufferAttachment(state, attachments) + data = imageObj['__data__'] + stream = StringIO(base64.b64decode(data)) + im = Image.open(stream) + im.save('test.png') + return im + + def getFramebufferAttachment(self, state, attachments): + framebufferObj = state['framebuffer'] + for attachment in attachments: + try: + attachmentObj = framebufferObj[attachment] + except KeyError: + pass + else: + return attachmentObj + raise Exception("no attachment found") + + def getState(self, callNo): + try: + state = self.stateCache[callNo] + except KeyError: + pass + else: + return state + + p = self._retrace(['-D', str(callNo)]) + state = json.load(p.stdout, strict=False) p.wait() if p.returncode != 0: - self.fail('tracedump returned code %i' % p.returncode) - if ref_line: - self.fail('missing call %' % ref_line) + fail('retrace returned code %i' % (p.returncode)) - def run(self): - self.standalone() - self.trace() - self.dump() + self.adjustSrcState(state) - self.pass_() - return + self.stateCache[callNo] = state - ref_prefix = os.path.abspath(os.path.join(self.results, self.name + '.ref.')) - src_prefix = os.path.join(self.results, self.name + '.src.') - diff_prefix = os.path.join(self.results, self.name + '.diff.') + return state + def adjustSrcState(self, state): + # Do some adjustments on the obtained state to eliminate failures from + # bugs/issues outside of apitrace - if not os.path.isfile(trace): - sys.stdout.write('SKIP (no trace)\n') + try: + parameters = state['parameters'] + except KeyError: return - args = [_get_build_path('glretrace')] - if swapbuffers: - args += ['-db'] - frames = swapbuffers - else: - args += ['-sb'] - frames = flushes - args += ['-s', src_prefix] - args += [trace] - p = popen(args, stdout=subprocess.PIPE) - image_re = re.compile(r'^Wrote (.*\.png)$') - images = [] - for line in p.stdout: - line = line.rstrip() - mo = image_re.match(line) - if mo: - image = mo.group(1) - if image.startswith(src_prefix): - image = image[len(src_prefix):] - images.append(image) - p.wait() - if p.returncode != 0: - sys.stdout.write('FAIL (glretrace)\n') + + # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1 + self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1) + + # On Gallium + if 'Gallium' in parameters['GL_RENDERER'].split(): + # Gallium drivers have wrong defaults for draw/read buffer state + self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK') + self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') + self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT') + self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT') + + def adjustRefState(self, state): + # Do some adjustments on reference state to eliminate failures from + # bugs/issues outside of apitrace + + try: + parameters = state['parameters'] + except KeyError: return - for image in images: - ref_image = ref_prefix + image - src_image = src_prefix + image - diff_image = diff_prefix + image - - if not os.path.isfile(ref_image): - continue - assert os.path.isfile(src_image) - - comparer = Comparer(ref_image, src_image) - match = comparer.ae() - sys.stdout.write('%s: %s bits\n' % (image, comparer.precision())) - if not match: - comparer.write_diff(diff_image) - #report.add_snapshot(ref_image, src_image, diff_image) - sys.stdout.write('FAIL (snapshot)\n') - return - - def fail(self, reason=None): - self._exit('FAIL', 1, reason) - - def skip(self, reason=None): - self._exit('SKIP', 0, reason) - - def pass_(self, reason=None): - self._exit('PASS', 0, reason) - - def _exit(self, status, code, reason=None): - if reason is None: - reason = '' + if platform.system() == 'Darwin': + # Mac OS X drivers fail on GL_COLOR_SUM + # XXX: investigate this + self.removeState(parameters, 'GL_COLOR_SUM') + + def replaceState(self, obj, key, srcValue, dstValue): + try: + value = obj[key] + except KeyError: + pass + else: + if value == srcValue: + obj[key] = dstValue + + def removeState(self, obj, key): + try: + del obj[key] + except KeyError: + pass + + def _retrace(self, args = None, stdout=subprocess.PIPE): + retrace = self.api_map[self.api] + 'retrace' + cmd = [_get_build_program(retrace)] + if self.doubleBuffer: + cmd += ['-db'] else: - reason = ' (%s)' % reason - sys.stdout.write('%s%s\n' % (status, reason)) - sys.exit(code) + cmd += ['-sb'] + if args: + cmd += args + cmd += [self.trace_file] + return popen(cmd, stdout=stdout) + + def run(self): + self.runApp() + self.traceApp() + self.checkTrace() + self.retrace() + pass_() def main(): global options + default_apitrace = 'apitrace' + if platform.system() == 'Windows': + default_apitrace += '.exe' + # Parse command line options optparser = optparse.OptionParser( - usage='\n\t%prog [options] -- program [args] ...', + usage='\n\t%prog [options] -- [TRACE|PROGRAM] ...', version='%%prog') optparser.add_option( - '-B', '--build', metavar='PATH', - type='string', dest='build', default='..', - help='path to apitrace build') + '-v', '--verbose', + action="store_true", + dest="verbose", default=False, + help="verbose output") + optparser.add_option( + '-a', '--api', metavar='API', + type='string', dest='api', default='gl', + help='api to trace') + optparser.add_option( + '--apitrace', metavar='PROGRAM', + type='string', dest='apitrace', default=default_apitrace, + help='path to apitrace executable') optparser.add_option( '-C', '--directory', metavar='PATH', type='string', dest='cwd', default=None, @@ -255,16 +536,26 @@ def main(): (options, args) = optparser.parse_args(sys.argv[1:]) if not args: - optparser.error('program must be specified') - - test = TestCase( - name = os.path.basename(args[0]), - args = args, - cwd = options.cwd, - build = options.build, - results = options.results, - ) + optparser.error('an argument must be specified') + + if not os.path.exists(options.results): + os.makedirs(options.results) + + print _get_scripts_path() + + sys.path.insert(0, _get_scripts_path()) + + test = TestCase() + test.verbose = options.verbose + + if args[0].endswith('.trace'): + test.trace_file = args[0] + else: + test.cmd = args + test.cwd = options.cwd + test.api = options.api test.ref_dump = options.ref_dump + test.results = options.results test.run()