X-Git-Url: https://git.cworth.org/git?a=blobdiff_plain;f=test.py;h=b3d2ed9722a51a886dc6e4807e752e49acc77a26;hb=8a63a6b0f98da88770d3479a1166ec92e8c72e7f;hp=3c96c12241dc12496b23fd5a826215db114fae76;hpb=cde1bf34498495236aee6e8541d39cac3aa54056;p=apitrace-tests diff --git a/test.py b/test.py index 3c96c12..b3d2ed9 100755 --- a/test.py +++ b/test.py @@ -27,13 +27,75 @@ '''Common test suite code.''' -import os.path +import math import optparse -import sys -import subprocess -import time +import os.path import re import signal +import subprocess +import sys +import time + +import Image +import ImageChops +import ImageEnhance + + +class Comparer: + '''Image comparer.''' + + def __init__(self, ref_image, src_image, alpha = False): + self.ref_im = Image.open(ref_image) + self.src_im = Image.open(src_image) + + # Crop to the minimum size + ref_w, ref_h = self.ref_im.size + src_w, src_h = self.src_im.size + w = min(ref_w, src_w) + h = min(ref_h, src_h) + self.ref_im = self.ref_im.crop((0, ref_h - h, w, ref_h)) + self.src_im = self.src_im.crop((0, src_h - h, w, src_h)) + + # Ignore alpha + if not alpha: + self.ref_im = self.ref_im.convert('RGB') + self.src_im = self.src_im.convert('RGB') + + self.diff = ImageChops.difference(self.src_im, self.ref_im) + + def write_diff(self, diff_image, fuzz = 0.05): + # make a difference image similar to ImageMagick's compare utility + mask = ImageEnhance.Brightness(self.diff).enhance(1.0/fuzz) + mask = mask.convert('L') + + lowlight = Image.new('RGB', self.src_im.size, (0xff, 0xff, 0xff)) + highlight = Image.new('RGB', self.src_im.size, (0xf1, 0x00, 0x1e)) + diff_im = Image.composite(highlight, lowlight, mask) + + diff_im = Image.blend(self.src_im, diff_im, 0xcc/255.0) + diff_im.save(diff_image) + + def precision(self): + # See also http://effbot.org/zone/pil-comparing-images.htm + h = self.diff.histogram() + square_error = 0 + for i in range(1, 256): + square_error += sum(h[i : 3*256: 256])*i*i + rel_error = float(square_error*2 + 1) / float(self.diff.size[0]*self.diff.size[1]*3*255*255*2) + bits = -math.log(rel_error)/math.log(2.0) + return bits + + def ae(self, chantol = 4, pixeltol = 0.03): + # Compute absolute error + # chantol = color channel tolerance + # pixeltol = ratio of pixels we allow to go completely off + + # TODO: this is approximate due to the grayscale conversion + h = self.diff.convert('L').histogram() + + ae = sum(h[int(chantol) + 1 : 256]) + + return ae <= pixeltol*self.diff.size[0]*self.diff.size[1] ansi_re = re.compile('\x1b\[[0-9]{1,2}(;[0-9]{1,2}){0,2}m') @@ -111,7 +173,11 @@ class Report: class TestCase: - def __init__(self, name, args, cwd=None, build = '.', results = '.'): + standalone = False + + max_frames = 1 + + def __init__(self, name, args, cwd=None, build=None, results = '.'): self.name = name self.args = args self.cwd = cwd @@ -121,56 +187,64 @@ class TestCase: if not os.path.exists(results): os.makedirs(results) + + def _get_build_path(self, path): + if self.build is not None: + path = os.path.abspath(os.path.join(self.build, path)) + if not os.path.exists(path): + sys.stderr.write('error: %s does not exist\n' % path) + sys.exit(1) + return path + def run(self, report): trace = os.path.abspath(os.path.join(self.results, self.name + '.trace')) + ref_prefix = os.path.abspath(os.path.join(self.results, self.name + '.ref.')) + src_prefix = os.path.join(self.results, self.name + '.src.') + diff_prefix = os.path.join(self.results, self.name + '.diff.') - ld_preload = os.path.abspath(os.path.join(self.build, 'glxtrace.so')) - if not os.path.exists(ld_preload): - sys.stderr.write('error: could not find %s\n' % ld_preload) - sys.exit(1) + ld_preload = self._get_build_path('glxtrace.so') env = os.environ.copy() env['LD_PRELOAD'] = ld_preload env['TRACE_FILE'] = trace + env['TRACE_SNAPSHOT'] = ref_prefix + env['TRACE_FRAMES'] = str(self.max_frames) - window_name = self.args[0] - - p = popen(self.args, cwd=self.cwd) - for i in range(3): - time.sleep(1) - if p.poll() is not None: - break - if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0: - break - if p.returncode is None: - p.terminate() - elif p.returncode: - sys.stdout.write('SKIP (app)\n') - return + if self.standalone: + p = popen(self.args, cwd=self.cwd) + for i in range(3): + time.sleep(1) + if p.poll() is not None: + break + if p.returncode is None: + p.terminate() + elif p.returncode: + sys.stdout.write('SKIP (app)\n') + return - ref_image = os.path.join(self.results, self.name + '.ref.png') p = popen(self.args, env=env, cwd=self.cwd) try: for i in range(5): time.sleep(1) if p.poll() is not None: break - if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0: - break - - if p.returncode is None: - subprocess.call("xwd -name '%s' | xwdtopnm | pnmtopng > '%s'" % (window_name, ref_image), shell=True) finally: if p.returncode is None: p.terminate() p.wait() elif p.returncode: - print p.returncode - sys.stdout.write('FAIL (trace)\n') + if self.standalone: + sys.stdout.write('FAIL (trace)\n') + else: + sys.stdout.write('SKIP (app)\n') return - p = popen([os.path.join(self.build, 'tracedump'), trace], stdout=subprocess.PIPE) + if not os.path.isfile(trace): + sys.stdout.write('SKIP (no trace)\n') + return + + p = popen([self._get_build_path('tracedump'), trace], stdout=subprocess.PIPE) call_re = re.compile('^([0-9]+) (\w+)\(') swapbuffers = 0 flushes = 0 @@ -187,57 +261,51 @@ class TestCase: swapbuffers += 1 if function_name in ('glFlush', 'glFinish'): flushes += 1 - #print orig_line p.wait() if p.returncode != 0: sys.stdout.write('FAIL (tracedump)\n') return - args = [os.path.join(self.build, 'glretrace')] + args = [self._get_build_path('glretrace')] if swapbuffers: args += ['-db'] frames = swapbuffers else: args += ['-sb'] frames = flushes - if os.path.exists(ref_image) and frames < 10: - snapshot_prefix = os.path.join(self.results, self.name + '.') - args += ['-s', snapshot_prefix] - else: - snapshot_prefix = None + args += ['-s', src_prefix] args += [trace] p = popen(args, stdout=subprocess.PIPE) image_re = re.compile('^Wrote (.*\.png)$') - image = None + images = [] for line in p.stdout: line = line.rstrip() mo = image_re.match(line) if mo: image = mo.group(1) + if image.startswith(src_prefix): + image = image[len(src_prefix):] + images.append(image) p.wait() if p.returncode != 0: sys.stdout.write('FAIL (glretrace)\n') return - if image: - delta_image = os.path.join(self.results, self.name + '.diff.png') - p = popen([ - 'compare', - '-alpha', 'opaque', - '-metric', 'AE', - '-fuzz', '5%', - '-dissimilarity-threshold', '1', - ref_image, image, delta_image - ], stderr = subprocess.PIPE) - _, stderr = p.communicate() - - try: - ae = int(stderr) - except ValueError: - ae = 9999 - - if ae: - report.add_snapshot(ref_image, image, delta_image) + for image in images: + ref_image = ref_prefix + image + src_image = src_prefix + image + diff_image = diff_prefix + image + + if not os.path.isfile(ref_image): + continue + assert os.path.isfile(src_image) + + comparer = Comparer(ref_image, src_image) + match = comparer.ae() + sys.stdout.write('%s: %s bits\n' % (image, comparer.precision())) + if not match: + comparer.write_diff(diff_image) + report.add_snapshot(ref_image, src_image, diff_image) sys.stdout.write('FAIL (snapshot)\n') return @@ -250,15 +318,15 @@ def main(): usage='\n\t%prog [options] -- program [args] ...', version='%%prog') optparser.add_option( - '--build', metavar='PATH', - type='string', dest='build', default='.', - help='path to apitrace build [default=%default]') + '-B', '--build', metavar='PATH', + type='string', dest='build', default=None, + help='path to apitrace build') optparser.add_option( - '--cwd', metavar='PATH', - type='string', dest='cwd', default='.', - help='change directory [default=%default]') + '-C', '--directory', metavar='PATH', + type='string', dest='cwd', default=None, + help='change to directory') optparser.add_option( - '--results', metavar='PATH', + '-R', '--results', metavar='PATH', type='string', dest='results', default='results', help='results directory [default=%default]')