'''Common test suite code.'''
-import os.path
+import math
import optparse
-import sys
-import subprocess
-import time
+import os.path
import re
import signal
+import subprocess
+import sys
+import time
+
+import Image
+import ImageChops
+import ImageEnhance
+
+
+class Comparer:
+ '''Image comparer.'''
+
+ def __init__(self, ref_image, src_image, alpha = False):
+ self.ref_im = Image.open(ref_image)
+ self.src_im = Image.open(src_image)
+
+ # Crop to the minimum size
+ ref_w, ref_h = self.ref_im.size
+ src_w, src_h = self.src_im.size
+ w = min(ref_w, src_w)
+ h = min(ref_h, src_h)
+ self.ref_im = self.ref_im.crop((0, ref_h - h, w, ref_h))
+ self.src_im = self.src_im.crop((0, src_h - h, w, src_h))
+
+ # Ignore alpha
+ if not alpha:
+ self.ref_im = self.ref_im.convert('RGB')
+ self.src_im = self.src_im.convert('RGB')
+
+ self.diff = ImageChops.difference(self.src_im, self.ref_im)
+
+ def write_diff(self, diff_image, fuzz = 0.05):
+ # make a difference image similar to ImageMagick's compare utility
+ mask = ImageEnhance.Brightness(self.diff).enhance(1.0/fuzz)
+ mask = mask.convert('L')
+
+ lowlight = Image.new('RGB', self.src_im.size, (0xff, 0xff, 0xff))
+ highlight = Image.new('RGB', self.src_im.size, (0xf1, 0x00, 0x1e))
+ diff_im = Image.composite(highlight, lowlight, mask)
+
+ diff_im = Image.blend(self.src_im, diff_im, 0xcc/255.0)
+ diff_im.save(diff_image)
+
+ def precision(self):
+ # See also http://effbot.org/zone/pil-comparing-images.htm
+ h = self.diff.histogram()
+ square_error = 0
+ for i in range(1, 256):
+ square_error += sum(h[i : 3*256: 256])*i*i
+ rel_error = float(square_error*2 + 1) / float(self.diff.size[0]*self.diff.size[1]*3*255*255*2)
+ bits = -math.log(rel_error)/math.log(2.0)
+ return bits
+
+ def ae(self, chantol = 4, pixeltol = 0.03):
+ # Compute absolute error
+ # chantol = color channel tolerance
+ # pixeltol = ratio of pixels we allow to go completely off
+
+ # TODO: this is approximate due to the grayscale conversion
+ h = self.diff.convert('L').histogram()
+
+ ae = sum(h[int(chantol) + 1 : 256])
+
+ return ae <= pixeltol*self.diff.size[0]*self.diff.size[1]
ansi_re = re.compile('\x1b\[[0-9]{1,2}(;[0-9]{1,2}){0,2}m')
class TestCase:
- def __init__(self, name, args, cwd=None, build = '.', results = '.'):
+ standalone = False
+
+ max_frames = 1
+
+ def __init__(self, name, args, cwd=None, build=None, results = '.'):
self.name = name
self.args = args
self.cwd = cwd
if not os.path.exists(results):
os.makedirs(results)
+
+ def _get_build_path(self, path):
+ if self.build is not None:
+ path = os.path.abspath(os.path.join(self.build, path))
+ if not os.path.exists(path):
+ sys.stderr.write('error: %s does not exist\n' % path)
+ sys.exit(1)
+ return path
+
def run(self, report):
trace = os.path.abspath(os.path.join(self.results, self.name + '.trace'))
+ ref_prefix = os.path.abspath(os.path.join(self.results, self.name + '.ref.'))
+ src_prefix = os.path.join(self.results, self.name + '.src.')
+ diff_prefix = os.path.join(self.results, self.name + '.diff.')
- ld_preload = os.path.abspath(os.path.join(self.build, 'glxtrace.so'))
- if not os.path.exists(ld_preload):
- sys.stderr.write('error: could not find %s\n' % ld_preload)
- sys.exit(1)
+ ld_preload = self._get_build_path('glxtrace.so')
env = os.environ.copy()
env['LD_PRELOAD'] = ld_preload
env['TRACE_FILE'] = trace
+ env['TRACE_SNAPSHOT'] = ref_prefix
+ env['TRACE_FRAMES'] = str(self.max_frames)
- window_name = self.args[0]
-
- p = popen(self.args, cwd=self.cwd)
- for i in range(3):
- time.sleep(1)
- if p.poll() is not None:
- break
- if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
- break
- if p.returncode is None:
- p.terminate()
- elif p.returncode:
- sys.stdout.write('SKIP (app)\n')
- return
+ if self.standalone:
+ p = popen(self.args, cwd=self.cwd)
+ for i in range(3):
+ time.sleep(1)
+ if p.poll() is not None:
+ break
+ if p.returncode is None:
+ p.terminate()
+ elif p.returncode:
+ sys.stdout.write('SKIP (app)\n')
+ return
- ref_image = os.path.join(self.results, self.name + '.ref.png')
p = popen(self.args, env=env, cwd=self.cwd)
try:
for i in range(5):
time.sleep(1)
if p.poll() is not None:
break
- if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
- break
-
- if p.returncode is None:
- subprocess.call("xwd -name '%s' | xwdtopnm | pnmtopng > '%s'" % (window_name, ref_image), shell=True)
finally:
if p.returncode is None:
p.terminate()
p.wait()
elif p.returncode:
- print p.returncode
- sys.stdout.write('FAIL (trace)\n')
+ if self.standalone:
+ sys.stdout.write('FAIL (trace)\n')
+ else:
+ sys.stdout.write('SKIP (app)\n')
return
- p = popen([os.path.join(self.build, 'tracedump'), trace], stdout=subprocess.PIPE)
+ if not os.path.isfile(trace):
+ sys.stdout.write('SKIP (no trace)\n')
+ return
+
+ p = popen([self._get_build_path('tracedump'), trace], stdout=subprocess.PIPE)
call_re = re.compile('^([0-9]+) (\w+)\(')
swapbuffers = 0
flushes = 0
swapbuffers += 1
if function_name in ('glFlush', 'glFinish'):
flushes += 1
- #print orig_line
p.wait()
if p.returncode != 0:
sys.stdout.write('FAIL (tracedump)\n')
return
- args = [os.path.join(self.build, 'glretrace')]
+ args = [self._get_build_path('glretrace')]
if swapbuffers:
args += ['-db']
frames = swapbuffers
else:
args += ['-sb']
frames = flushes
- if os.path.exists(ref_image) and frames < 10:
- snapshot_prefix = os.path.join(self.results, self.name + '.')
- args += ['-s', snapshot_prefix]
- else:
- snapshot_prefix = None
+ args += ['-s', src_prefix]
args += [trace]
p = popen(args, stdout=subprocess.PIPE)
image_re = re.compile('^Wrote (.*\.png)$')
- image = None
+ images = []
for line in p.stdout:
line = line.rstrip()
mo = image_re.match(line)
if mo:
image = mo.group(1)
+ if image.startswith(src_prefix):
+ image = image[len(src_prefix):]
+ images.append(image)
p.wait()
if p.returncode != 0:
sys.stdout.write('FAIL (glretrace)\n')
return
- if image:
- delta_image = os.path.join(self.results, self.name + '.diff.png')
- p = popen([
- 'compare',
- '-alpha', 'opaque',
- '-metric', 'AE',
- '-fuzz', '5%',
- '-dissimilarity-threshold', '1',
- ref_image, image, delta_image
- ], stderr = subprocess.PIPE)
- _, stderr = p.communicate()
-
- try:
- ae = int(stderr)
- except ValueError:
- ae = 9999
-
- if ae:
- report.add_snapshot(ref_image, image, delta_image)
+ for image in images:
+ ref_image = ref_prefix + image
+ src_image = src_prefix + image
+ diff_image = diff_prefix + image
+
+ if not os.path.isfile(ref_image):
+ continue
+ assert os.path.isfile(src_image)
+
+ comparer = Comparer(ref_image, src_image)
+ match = comparer.ae()
+ sys.stdout.write('%s: %s bits\n' % (image, comparer.precision()))
+ if not match:
+ comparer.write_diff(diff_image)
+ report.add_snapshot(ref_image, src_image, diff_image)
sys.stdout.write('FAIL (snapshot)\n')
return
usage='\n\t%prog [options] -- program [args] ...',
version='%%prog')
optparser.add_option(
- '--build', metavar='PATH',
- type='string', dest='build', default='.',
- help='path to apitrace build [default=%default]')
+ '-B', '--build', metavar='PATH',
+ type='string', dest='build', default=None,
+ help='path to apitrace build')
optparser.add_option(
- '--cwd', metavar='PATH',
- type='string', dest='cwd', default='.',
- help='change directory [default=%default]')
+ '-C', '--directory', metavar='PATH',
+ type='string', dest='cwd', default=None,
+ help='change to directory')
optparser.add_option(
- '--results', metavar='PATH',
+ '-R', '--results', metavar='PATH',
type='string', dest='results', default='results',
help='results directory [default=%default]')
projects / apitrace-tests / blobdiff