git.cworth.org Git - apitrace-tests/blob - test.py

   1 #!/usr/bin/env python
   2 ##########################################################################
   3 #
   4 # Copyright 2011 Jose Fonseca
   5 # All Rights Reserved.
   6 #
   7 # Permission is hereby granted, free of charge, to any person obtaining a copy
   8 # of this software and associated documentation files (the "Software"), to deal
   9 # in the Software without restriction, including without limitation the rights
  10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11 # copies of the Software, and to permit persons to whom the Software is
  12 # furnished to do so, subject to the following conditions:
  13 #
  14 # The above copyright notice and this permission notice shall be included in
  15 # all copies or substantial portions of the Software.
  16 #
  17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23 # THE SOFTWARE.
  24 #
  25 ##########################################################################/
  26
  27 '''Common test suite code.'''
  28
  29
  30 import math
  31 import optparse
  32 import os.path
  33 import re
  34 import signal
  35 import subprocess
  36 import sys
  37 import time
  38
  39 import Image
  40 import ImageChops
  41 import ImageEnhance
  42
  43
  44 class Comparer:
  45     '''Image comparer.'''
  46
  47     def __init__(self, ref_image, src_image, alpha = False):
  48         self.ref_im = Image.open(ref_image)
  49         self.src_im = Image.open(src_image)
  50
  51         # Crop to the minimum size
  52         ref_w, ref_h = self.ref_im.size
  53         src_w, src_h = self.src_im.size
  54         w = min(ref_w, src_w)
  55         h = min(ref_h, src_h)
  56         self.ref_im = self.ref_im.crop((0, ref_h - h, w, ref_h))
  57         self.src_im = self.src_im.crop((0, src_h - h, w, src_h))
  58
  59         # Ignore alpha
  60         if not alpha:
  61             self.ref_im = self.ref_im.convert('RGB')
  62             self.src_im = self.src_im.convert('RGB')
  63
  64         self.diff = ImageChops.difference(self.src_im, self.ref_im)
  65
  66     def write_diff(self, diff_image, fuzz = 0.05):
  67         # make a difference image similar to ImageMagick's compare utility
  68         mask = ImageEnhance.Brightness(self.diff).enhance(1.0/fuzz)
  69         mask = mask.convert('L')
  70
  71         lowlight = Image.new('RGB', self.src_im.size, (0xff, 0xff, 0xff))
  72         highlight = Image.new('RGB', self.src_im.size, (0xf1, 0x00, 0x1e))
  73         diff_im = Image.composite(highlight, lowlight, mask)
  74
  75         diff_im = Image.blend(self.src_im, diff_im, 0xcc/255.0)
  76         diff_im.save(diff_image)
  77
  78     def precision(self):
  79         # See also http://effbot.org/zone/pil-comparing-images.htm
  80         h = self.diff.histogram()
  81         square_error = 0
  82         for i in range(1, 256):
  83             square_error += sum(h[i : 3*256: 256])*i*i
  84         rel_error = float(square_error*2 + 1) / float(self.diff.size[0]*self.diff.size[1]*3*255*255*2)
  85         bits = -math.log(rel_error)/math.log(2.0)
  86         return bits
  87
  88     def ae(self, chantol = 4, pixeltol = 0.03):
  89         # Compute absolute error
  90         # chantol = color channel tolerance
  91         # pixeltol = ratio of pixels we allow to go completely off
  92
  93         # TODO: this is approximate due to the grayscale conversion
  94         h = self.diff.convert('L').histogram()
  95
  96         ae = sum(h[int(chantol) + 1 : 256])
  97
  98         return ae <= pixeltol*self.diff.size[0]*self.diff.size[1]
  99
 100
 101 ansi_re = re.compile('\x1b\[[0-9]{1,2}(;[0-9]{1,2}){0,2}m')
 102
 103
 104 def ansi_strip(s):
 105     # http://www.theeggeadventure.com/wikimedia/index.php/Linux_Tips#Use_sed_to_remove_ANSI_colors
 106     return ansi_re.sub('', s)
 107
 108
 109 def popen(command, *args, **kwargs):
 110     if 'cwd' in kwargs:
 111         sys.stdout.write('cd %s && ' % kwargs['cwd'])
 112     if 'env' in kwargs:
 113         for name, value in kwargs['env'].iteritems():
 114             if value != os.environ.get(name, None):
 115                 sys.stdout.write('%s=%s ' % (name, value))
 116     sys.stdout.write(' '.join(command) + '\n')
 117     sys.stdout.flush()
 118     return subprocess.Popen(command, *args, **kwargs)
 119
 120
 121 ignored_function_names = set([
 122     'glGetString',
 123     'glXGetCurrentDisplay',
 124     'glXGetClientString',
 125     'glXGetProcAddress',
 126     'glXGetProcAddressARB',
 127     'glXQueryVersion',
 128     'glXGetVisualFromFBConfig',
 129     'glXChooseFBConfig',
 130     'glXCreateNewContext',
 131     'glXMakeContextCurrent',
 132     'glXQueryExtension',
 133     'glXIsDirect',
 134 ])
 135
 136
 137 class Report:
 138
 139     def __init__(self, basedir):
 140         self.basedir = basedir
 141         if not os.path.exists(basedir):
 142             os.makedirs(basedir)
 143         self.html = open(os.path.join(basedir, 'index.html'), 'wt')
 144         self._header()
 145
 146     def _header(self):
 147         self.html.write('<html>\n')
 148         self.html.write('  <body>\n')
 149         self.html.write('    <table border="1">\n')
 150         self.html.write('      <tr><th>Ref</th><th>Src</th><th>&Delta;</th></tr>\n')
 151
 152     def _image_tag(self, image):
 153         url = os.path.relpath(image, self.basedir)
 154         self.html.write('        <td><a href="%s"><img src="%s"/></a></td>\n' % (url, url))
 155
 156     def add_snapshot(self, ref_image, src_image, diff_image):
 157         self.html.write('      <tr>\n')
 158         self._image_tag(ref_image)
 159         self._image_tag(src_image)
 160         self._image_tag(diff_image)
 161         self.html.write('      </tr>\n')
 162         self.html.flush()
 163
 164     def _footer(self):
 165         self.html.write('    </table>\n')
 166         self.html.write('  </body>\n')
 167         self.html.write('</html>\n')
 168
 169     def __del__(self):
 170         self._footer()
 171         self.html.close()
 172
 173
 174 class TestCase:
 175
 176     standalone = False
 177
 178     max_frames = 1
 179
 180     def __init__(self, name, args, cwd=None, build=None, results = '.'):
 181         self.name = name
 182         self.args = args
 183         self.cwd = cwd
 184         self.build = build
 185         self.results = results
 186
 187         if not os.path.exists(results):
 188             os.makedirs(results)
 189
 190
 191     def _get_build_path(self, path):
 192         if self.build is not None:
 193             path = os.path.abspath(os.path.join(self.build, path))
 194             if not os.path.exists(path):
 195                 sys.stderr.write('error: %s does not exist\n' % path)
 196                 sys.exit(1)
 197         return path
 198
 199     def run(self, report):
 200
 201         trace = os.path.abspath(os.path.join(self.results, self.name + '.trace'))
 202         ref_prefix = os.path.abspath(os.path.join(self.results, self.name + '.ref.'))
 203         src_prefix = os.path.join(self.results, self.name + '.src.')
 204         diff_prefix = os.path.join(self.results, self.name + '.diff.')
 205
 206         ld_preload = self._get_build_path('glxtrace.so')
 207
 208         env = os.environ.copy()
 209         env['LD_PRELOAD'] = ld_preload
 210         env['TRACE_FILE'] = trace
 211         env['TRACE_SNAPSHOT'] = ref_prefix
 212         env['TRACE_FRAMES'] = str(self.max_frames)
 213
 214         if self.standalone:
 215             p = popen(self.args, cwd=self.cwd)
 216             for i in range(3):
 217                 time.sleep(1)
 218                 if p.poll() is not None:
 219                     break
 220             if p.returncode is None:
 221                 p.terminate()
 222             elif p.returncode:
 223                 sys.stdout.write('SKIP (app)\n')
 224                 return
 225
 226         p = popen(self.args, env=env, cwd=self.cwd)
 227         try:
 228             for i in range(5):
 229                 time.sleep(1)
 230                 if p.poll() is not None:
 231                     break
 232         finally:
 233             if p.returncode is None:
 234                 p.terminate()
 235                 p.wait()
 236             elif p.returncode:
 237                 if self.standalone:
 238                     sys.stdout.write('FAIL (trace)\n')
 239                 else:
 240                     sys.stdout.write('SKIP (app)\n')
 241                 return
 242
 243         if not os.path.isfile(trace):
 244             sys.stdout.write('SKIP (no trace)\n')
 245             return
 246
 247         p = popen([self._get_build_path('tracedump'), trace], stdout=subprocess.PIPE)
 248         call_re = re.compile('^([0-9]+) (\w+)\(')
 249         swapbuffers = 0
 250         flushes = 0
 251         for orig_line in p.stdout:
 252             orig_line = orig_line.rstrip()
 253             line = ansi_strip(orig_line)
 254             mo = call_re.match(line)
 255             if mo:
 256                 call_no = int(mo.group(1))
 257                 function_name = mo.group(2)
 258                 if function_name in ignored_function_names:
 259                     continue
 260                 if function_name == 'glXSwapBuffers':
 261                     swapbuffers += 1
 262                 if function_name in ('glFlush', 'glFinish'):
 263                     flushes += 1
 264         p.wait()
 265         if p.returncode != 0:
 266             sys.stdout.write('FAIL (tracedump)\n')
 267             return
 268
 269         args = [self._get_build_path('glretrace')]
 270         if swapbuffers:
 271             args += ['-db']
 272             frames = swapbuffers
 273         else:
 274             args += ['-sb']
 275             frames = flushes
 276         args += ['-s', src_prefix]
 277         args += [trace]
 278         p = popen(args, stdout=subprocess.PIPE)
 279         image_re = re.compile('^Wrote (.*\.png)$')
 280         images = []
 281         for line in p.stdout:
 282             line = line.rstrip()
 283             mo = image_re.match(line)
 284             if mo:
 285                 image = mo.group(1)
 286                 if image.startswith(src_prefix):
 287                     image = image[len(src_prefix):]
 288                     images.append(image)
 289         p.wait()
 290         if p.returncode != 0:
 291             sys.stdout.write('FAIL (glretrace)\n')
 292             return
 293
 294         for image in images:
 295             ref_image = ref_prefix + image
 296             src_image = src_prefix + image
 297             diff_image = diff_prefix + image
 298
 299             if not os.path.isfile(ref_image):
 300                 continue
 301             assert os.path.isfile(src_image)
 302
 303             comparer = Comparer(ref_image, src_image)
 304             match = comparer.ae()
 305             sys.stdout.write('%s: %s bits\n' % (image, comparer.precision()))
 306             if not match:
 307                 comparer.write_diff(diff_image)
 308                 report.add_snapshot(ref_image, src_image, diff_image)
 309                 sys.stdout.write('FAIL (snapshot)\n')
 310                 return
 311
 312         sys.stdout.write('PASS\n')
 313
 314
 315 def main():
 316     # Parse command line options
 317     optparser = optparse.OptionParser(
 318         usage='\n\t%prog [options] -- program [args] ...',
 319         version='%%prog')
 320     optparser.add_option(
 321         '-B', '--build', metavar='PATH',
 322         type='string', dest='build', default=None,
 323         help='path to apitrace build')
 324     optparser.add_option(
 325         '-C', '--directory', metavar='PATH',
 326         type='string', dest='cwd', default=None,
 327         help='change to directory')
 328     optparser.add_option(
 329         '-R', '--results', metavar='PATH',
 330         type='string', dest='results', default='results',
 331         help='results directory [default=%default]')
 332
 333     (options, args) = optparser.parse_args(sys.argv[1:])
 334     if not args:
 335         optparser.error('program must be specified')
 336
 337     test = TestCase(
 338         name = os.path.basename(args[0]),
 339         args = args,
 340         cwd = options.cwd,
 341         build = options.build,
 342         results = options.results,
 343     )
 344
 345     report = Report(options.results)
 346     test.run(report)
 347
 348
 349 if __name__ == '__main__':
 350     main()