2 ##########################################################################
4 # Copyright 2011 Jose Fonseca
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 ##########################################################################/
27 '''Common test suite code.'''
47 def __init__(self, ref_image, src_image, alpha = False):
48 self.ref_im = Image.open(ref_image)
49 self.src_im = Image.open(src_image)
51 # Crop to the minimum size
52 ref_w, ref_h = self.ref_im.size
53 src_w, src_h = self.src_im.size
56 self.ref_im = self.ref_im.crop((0, ref_h - h, w, ref_h))
57 self.src_im = self.src_im.crop((0, src_h - h, w, src_h))
61 self.ref_im = self.ref_im.convert('RGB')
62 self.src_im = self.src_im.convert('RGB')
64 self.diff = ImageChops.difference(self.src_im, self.ref_im)
66 def write_diff(self, diff_image, fuzz = 0.05):
67 # make a difference image similar to ImageMagick's compare utility
68 mask = ImageEnhance.Brightness(self.diff).enhance(1.0/fuzz)
69 mask = mask.convert('L')
71 lowlight = Image.new('RGB', self.src_im.size, (0xff, 0xff, 0xff))
72 highlight = Image.new('RGB', self.src_im.size, (0xf1, 0x00, 0x1e))
73 diff_im = Image.composite(highlight, lowlight, mask)
75 diff_im = Image.blend(self.src_im, diff_im, 0xcc/255.0)
76 diff_im.save(diff_image)
79 # See also http://effbot.org/zone/pil-comparing-images.htm
80 h = self.diff.histogram()
82 for i in range(1, 256):
83 square_error += sum(h[i : 3*256: 256])*i*i
84 rel_error = float(square_error*2 + 1) / float(self.diff.size[0]*self.diff.size[1]*3*255*255*2)
85 bits = -math.log(rel_error)/math.log(2.0)
88 def ae(self, chantol = 4, pixeltol = 0.03):
89 # Compute absolute error
90 # chantol = color channel tolerance
91 # pixeltol = ratio of pixels we allow to go completely off
93 # TODO: this is approximate due to the grayscale conversion
94 h = self.diff.convert('L').histogram()
96 ae = sum(h[int(chantol) + 1 : 256])
98 return ae <= pixeltol*self.diff.size[0]*self.diff.size[1]
101 ansi_re = re.compile('\x1b\[[0-9]{1,2}(;[0-9]{1,2}){0,2}m')
105 # http://www.theeggeadventure.com/wikimedia/index.php/Linux_Tips#Use_sed_to_remove_ANSI_colors
106 return ansi_re.sub('', s)
109 def popen(command, *args, **kwargs):
111 sys.stdout.write('cd %s && ' % kwargs['cwd'])
113 for name, value in kwargs['env'].iteritems():
114 if value != os.environ.get(name, None):
115 sys.stdout.write('%s=%s ' % (name, value))
116 sys.stdout.write(' '.join(command) + '\n')
118 return subprocess.Popen(command, *args, **kwargs)
121 ignored_function_names = set([
123 'glXGetCurrentDisplay',
124 'glXGetClientString',
126 'glXGetProcAddressARB',
128 'glXGetVisualFromFBConfig',
130 'glXCreateNewContext',
131 'glXMakeContextCurrent',
139 def __init__(self, basedir):
140 self.basedir = basedir
141 if not os.path.exists(basedir):
143 self.html = open(os.path.join(basedir, 'index.html'), 'wt')
147 self.html.write('<html>\n')
148 self.html.write(' <body>\n')
149 self.html.write(' <table border="1">\n')
150 self.html.write(' <tr><th>Ref</th><th>Src</th><th>Δ</th></tr>\n')
152 def _image_tag(self, image):
153 url = os.path.relpath(image, self.basedir)
154 self.html.write(' <td><a href="%s"><img src="%s"/></a></td>\n' % (url, url))
156 def add_snapshot(self, ref_image, src_image, diff_image):
157 self.html.write(' <tr>\n')
158 self._image_tag(ref_image)
159 self._image_tag(src_image)
160 self._image_tag(diff_image)
161 self.html.write(' </tr>\n')
165 self.html.write(' </table>\n')
166 self.html.write(' </body>\n')
167 self.html.write('</html>\n')
180 def __init__(self, name, args, cwd=None, build=None, results = '.'):
185 self.results = results
187 if not os.path.exists(results):
191 def _get_build_path(self, path):
192 if self.build is not None:
193 path = os.path.abspath(os.path.join(self.build, path))
194 if not os.path.exists(path):
195 sys.stderr.write('error: %s does not exist\n' % path)
199 def run(self, report):
201 trace = os.path.abspath(os.path.join(self.results, self.name + '.trace'))
202 ref_prefix = os.path.abspath(os.path.join(self.results, self.name + '.ref.'))
203 src_prefix = os.path.join(self.results, self.name + '.src.')
204 diff_prefix = os.path.join(self.results, self.name + '.diff.')
206 ld_preload = self._get_build_path('glxtrace.so')
208 env = os.environ.copy()
209 env['LD_PRELOAD'] = ld_preload
210 env['TRACE_FILE'] = trace
211 env['TRACE_SNAPSHOT'] = ref_prefix
212 env['TRACE_FRAMES'] = str(self.max_frames)
215 p = popen(self.args, cwd=self.cwd)
218 if p.poll() is not None:
220 if p.returncode is None:
223 sys.stdout.write('SKIP (app)\n')
226 p = popen(self.args, env=env, cwd=self.cwd)
230 if p.poll() is not None:
233 if p.returncode is None:
238 sys.stdout.write('FAIL (trace)\n')
240 sys.stdout.write('SKIP (app)\n')
243 if not os.path.isfile(trace):
244 sys.stdout.write('SKIP (no trace)\n')
247 p = popen([self._get_build_path('tracedump'), trace], stdout=subprocess.PIPE)
248 call_re = re.compile('^([0-9]+) (\w+)\(')
251 for orig_line in p.stdout:
252 orig_line = orig_line.rstrip()
253 line = ansi_strip(orig_line)
254 mo = call_re.match(line)
256 call_no = int(mo.group(1))
257 function_name = mo.group(2)
258 if function_name in ignored_function_names:
260 if function_name == 'glXSwapBuffers':
262 if function_name in ('glFlush', 'glFinish'):
265 if p.returncode != 0:
266 sys.stdout.write('FAIL (tracedump)\n')
269 args = [self._get_build_path('glretrace')]
276 args += ['-s', src_prefix]
278 p = popen(args, stdout=subprocess.PIPE)
279 image_re = re.compile('^Wrote (.*\.png)$')
281 for line in p.stdout:
283 mo = image_re.match(line)
286 if image.startswith(src_prefix):
287 image = image[len(src_prefix):]
290 if p.returncode != 0:
291 sys.stdout.write('FAIL (glretrace)\n')
295 ref_image = ref_prefix + image
296 src_image = src_prefix + image
297 diff_image = diff_prefix + image
299 if not os.path.isfile(ref_image):
301 assert os.path.isfile(src_image)
303 comparer = Comparer(ref_image, src_image)
304 match = comparer.ae()
305 sys.stdout.write('%s: %s bits\n' % (image, comparer.precision()))
307 comparer.write_diff(diff_image)
308 report.add_snapshot(ref_image, src_image, diff_image)
309 sys.stdout.write('FAIL (snapshot)\n')
312 sys.stdout.write('PASS\n')
316 # Parse command line options
317 optparser = optparse.OptionParser(
318 usage='\n\t%prog [options] -- program [args] ...',
320 optparser.add_option(
321 '-B', '--build', metavar='PATH',
322 type='string', dest='build', default=None,
323 help='path to apitrace build')
324 optparser.add_option(
325 '-C', '--directory', metavar='PATH',
326 type='string', dest='cwd', default=None,
327 help='change to directory')
328 optparser.add_option(
329 '-R', '--results', metavar='PATH',
330 type='string', dest='results', default='results',
331 help='results directory [default=%default]')
333 (options, args) = optparser.parse_args(sys.argv[1:])
335 optparser.error('program must be specified')
338 name = os.path.basename(args[0]),
341 build = options.build,
342 results = options.results,
345 report = Report(options.results)
349 if __name__ == '__main__':