2 ##########################################################################
4 # Copyright 2011 Jose Fonseca
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 ##########################################################################/
27 '''Application test driver.'''
40 from cStringIO import StringIO
42 from StringIO import StringIO
45 from base_driver import *
50 def __init__(self, srcStream, refFileName, verbose=False):
51 self.srcStream = srcStream
52 self.refFileName = refFileName
54 self.refStream = open(refFileName, 'rt')
57 self.verbose = verbose
58 self.doubleBuffer = False
64 call_re = re.compile(r'^([0-9]+) (\w+)\(')
73 for line in self.srcStream:
76 sys.stdout.write(line + '\n')
77 mo = self.call_re.match(line)
79 self.callNo = int(mo.group(1))
80 function_name = mo.group(2)
81 if function_name.find('SwapBuffers') != -1 or \
82 line.find('kCGLPFADoubleBuffer') != -1:
84 if function_name in ('glFlush', 'glFinish'):
86 srcLine = line[mo.start(2):]
90 if srcLine == self.refLine:
94 srcLines.append(srcLine)
98 fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0]))
100 fail('missing call %s' % self.refLine)
103 self.doubleBuffer = True
105 self.doubleBuffer = False
107 def consumeRefLine(self):
108 if not self.refStream:
113 line = self.refStream.readline()
117 if line.startswith('#'):
118 self.handlePragma(line)
123 def handlePragma(self, line):
124 pragma, rest = line.split(None, 1)
125 if pragma == '#image':
126 imageFileName = self.getAbsPath(rest)
127 self.images.append((self.callNo, imageFileName))
128 elif pragma == '#state':
129 stateFileName = self.getAbsPath(rest)
130 self.states.append((self.callNo, stateFileName))
134 def getAbsPath(self, path):
135 '''Get the absolute from a path relative to the reference filename'''
136 return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path))
140 class AppDriver(Driver):
155 threshold_precision = 12.0
158 Driver.__init__(self)
162 '''Run the application standalone, skipping this test if it fails by
168 p = popen(self.cmd, cwd=self.cwd)
171 skip('application returned code %i' % p.returncode)
182 'd3d10_1': 'd3d10_1',
188 'egl_gl': 'eglretrace',
189 'egl_gles1': 'eglretrace',
190 'egl_gles2': 'eglretrace',
191 'd3d9': 'd3dretrace',
198 if self.trace_file is None:
199 if self.ref_dump is not None:
203 name, ext = os.path.splitext(os.path.basename(name))
205 name, ext = os.path.splitext(os.path.basename(name))
206 self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
207 if os.path.exists(self.trace_file):
208 os.remove(self.trace_file)
210 trace_dir = os.path.dirname(self.trace_file)
211 if not os.path.exists(trace_dir):
212 os.makedirs(trace_dir)
215 env = os.environ.copy()
218 options.apitrace, 'trace',
219 '--api', self.api_trace_map[self.api],
220 '--output', self.trace_file,
223 if self.max_frames is not None:
224 env['TRACE_FRAMES'] = str(self.max_frames)
226 p = popen(cmd, env=env, cwd=self.cwd)
229 if not os.path.exists(self.trace_file):
230 fail('no trace file generated\n')
232 def checkTrace(self):
233 cmd = [options.apitrace, 'dump', '--color=never', self.trace_file]
234 p = popen(cmd, stdout=subprocess.PIPE)
236 checker = TraceChecker(p.stdout, self.ref_dump, self.verbose)
239 if p.returncode != 0:
240 fail('`apitrace dump` returned code %i' % p.returncode)
242 self.doubleBuffer = checker.doubleBuffer
244 if self.api not in self.api_retrace_map:
247 for callNo, refImageFileName in checker.images:
248 self.checkImage(callNo, refImageFileName)
249 for callNo, refStateFileName in checker.states:
250 self.checkState(callNo, refStateFileName)
252 def checkImage(self, callNo, refImageFileName):
254 from PIL import Image
258 srcImage = self.getImage(callNo)
259 refImage = Image.open(refImageFileName)
261 from snapdiff import Comparer
262 comparer = Comparer(refImage, srcImage)
263 precision = comparer.precision(filter=True)
264 sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
265 if precision < self.threshold_precision:
266 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
267 srcImageFileName = prefix + '.src.png'
268 srcImage.save(srcImageFileName)
269 diffImageFileName = prefix + '.diff.png'
270 comparer.write_diff(diffImageFileName)
271 fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
273 def checkState(self, callNo, refStateFileName):
274 srcState = self.getState(callNo)
275 refState = self.getRefState(refStateFileName)
277 from jsondiff import Comparer, Differ
278 comparer = Comparer(ignore_added = True)
279 match = comparer.visit(refState, srcState)
281 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
282 srcStateFileName = prefix + '.src.json'
283 diffStateFileName = prefix + '.diff.json'
284 self.saveState(srcState, srcStateFileName)
285 #diffStateFile = open(diffStateFileName, 'wt')
286 diffStateFile = sys.stdout
287 differ = Differ(diffStateFile, ignore_added = True)
288 differ.visit(refState, srcState)
289 fail('state from call %u does not match %s' % (callNo, refStateFileName))
291 def getRefState(self, refStateFileName):
292 stream = open(refStateFileName, 'rt')
293 from jsondiff import load
295 self.adjustRefState(state)
298 def getNamePrefix(self):
299 name = os.path.basename(self.ref_dump)
301 index = name.index('.')
308 def saveState(self, state, filename):
309 s = json.dumps(state, sort_keys=True, indent=2)
310 open(filename, 'wt').write(s)
313 if self.api not in self.api_retrace_map:
318 if p.returncode != 0:
319 fail('retrace failed with code %i' % (p.returncode))
321 def getImage(self, callNo):
322 from PIL import Image
323 state = self.getState(callNo)
324 if self.doubleBuffer:
325 attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0']
327 attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0']
328 imageObj = self.getFramebufferAttachment(state, attachments)
329 data = imageObj['__data__']
330 stream = StringIO(base64.b64decode(data))
331 im = Image.open(stream)
335 def getFramebufferAttachment(self, state, attachments):
336 framebufferObj = state['framebuffer']
337 for attachment in attachments:
339 attachmentObj = framebufferObj[attachment]
344 raise Exception("no attachment found")
346 def getState(self, callNo):
348 state = self.stateCache[callNo]
354 p = self._retrace(['-D', str(callNo)])
355 state = json.load(p.stdout, strict=False)
357 if p.returncode != 0:
358 fail('retrace returned code %i' % (p.returncode))
360 self.adjustSrcState(state)
362 self.stateCache[callNo] = state
366 def adjustSrcState(self, state):
367 # Do some adjustments on the obtained state to eliminate failures from
368 # bugs/issues outside of apitrace
371 parameters = state['parameters']
375 # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1
376 self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
379 if 'Gallium' in parameters['GL_RENDERER'].split():
380 # Gallium drivers have wrong defaults for draw/read buffer state
381 self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
382 self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK')
383 self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
384 self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
385 self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT')
386 self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
388 def adjustRefState(self, state):
389 # Do some adjustments on reference state to eliminate failures from
390 # bugs/issues outside of apitrace
393 parameters = state['parameters']
397 if platform.system() == 'Darwin':
398 # Mac OS X drivers fail on GL_COLOR_SUM
399 # XXX: investigate this
400 self.removeState(parameters, 'GL_COLOR_SUM')
402 def replaceState(self, obj, key, srcValue, dstValue):
408 if value == srcValue:
411 def removeState(self, obj, key):
417 def _retrace(self, args = None, stdout=subprocess.PIPE):
418 retrace = self.api_retrace_map[self.api]
419 cmd = [get_build_program(retrace)]
420 if self.doubleBuffer:
426 cmd += [self.trace_file]
427 return popen(cmd, stdout=stdout)
429 def createOptParser(self):
430 optparser = Driver.createOptParser(self)
432 optparser.add_option(
433 '-a', '--api', metavar='API',
434 type='string', dest='api', default='gl',
436 optparser.add_option(
437 '-R', '--results', metavar='PATH',
438 type='string', dest='results', default='.',
439 help='results directory [default=%default]')
440 optparser.add_option(
441 '--ref-dump', metavar='PATH',
442 type='string', dest='ref_dump', default=None,
443 help='reference dump')
450 (options, args) = self.parseOptions()
452 if not os.path.exists(options.results):
453 os.makedirs(options.results)
455 self.verbose = options.verbose
458 self.cwd = options.cwd
459 self.api = options.api
460 self.ref_dump = options.ref_dump
461 self.results = options.results
470 if __name__ == '__main__':