2 ##########################################################################
4 # Copyright 2011 Jose Fonseca
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 ##########################################################################/
27 '''Application test driver.'''
40 from cStringIO import StringIO
42 from StringIO import StringIO
46 from base_driver import *
49 class RefTraceParser(tracematch.RefTraceParser):
51 def __init__(self, fileName):
52 tracematch.RefTraceParser.__init__(self, open(fileName, 'rt'))
53 self.fileName = fileName
58 def handlePragma(self, line):
60 lastCall = self.calls[-1]
61 if lastCall.callNo is None:
62 paramName = 'pragma%u' % self.pragmaNo
63 lastCall.callNo = tracematch.WildcardMatcher(paramName)
65 paramName = lastCall.callNo.name
70 pragma, rest = line.split(None, 1)
71 if pragma == '#image':
72 imageFileName = self.getAbsPath(rest)
73 self.images.append((paramName, imageFileName))
74 elif pragma == '#state':
75 stateFileName = self.getAbsPath(rest)
76 self.states.append((paramName, stateFileName))
80 def getAbsPath(self, path):
81 '''Get the absolute from a path relative to the reference filename'''
82 return os.path.abspath(os.path.join(os.path.dirname(self.fileName), path))
85 class SrcTraceParser(tracematch.SrcTraceParser):
87 def __init__(self, stream):
88 tracematch.SrcTraceParser.__init__(self, stream)
91 def handleCall(self, callNo, functionName, args, ret):
92 tracematch.SrcTraceParser.handleCall(self, callNo, functionName, args, ret)
94 if functionName.find('SwapBuffers') != -1 or \
95 repr(args).find('kCGLPFADoubleBuffer') != -1:
101 def __init__(self, srcStream, refFileName):
102 self.srcStream = srcStream
103 self.refFileName = refFileName
104 self.doubleBuffer = False
110 srcParser = SrcTraceParser(self.srcStream)
111 srcTrace = srcParser.parse()
112 self.doubleBuffer = srcParser.swapbuffers > 0
115 refParser = RefTraceParser(self.refFileName)
116 refTrace = refParser.parse()
119 mo = refTrace.match(srcTrace)
120 except tracematch.TraceMismatch, ex:
123 for paramName, imageFileName in refParser.images:
124 if isinstance(paramName, int):
127 callNo = mo.params[paramName]
128 self.images.append((callNo, imageFileName))
129 for paramName, stateFileName in refParser.states:
130 if isinstance(paramName, int):
133 callNo = mo.params[paramName]
134 self.states.append((callNo, stateFileName))
137 class AppDriver(Driver):
152 threshold_precision = 12.0
155 Driver.__init__(self)
159 '''Run the application standalone, skipping this test if it fails by
165 p = popen(self.cmd, cwd=self.cwd)
167 if p.returncode != 0:
168 skip('application returned code %i' % p.returncode)
179 'd3d10_1': 'd3d10_1',
186 'egl_gl': 'eglretrace',
187 'egl_gles1': 'eglretrace',
188 'egl_gles2': 'eglretrace',
189 #'d3d8': 'd3dretrace',
190 'd3d9': 'd3dretrace',
191 'd3d10': 'd3dretrace',
192 'd3d10_1': 'd3dretrace',
193 'd3d11': 'd3dretrace',
194 'd3d11_1': 'd3dretrace',
201 if self.trace_file is None:
202 if self.ref_dump is not None:
206 name, ext = os.path.splitext(os.path.basename(name))
208 name, ext = os.path.splitext(os.path.basename(name))
209 self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
210 if os.path.exists(self.trace_file):
211 os.remove(self.trace_file)
213 trace_dir = os.path.dirname(self.trace_file)
214 if not os.path.exists(trace_dir):
215 os.makedirs(trace_dir)
218 env = os.environ.copy()
221 options.apitrace, 'trace',
222 '--api', self.api_trace_map[self.api],
223 '--output', self.trace_file,
226 if self.max_frames is not None:
227 env['TRACE_FRAMES'] = str(self.max_frames)
229 p = popen(cmd, env=env, cwd=self.cwd)
231 if p.returncode != 0:
232 fail('`apitrace trace` returned code %i' % p.returncode)
234 if not os.path.exists(self.trace_file):
235 fail('no trace file generated\n')
237 def checkTrace(self):
238 cmd = [options.apitrace, 'dump', '--color=never', self.trace_file]
239 p = popen(cmd, stdout=subprocess.PIPE)
241 checker = TraceChecker(p.stdout, self.ref_dump)
244 if p.returncode != 0:
245 fail('`apitrace dump` returned code %i' % p.returncode)
247 self.doubleBuffer = checker.doubleBuffer
249 if self.api not in self.api_retrace_map:
252 for callNo, refImageFileName in checker.images:
253 self.checkImage(callNo, refImageFileName)
254 for callNo, refStateFileName in checker.states:
255 self.checkState(callNo, refStateFileName)
257 def checkImage(self, callNo, refImageFileName):
259 from PIL import Image
263 srcImage = self.getImage(callNo)
264 refImage = Image.open(refImageFileName)
266 from snapdiff import Comparer
267 comparer = Comparer(refImage, srcImage)
268 precision = comparer.precision(filter=True)
269 sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
270 if precision < self.threshold_precision:
271 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
272 srcImageFileName = prefix + '.src.png'
273 srcImage.save(srcImageFileName)
274 diffImageFileName = prefix + '.diff.png'
275 comparer.write_diff(diffImageFileName)
276 fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
278 def checkState(self, callNo, refStateFileName):
279 srcState = self.getState(callNo)
280 refState = self.getRefState(refStateFileName)
282 from jsondiff import Comparer, Differ
283 comparer = Comparer(ignore_added = True)
284 match = comparer.visit(refState, srcState)
286 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
287 srcStateFileName = prefix + '.src.json'
288 diffStateFileName = prefix + '.diff.json'
289 self.saveState(srcState, srcStateFileName)
290 #diffStateFile = open(diffStateFileName, 'wt')
291 diffStateFile = sys.stdout
292 differ = Differ(diffStateFile, ignore_added = True)
293 differ.visit(refState, srcState)
294 fail('state from call %u does not match %s' % (callNo, refStateFileName))
296 def getRefState(self, refStateFileName):
297 stream = open(refStateFileName, 'rt')
298 from jsondiff import load
300 self.adjustRefState(state)
303 def getNamePrefix(self):
304 name = os.path.basename(self.ref_dump)
306 index = name.index('.')
313 def saveState(self, state, filename):
314 s = json.dumps(state, sort_keys=True, indent=2)
315 open(filename, 'wt').write(s)
318 if self.api not in self.api_retrace_map:
323 if p.returncode != 0:
324 fail('retrace failed with code %i' % (p.returncode))
326 def getImage(self, callNo):
327 from PIL import Image
328 state = self.getState(callNo)
329 if self.doubleBuffer:
330 attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0']
332 attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0']
333 imageObj = self.getFramebufferAttachment(state, attachments)
334 data = imageObj['__data__']
335 stream = StringIO(base64.b64decode(data))
336 im = Image.open(stream)
340 def getFramebufferAttachment(self, state, attachments):
341 framebufferObj = state['framebuffer']
342 for attachment in attachments:
344 attachmentObj = framebufferObj[attachment]
349 raise Exception("no attachment found")
351 def getState(self, callNo):
353 state = self.stateCache[callNo]
359 p = self._retrace(['-D', str(callNo)])
360 state = json.load(p.stdout, strict=False)
362 if p.returncode != 0:
363 fail('retrace returned code %i' % (p.returncode))
365 self.adjustSrcState(state)
367 self.stateCache[callNo] = state
371 def adjustSrcState(self, state):
372 # Do some adjustments on the obtained state to eliminate failures from
373 # bugs/issues outside of apitrace
376 parameters = state['parameters']
380 # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns 255
381 self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
383 def adjustRefState(self, state):
384 # Do some adjustments on reference state to eliminate failures from
385 # bugs/issues outside of apitrace
388 parameters = state['parameters']
392 if platform.system() == 'Darwin':
393 # Mac OS X drivers fail on GL_COLOR_SUM
394 # XXX: investigate this
395 self.removeState(parameters, 'GL_COLOR_SUM')
397 def replaceState(self, obj, key, srcValue, dstValue):
403 if value == srcValue:
406 def removeState(self, obj, key):
412 def _retrace(self, args = None, stdout=subprocess.PIPE):
413 retrace = self.api_retrace_map[self.api]
414 #cmd = [get_build_program(retrace)]
415 cmd = [options.apitrace, 'retrace']
416 if self.doubleBuffer:
422 cmd += [self.trace_file]
423 return popen(cmd, stdout=stdout)
425 def createOptParser(self):
426 optparser = Driver.createOptParser(self)
428 optparser.add_option(
429 '-a', '--api', metavar='API',
430 type='string', dest='api', default='gl',
432 optparser.add_option(
433 '-R', '--results', metavar='PATH',
434 type='string', dest='results', default='.',
435 help='results directory [default=%default]')
436 optparser.add_option(
437 '--ref-dump', metavar='PATH',
438 type='string', dest='ref_dump', default=None,
439 help='reference dump')
446 (options, args) = self.parseOptions()
448 if not os.path.exists(options.results):
449 os.makedirs(options.results)
451 self.verbose = options.verbose
454 self.cwd = options.cwd
455 self.api = options.api
456 self.ref_dump = options.ref_dump
457 self.results = options.results
466 if __name__ == '__main__':