2 ##########################################################################
4 # Copyright 2011 Jose Fonseca
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 ##########################################################################/
27 '''Application test driver.'''
40 from cStringIO import StringIO
42 from StringIO import StringIO
46 from base_driver import *
49 class SrcTraceParser(tracematch.SrcTraceParser):
51 def __init__(self, stream):
52 tracematch.SrcTraceParser.__init__(self, stream)
55 def handleCall(self, callNo, functionName, args, ret):
56 tracematch.SrcTraceParser.handleCall(self, callNo, functionName, args, ret)
58 if functionName.find('SwapBuffers') != -1 or \
59 repr(args).find('kCGLPFADoubleBuffer') != -1:
63 class AppDriver(Driver):
78 threshold_precision = 12.0
85 '''Run the application standalone, skipping this test if it fails by
91 sys.stderr.write('Run application...\n')
93 p = popen(self.cmd, cwd=self.cwd)
96 skip('application returned code %i' % p.returncode)
99 sys.stderr.write('\n')
110 'd3d10_1': 'd3d10_1',
117 'egl_gl': 'eglretrace',
118 'egl_gles1': 'eglretrace',
119 'egl_gles2': 'eglretrace',
120 'd3d8': 'd3dretrace',
121 'd3d9': 'd3dretrace',
122 'd3d10': 'd3dretrace',
123 'd3d10_1': 'd3dretrace',
124 'd3d11': 'd3dretrace',
125 'd3d11_1': 'd3dretrace',
132 sys.stderr.write('Capturing trace...\n')
134 if self.trace_file is None:
135 if self.ref_dump is not None:
139 name, ext = os.path.splitext(os.path.basename(name))
141 name, ext = os.path.splitext(os.path.basename(name))
142 self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
143 if os.path.exists(self.trace_file):
144 os.remove(self.trace_file)
146 trace_dir = os.path.dirname(self.trace_file)
147 if not os.path.exists(trace_dir):
148 os.makedirs(trace_dir)
151 env = os.environ.copy()
154 options.apitrace, 'trace',
156 '-a', self.api_trace_map[self.api],
157 '-o', self.trace_file,
160 if self.max_frames is not None:
161 env['TRACE_FRAMES'] = str(self.max_frames)
163 p = popen(cmd, env=env, cwd=self.cwd)
165 if p.returncode != 0:
166 fail('`apitrace trace` returned code %i' % p.returncode)
168 if not os.path.exists(self.trace_file):
169 fail('no trace file generated\n')
172 sys.stderr.write('\n')
174 def checkTrace(self):
175 sys.stderr.write('Comparing trace %s against %s...\n' % (self.trace_file, self.ref_dump))
177 cmd = [options.apitrace, 'dump', '--verbose', '--color=never', self.trace_file]
178 p = popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
180 srcParser = SrcTraceParser(p.stdout)
181 srcTrace = srcParser.parse()
182 self.doubleBuffer = srcParser.swapbuffers > 0
188 refParser = tracematch.RefTraceParser(self.ref_dump)
189 refTrace = refParser.parse()
192 mo = refTrace.match(srcTrace)
193 except tracematch.TraceMismatch, ex:
196 dirName, baseName = os.path.split(os.path.abspath(self.ref_dump))
197 prefix, _ = os.path.splitext(baseName)
199 fileNames = os.listdir(dirName)
200 for fileName in fileNames:
201 if fileName.startswith(prefix) and fileName != self.ref_dump:
202 rest = fileName[len(prefix):]
203 paramName, ext = os.path.splitext(rest)
204 if ext in ('.json', '.png'):
205 if paramName.isdigit():
206 callNo = int(paramName)
209 callNo = mo.params[paramName]
211 fail('could not find parameter %s for %s' % (paramName, fileName))
212 filePath = os.path.join(dirName, fileName)
214 images.append((callNo, filePath))
216 states.append((callNo, filePath))
218 if p.returncode != 0:
219 fail('`apitrace dump` returned code %i' % p.returncode)
222 sys.stderr.write('\n')
224 if self.api not in self.api_retrace_map:
227 for callNo, refImageFileName in images:
228 self.checkImage(callNo, refImageFileName)
229 for callNo, refStateFileName in states:
230 self.checkState(callNo, refStateFileName)
232 def checkImage(self, callNo, refImageFileName):
233 sys.stderr.write('Comparing snapshot from call %u against %s...\n' % (callNo, refImageFileName))
235 from PIL import Image
237 sys.stderr.write('warning: PIL not found, skipping image comparison\n');
240 srcImage = self.getImage(callNo)
241 refImage = Image.open(refImageFileName)
243 from snapdiff import Comparer
244 comparer = Comparer(refImage, srcImage)
245 precision = comparer.precision(filter=True)
246 sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
247 if precision < self.threshold_precision:
248 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
249 srcImageFileName = prefix + '.src.png'
250 srcImage.save(srcImageFileName)
251 diffImageFileName = prefix + '.diff.png'
252 comparer.write_diff(diffImageFileName)
253 fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
256 sys.stderr.write('\n')
258 def checkState(self, callNo, refStateFileName):
259 sys.stderr.write('Comparing state dump from call %u against %s...\n' % (callNo, refStateFileName))
261 srcState = self.getState(callNo)
262 refState = self.getRefState(refStateFileName)
264 from jsondiff import Comparer, Differ
265 comparer = Comparer(ignore_added = True)
266 match = comparer.visit(refState, srcState)
268 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
269 srcStateFileName = prefix + '.src.json'
270 diffStateFileName = prefix + '.diff.json'
271 self.saveState(srcState, srcStateFileName)
272 #diffStateFile = open(diffStateFileName, 'wt')
273 diffStateFile = sys.stdout
274 differ = Differ(diffStateFile, ignore_added = True)
275 differ.visit(refState, srcState)
276 fail('state from call %u does not match %s' % (callNo, refStateFileName))
279 sys.stderr.write('\n')
281 def getRefState(self, refStateFileName):
282 stream = open(refStateFileName, 'rt')
283 from jsondiff import load
285 self.adjustRefState(state)
288 def getNamePrefix(self):
289 name = os.path.basename(self.ref_dump)
291 index = name.index('.')
298 def saveState(self, state, filename):
299 s = json.dumps(state, sort_keys=True, indent=2)
300 open(filename, 'wt').write(s)
303 if self.api not in self.api_retrace_map:
306 sys.stderr.write('Retracing %s...\n' % (self.trace_file,))
310 if p.returncode != 0:
311 fail('retrace failed with code %i' % (p.returncode))
314 sys.stderr.write('\n')
316 def getImage(self, callNo):
317 from PIL import Image
318 state = self.getState(callNo)
319 if self.doubleBuffer:
320 attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0', 'RENDER_TARGET_0']
322 attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0', 'RENDER_TARGET_0']
323 imageObj = self.getFramebufferAttachment(state, attachments)
324 data = imageObj['__data__']
325 stream = StringIO(base64.b64decode(data))
326 im = Image.open(stream)
329 def getFramebufferAttachment(self, state, attachments):
330 framebufferObj = state['framebuffer']
331 for attachment in attachments:
333 attachmentObj = framebufferObj[attachment]
338 raise Exception("no attachment found")
340 def getState(self, callNo):
342 state = self.stateCache[callNo]
348 p = self._retrace(['-D', str(callNo)])
349 state = json.load(p.stdout, strict=False)
351 if p.returncode != 0:
352 fail('retrace returned code %i' % (p.returncode))
354 self.adjustSrcState(state)
356 self.stateCache[callNo] = state
360 def adjustSrcState(self, state):
361 # Do some adjustments on the obtained state to eliminate failures from
362 # bugs/issues outside of apitrace
365 parameters = state['parameters']
369 # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns 255
370 self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
372 def adjustRefState(self, state):
373 # Do some adjustments on reference state to eliminate failures from
374 # bugs/issues outside of apitrace
377 parameters = state['parameters']
381 if platform.system() == 'Darwin':
382 # Mac OS X drivers fail on GL_COLOR_SUM
383 # XXX: investigate this
384 self.removeState(parameters, 'GL_COLOR_SUM')
386 def replaceState(self, obj, key, srcValue, dstValue):
392 if value == srcValue:
395 def removeState(self, obj, key):
401 def _retrace(self, args = None, stdout=subprocess.PIPE):
402 retrace = self.api_retrace_map[self.api]
403 #cmd = [get_build_program(retrace)]
404 cmd = [options.apitrace, 'retrace']
405 if self.doubleBuffer:
411 cmd += [self.trace_file]
412 return popen(cmd, stdout=stdout)
414 def createOptParser(self):
415 optparser = Driver.createOptParser(self)
417 optparser.add_option(
418 '-a', '--api', metavar='API',
419 type='string', dest='api', default='gl',
421 optparser.add_option(
422 '-R', '--results', metavar='PATH',
423 type='string', dest='results', default='.',
424 help='results directory [default=%default]')
425 optparser.add_option(
426 '--ref-dump', metavar='PATH',
427 type='string', dest='ref_dump', default=None,
428 help='reference dump')
435 (options, args) = self.parseOptions()
437 if not os.path.exists(options.results):
438 os.makedirs(options.results)
440 self.verbose = options.verbose
443 self.cwd = options.cwd
444 self.api = options.api
445 self.ref_dump = options.ref_dump
446 self.results = options.results
455 if __name__ == '__main__':