2 ##########################################################################
4 # Copyright 2011 Jose Fonseca
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 ##########################################################################/
27 '''Main test driver.'''
44 from cStringIO import StringIO
46 from StringIO import StringIO
49 def _exit(status, code, reason=None):
53 reason = ' (%s)' % reason
54 sys.stdout.write('%s%s\n' % (status, reason))
57 def fail(reason=None):
58 _exit('FAIL', 1, reason)
60 def skip(reason=None):
61 _exit('SKIP', 0, reason)
63 def pass_(reason=None):
64 _exit('PASS', 0, reason)
67 def popen(command, *args, **kwargs):
68 if kwargs.get('cwd', None) is not None:
69 sys.stdout.write('cd %s && ' % kwargs['cwd'])
72 env = kwargs.pop('env')
80 if value != os.environ.get(name, None):
81 sys.stdout.write('%s=%s ' % (name, value))
82 env[name] = str(value)
84 sys.stdout.write(' '.join(command) + '\n')
87 return subprocess.Popen(command, *args, env=env, **kwargs)
90 def _get_build_path(path):
91 if options.build is not None:
92 path = os.path.abspath(os.path.join(options.build, path))
93 if not os.path.exists(path):
94 sys.stderr.write('error: %s does not exist\n' % path)
98 def _get_build_program(program):
99 if platform.system() == 'Windows':
101 return _get_build_path(program)
103 def _get_source_path(path):
104 cache = _get_build_path('CMakeCache.txt')
105 for line in open(cache, 'rt'):
106 if line.startswith('CMAKE_HOME_DIRECTORY:INTERNAL='):
107 _, source_root = line.strip().split('=', 1)
108 return os.path.join(source_root, path)
114 def __init__(self, srcStream, refFileName, verbose=False):
115 self.srcStream = srcStream
116 self.refFileName = refFileName
118 self.refStream = open(refFileName, 'rt')
120 self.refStream = None
121 self.verbose = verbose
122 self.doubleBuffer = False
128 call_re = re.compile(r'^([0-9]+) (\w+)\(')
136 self.consumeRefLine()
137 for line in self.srcStream:
140 sys.stdout.write(line + '\n')
141 mo = self.call_re.match(line)
143 self.callNo = int(mo.group(1))
144 function_name = mo.group(2)
145 if function_name.find('SwapBuffers') != -1 or \
146 line.find('kCGLPFADoubleBuffer') != -1:
148 if function_name in ('glFlush', 'glFinish'):
150 srcLine = line[mo.start(2):]
154 if srcLine == self.refLine:
155 self.consumeRefLine()
158 srcLines.append(srcLine)
162 fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0]))
164 fail('missing call %s' % self.refLine)
167 self.doubleBuffer = True
169 self.doubleBuffer = False
171 def consumeRefLine(self):
172 if not self.refStream:
177 line = self.refStream.readline()
181 if line.startswith('#'):
182 self.handlePragma(line)
187 def handlePragma(self, line):
188 pragma, rest = line.split(None, 1)
189 if pragma == '#image':
190 imageFileName = self.getAbsPath(rest)
191 self.images.append((self.callNo, imageFileName))
192 elif pragma == '#state':
193 stateFileName = self.getAbsPath(rest)
194 self.states.append((self.callNo, stateFileName))
198 def getAbsPath(self, path):
199 '''Get the absolute from a path relative to the reference filename'''
200 return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path))
219 threshold_precision = 12.0
225 '''Run the application standalone, skipping this test if it fails by
231 p = popen(self.cmd, cwd=self.cwd)
234 skip('application returned code %i' % p.returncode)
247 if self.trace_file is None:
248 if self.ref_dump is not None:
252 name, ext = os.path.splitext(os.path.basename(name))
254 name, ext = os.path.splitext(os.path.basename(name))
255 self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
256 if os.path.exists(self.trace_file):
257 os.remove(self.trace_file)
259 trace_dir = os.path.dirname(self.trace_file)
260 if not os.path.exists(trace_dir):
261 os.makedirs(trace_dir)
264 env = os.environ.copy()
266 system = platform.system()
268 if system == 'Windows':
269 wrapper = _get_build_path('wrappers/opengl32.dll')
270 local_wrapper = os.path.join(os.path.dirname(self.cmd[0]), os.path.basename(wrapper))
271 shutil.copy(wrapper, local_wrapper)
272 env['TRACE_FILE'] = str(self.trace_file)
274 apitrace = _get_build_program('apitrace')
277 '--api', self.api_map[self.api],
278 '--output', self.trace_file,
281 if self.max_frames is not None:
282 env['TRACE_FRAMES'] = str(self.max_frames)
285 p = popen(cmd, env=env, cwd=self.cwd)
288 if local_wrapper is not None:
289 os.remove(local_wrapper)
291 if not os.path.exists(self.trace_file):
292 fail('no trace file generated\n')
294 def checkTrace(self):
295 cmd = [_get_build_program('apitrace'), 'dump', '--color=never', self.trace_file]
296 p = popen(cmd, stdout=subprocess.PIPE)
298 checker = TraceChecker(p.stdout, self.ref_dump, self.verbose)
301 if p.returncode != 0:
302 fail('`apitrace dump` returned code %i' % p.returncode)
304 self.doubleBuffer = checker.doubleBuffer
306 for callNo, refImageFileName in checker.images:
307 self.checkImage(callNo, refImageFileName)
308 for callNo, refStateFileName in checker.states:
309 self.checkState(callNo, refStateFileName)
311 def checkImage(self, callNo, refImageFileName):
312 srcImage = self.getImage(callNo)
313 refImage = Image.open(refImageFileName)
315 from snapdiff import Comparer
316 comparer = Comparer(refImage, srcImage)
317 precision = comparer.precision(filter=True)
318 sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
319 if precision < self.threshold_precision:
320 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
321 srcImageFileName = prefix + '.src.png'
322 srcImage.save(srcImageFileName)
323 diffImageFileName = prefix + '.diff.png'
324 comparer.write_diff(diffImageFileName)
325 fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
327 def checkState(self, callNo, refStateFileName):
328 srcState = self.getState(callNo)
329 refState = self.getRefState(refStateFileName)
331 from jsondiff import Comparer, Differ
332 comparer = Comparer(ignore_added = True)
333 match = comparer.visit(refState, srcState)
335 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
336 srcStateFileName = prefix + '.src.json'
337 diffStateFileName = prefix + '.diff.json'
338 self.saveState(srcState, srcStateFileName)
339 #diffStateFile = open(diffStateFileName, 'wt')
340 diffStateFile = sys.stdout
341 differ = Differ(diffStateFile, ignore_added = True)
342 differ.visit(refState, srcState)
343 fail('state from call %u does not match %s' % (callNo, refStateFileName))
345 def getRefState(self, refStateFileName):
346 stream = open(refStateFileName, 'rt')
347 from jsondiff import load
349 self.adjustRefState(state)
352 def getNamePrefix(self):
353 name = os.path.basename(self.ref_dump)
355 index = name.index('.')
362 def saveState(self, state, filename):
363 s = json.dumps(state, sort_keys=True, indent=2)
364 open(filename, 'wt').write(s)
369 if p.returncode != 0:
370 fail('retrace failed with code %i' % (p.returncode))
372 def getImage(self, callNo):
373 state = self.getState(callNo)
374 if self.doubleBuffer:
375 attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0']
377 attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0']
378 imageObj = self.getFramebufferAttachment(state, attachments)
379 data = imageObj['__data__']
380 stream = StringIO(base64.b64decode(data))
381 im = Image.open(stream)
385 def getFramebufferAttachment(self, state, attachments):
386 framebufferObj = state['framebuffer']
387 for attachment in attachments:
389 attachmentObj = framebufferObj[attachment]
394 raise Exception("no attachment found")
396 def getState(self, callNo):
398 state = self.stateCache[callNo]
404 p = self._retrace(['-D', str(callNo)])
405 state = json.load(p.stdout, strict=False)
407 if p.returncode != 0:
408 fail('retrace returned code %i' % (p.returncode))
410 self.adjustSrcState(state)
412 self.stateCache[callNo] = state
416 def adjustSrcState(self, state):
417 # Do some adjustments on the obtained state to eliminate failures from
418 # bugs/issues outside of apitrace
421 parameters = state['parameters']
425 # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1
426 self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
429 if 'Gallium' in parameters['GL_RENDERER'].split():
430 # Gallium drivers have wrong defaults for draw/read buffer state
431 self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
432 self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK')
433 self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
434 self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
435 self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT')
436 self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
438 def adjustRefState(self, state):
439 # Do some adjustments on reference state to eliminate failures from
440 # bugs/issues outside of apitrace
443 parameters = state['parameters']
447 if platform.system() == 'Darwin':
448 # Mac OS X drivers fail on GL_COLOR_SUM
449 # XXX: investigate this
450 self.removeState(parameters, 'GL_COLOR_SUM')
452 def replaceState(self, obj, key, srcValue, dstValue):
458 if value == srcValue:
461 def removeState(self, obj, key):
467 def _retrace(self, args = None, stdout=subprocess.PIPE):
468 retrace = self.api_map[self.api] + 'retrace'
469 cmd = [_get_build_program(retrace)]
470 if self.doubleBuffer:
476 cmd += [self.trace_file]
477 return popen(cmd, stdout=stdout)
491 # Parse command line options
492 optparser = optparse.OptionParser(
493 usage='\n\t%prog [options] -- [TRACE|PROGRAM] ...',
495 optparser.add_option(
498 dest="verbose", default=False,
499 help="verbose output")
500 optparser.add_option(
501 '-a', '--api', metavar='API',
502 type='string', dest='api', default='gl',
504 optparser.add_option(
505 '-B', '--build', metavar='PATH',
506 type='string', dest='build', default='..',
507 help='path to apitrace build')
508 optparser.add_option(
509 '-C', '--directory', metavar='PATH',
510 type='string', dest='cwd', default=None,
511 help='change to directory')
512 optparser.add_option(
513 '-R', '--results', metavar='PATH',
514 type='string', dest='results', default='.',
515 help='results directory [default=%default]')
516 optparser.add_option(
517 '--ref-dump', metavar='PATH',
518 type='string', dest='ref_dump', default=None,
519 help='reference dump')
521 (options, args) = optparser.parse_args(sys.argv[1:])
523 optparser.error('an argument must be specified')
525 if not os.path.exists(options.results):
526 os.makedirs(options.results)
528 sys.path.insert(0, _get_source_path('scripts'))
531 test.verbose = options.verbose
533 if args[0].endswith('.trace'):
534 test.trace_file = args[0]
537 test.cwd = options.cwd
538 test.api = options.api
539 test.ref_dump = options.ref_dump
540 test.results = options.results
545 if __name__ == '__main__':