2 ##########################################################################
4 # Copyright 2011 Jose Fonseca
7 # Permission is hereby granted, free of charge, to any person obtaining a copy
8 # of this software and associated documentation files (the "Software"), to deal
9 # in the Software without restriction, including without limitation the rights
10 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 # copies of the Software, and to permit persons to whom the Software is
12 # furnished to do so, subject to the following conditions:
14 # The above copyright notice and this permission notice shall be included in
15 # all copies or substantial portions of the Software.
17 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
25 ##########################################################################/
27 '''Main test driver.'''
41 from cStringIO import StringIO
43 from StringIO import StringIO
46 def _exit(status, code, reason=None):
50 reason = ' (%s)' % reason
51 sys.stdout.write('%s%s\n' % (status, reason))
54 def fail(reason=None):
55 _exit('FAIL', 1, reason)
57 def skip(reason=None):
58 _exit('SKIP', 0, reason)
60 def pass_(reason=None):
61 _exit('PASS', 0, reason)
64 def popen(command, *args, **kwargs):
65 if kwargs.get('cwd', None) is not None:
66 sys.stdout.write('cd %s && ' % kwargs['cwd'])
69 env = kwargs.pop('env')
77 if value != os.environ.get(name, None):
78 sys.stdout.write('%s=%s ' % (name, value))
79 env[name] = str(value)
81 sys.stdout.write(' '.join(command) + '\n')
84 return subprocess.Popen(command, *args, env=env, **kwargs)
87 def which(executable):
88 dirs = os.environ['PATH'].split(os.path.pathsep)
90 path = os.path.join(dir, executable)
91 if os.path.exists(path):
97 if os.path.exists(options.apitrace):
98 apitrace_abspath = os.path.abspath(options.apitrace)
100 apitrace_abspath = which(options.apitrace)
101 if apitrace_abspath is None:
102 sys.stderr.write('error: could not determine the absolute path of\n' % options.apitrace)
104 return os.path.dirname(apitrace_abspath)
107 def _get_build_program(program):
108 bin_path = _get_bin_path()
109 if platform.system() == 'Windows':
111 path = os.path.join(bin_path, program)
112 if not os.path.exists(path):
113 sys.stderr.write('error: %s does not exist\n' % path)
117 def _get_scripts_path():
118 bin_path = _get_bin_path()
123 '../lib/apitrace/scripts',
126 for try_path in try_paths:
127 path = os.path.join(bin_path, try_path)
128 if os.path.exists(path):
129 return os.path.abspath(path)
131 sys.stderr.write('error: could not find scripts directory\n')
137 def __init__(self, srcStream, refFileName, verbose=False):
138 self.srcStream = srcStream
139 self.refFileName = refFileName
141 self.refStream = open(refFileName, 'rt')
143 self.refStream = None
144 self.verbose = verbose
145 self.doubleBuffer = False
151 call_re = re.compile(r'^([0-9]+) (\w+)\(')
159 self.consumeRefLine()
160 for line in self.srcStream:
163 sys.stdout.write(line + '\n')
164 mo = self.call_re.match(line)
166 self.callNo = int(mo.group(1))
167 function_name = mo.group(2)
168 if function_name.find('SwapBuffers') != -1 or \
169 line.find('kCGLPFADoubleBuffer') != -1:
171 if function_name in ('glFlush', 'glFinish'):
173 srcLine = line[mo.start(2):]
177 if srcLine == self.refLine:
178 self.consumeRefLine()
181 srcLines.append(srcLine)
185 fail('missing call `%s` (found `%s`)' % (self.refLine, srcLines[0]))
187 fail('missing call %s' % self.refLine)
190 self.doubleBuffer = True
192 self.doubleBuffer = False
194 def consumeRefLine(self):
195 if not self.refStream:
200 line = self.refStream.readline()
204 if line.startswith('#'):
205 self.handlePragma(line)
210 def handlePragma(self, line):
211 pragma, rest = line.split(None, 1)
212 if pragma == '#image':
213 imageFileName = self.getAbsPath(rest)
214 self.images.append((self.callNo, imageFileName))
215 elif pragma == '#state':
216 stateFileName = self.getAbsPath(rest)
217 self.states.append((self.callNo, stateFileName))
221 def getAbsPath(self, path):
222 '''Get the absolute from a path relative to the reference filename'''
223 return os.path.abspath(os.path.join(os.path.dirname(self.refFileName), path))
242 threshold_precision = 12.0
248 '''Run the application standalone, skipping this test if it fails by
254 p = popen(self.cmd, cwd=self.cwd)
257 skip('application returned code %i' % p.returncode)
270 if self.trace_file is None:
271 if self.ref_dump is not None:
275 name, ext = os.path.splitext(os.path.basename(name))
277 name, ext = os.path.splitext(os.path.basename(name))
278 self.trace_file = os.path.abspath(os.path.join(self.results, name + '.trace'))
279 if os.path.exists(self.trace_file):
280 os.remove(self.trace_file)
282 trace_dir = os.path.dirname(self.trace_file)
283 if not os.path.exists(trace_dir):
284 os.makedirs(trace_dir)
287 env = os.environ.copy()
290 options.apitrace, 'trace',
291 '--api', self.api_map[self.api],
292 '--output', self.trace_file,
295 if self.max_frames is not None:
296 env['TRACE_FRAMES'] = str(self.max_frames)
298 p = popen(cmd, env=env, cwd=self.cwd)
301 if not os.path.exists(self.trace_file):
302 fail('no trace file generated\n')
304 def checkTrace(self):
305 cmd = [options.apitrace, 'dump', '--color=never', self.trace_file]
306 p = popen(cmd, stdout=subprocess.PIPE)
308 checker = TraceChecker(p.stdout, self.ref_dump, self.verbose)
311 if p.returncode != 0:
312 fail('`apitrace dump` returned code %i' % p.returncode)
314 self.doubleBuffer = checker.doubleBuffer
316 for callNo, refImageFileName in checker.images:
317 self.checkImage(callNo, refImageFileName)
318 for callNo, refStateFileName in checker.states:
319 self.checkState(callNo, refStateFileName)
321 def checkImage(self, callNo, refImageFileName):
323 from PIL import Image
327 srcImage = self.getImage(callNo)
328 refImage = Image.open(refImageFileName)
330 from snapdiff import Comparer
331 comparer = Comparer(refImage, srcImage)
332 precision = comparer.precision(filter=True)
333 sys.stdout.write('precision of %f bits against %s\n' % (precision, refImageFileName))
334 if precision < self.threshold_precision:
335 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
336 srcImageFileName = prefix + '.src.png'
337 srcImage.save(srcImageFileName)
338 diffImageFileName = prefix + '.diff.png'
339 comparer.write_diff(diffImageFileName)
340 fail('snapshot from call %u does not match %s' % (callNo, refImageFileName))
342 def checkState(self, callNo, refStateFileName):
343 srcState = self.getState(callNo)
344 refState = self.getRefState(refStateFileName)
346 from jsondiff import Comparer, Differ
347 comparer = Comparer(ignore_added = True)
348 match = comparer.visit(refState, srcState)
350 prefix = '%s.%u' % (self.getNamePrefix(), callNo)
351 srcStateFileName = prefix + '.src.json'
352 diffStateFileName = prefix + '.diff.json'
353 self.saveState(srcState, srcStateFileName)
354 #diffStateFile = open(diffStateFileName, 'wt')
355 diffStateFile = sys.stdout
356 differ = Differ(diffStateFile, ignore_added = True)
357 differ.visit(refState, srcState)
358 fail('state from call %u does not match %s' % (callNo, refStateFileName))
360 def getRefState(self, refStateFileName):
361 stream = open(refStateFileName, 'rt')
362 from jsondiff import load
364 self.adjustRefState(state)
367 def getNamePrefix(self):
368 name = os.path.basename(self.ref_dump)
370 index = name.index('.')
377 def saveState(self, state, filename):
378 s = json.dumps(state, sort_keys=True, indent=2)
379 open(filename, 'wt').write(s)
384 if p.returncode != 0:
385 fail('retrace failed with code %i' % (p.returncode))
387 def getImage(self, callNo):
388 from PIL import Image
389 state = self.getState(callNo)
390 if self.doubleBuffer:
391 attachments = ['GL_BACK', 'GL_BACK_LEFT', 'GL_BACK_RIGHT', 'GL_COLOR_ATTACHMENT0']
393 attachments = ['GL_FRONT', 'GL_FRONT_LEFT', 'GL_FRONT_RIGHT', 'GL_COLOR_ATTACHMENT0']
394 imageObj = self.getFramebufferAttachment(state, attachments)
395 data = imageObj['__data__']
396 stream = StringIO(base64.b64decode(data))
397 im = Image.open(stream)
401 def getFramebufferAttachment(self, state, attachments):
402 framebufferObj = state['framebuffer']
403 for attachment in attachments:
405 attachmentObj = framebufferObj[attachment]
410 raise Exception("no attachment found")
412 def getState(self, callNo):
414 state = self.stateCache[callNo]
420 p = self._retrace(['-D', str(callNo)])
421 state = json.load(p.stdout, strict=False)
423 if p.returncode != 0:
424 fail('retrace returned code %i' % (p.returncode))
426 self.adjustSrcState(state)
428 self.stateCache[callNo] = state
432 def adjustSrcState(self, state):
433 # Do some adjustments on the obtained state to eliminate failures from
434 # bugs/issues outside of apitrace
437 parameters = state['parameters']
441 # On NVIDIA drivers glGetIntegerv(GL_INDEX_WRITEMASK) returns -1
442 self.replaceState(parameters, 'GL_INDEX_WRITEMASK', 255, -1)
445 if 'Gallium' in parameters['GL_RENDERER'].split():
446 # Gallium drivers have wrong defaults for draw/read buffer state
447 self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
448 self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_BACK_LEFT', 'GL_BACK')
449 self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_BACK_LEFT', 'GL_BACK')
450 self.replaceState(parameters, 'GL_DRAW_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
451 self.replaceState(parameters, 'GL_DRAW_BUFFER0', 'GL_FRONT_LEFT', 'GL_FRONT')
452 self.replaceState(parameters, 'GL_READ_BUFFER', 'GL_FRONT_LEFT', 'GL_FRONT')
454 def adjustRefState(self, state):
455 # Do some adjustments on reference state to eliminate failures from
456 # bugs/issues outside of apitrace
459 parameters = state['parameters']
463 if platform.system() == 'Darwin':
464 # Mac OS X drivers fail on GL_COLOR_SUM
465 # XXX: investigate this
466 self.removeState(parameters, 'GL_COLOR_SUM')
468 def replaceState(self, obj, key, srcValue, dstValue):
474 if value == srcValue:
477 def removeState(self, obj, key):
483 def _retrace(self, args = None, stdout=subprocess.PIPE):
484 retrace = self.api_map[self.api] + 'retrace'
485 cmd = [_get_build_program(retrace)]
486 if self.doubleBuffer:
492 cmd += [self.trace_file]
493 return popen(cmd, stdout=stdout)
507 default_apitrace = 'apitrace'
508 if platform.system() == 'Windows':
509 default_apitrace += '.exe'
511 # Parse command line options
512 optparser = optparse.OptionParser(
513 usage='\n\t%prog [options] -- [TRACE|PROGRAM] ...',
515 optparser.add_option(
518 dest="verbose", default=False,
519 help="verbose output")
520 optparser.add_option(
521 '-a', '--api', metavar='API',
522 type='string', dest='api', default='gl',
524 optparser.add_option(
525 '--apitrace', metavar='PROGRAM',
526 type='string', dest='apitrace', default=default_apitrace,
527 help='path to apitrace executable')
528 optparser.add_option(
529 '-C', '--directory', metavar='PATH',
530 type='string', dest='cwd', default=None,
531 help='change to directory')
532 optparser.add_option(
533 '-R', '--results', metavar='PATH',
534 type='string', dest='results', default='.',
535 help='results directory [default=%default]')
536 optparser.add_option(
537 '--ref-dump', metavar='PATH',
538 type='string', dest='ref_dump', default=None,
539 help='reference dump')
541 (options, args) = optparser.parse_args(sys.argv[1:])
543 optparser.error('an argument must be specified')
545 if not os.path.exists(options.results):
546 os.makedirs(options.results)
548 print _get_scripts_path()
550 sys.path.insert(0, _get_scripts_path())
553 test.verbose = options.verbose
555 if args[0].endswith('.trace'):
556 test.trace_file = args[0]
559 test.cwd = options.cwd
560 test.api = options.api
561 test.ref_dump = options.ref_dump
562 test.results = options.results
567 if __name__ == '__main__':