From: José Fonseca <jose.r.fonseca@gmail.com>
Date: Wed, 8 Jun 2011 21:49:40 +0000 (+0100)
Subject: Split testing into common/different parts.
X-Git-Url: https://git.cworth.org/git?p=apitrace-tests;a=commitdiff_plain;h=cde1bf34498495236aee6e8541d39cac3aa54056

Split testing into common/different parts.
---

diff --git a/mesademos.py b/mesademos.py
index 318c6a2..21b0af4 100755
--- a/mesademos.py
+++ b/mesademos.py
@@ -32,180 +32,25 @@
 import os.path
 import optparse
 import sys
-import subprocess
-import time
-import re
-import signal
 
+from test import Report, TestCase
 
-ansi_re = re.compile('\x1b\[[0-9]{1,2}(;[0-9]{1,2}){0,2}m')
-
-
-def ansi_strip(s):
-    # http://www.theeggeadventure.com/wikimedia/index.php/Linux_Tips#Use_sed_to_remove_ANSI_colors
-    return ansi_re.sub('', s)
-
-
-def popen(command, *args, **kwargs):
-    if 'cwd' in kwargs:
-        sys.stdout.write('cd %s && ' % kwargs['cwd'])
-    if 'env' in kwargs:
-        for name, value in kwargs['env'].iteritems():
-            if value != os.environ.get(name, None):
-                sys.stdout.write('%s=%s ' % (name, value))
-    sys.stdout.write(' '.join(command) + '\n')
-    sys.stdout.flush()
-    return subprocess.Popen(command, *args, **kwargs)
-
-
-ignored_function_names = set([
-    'glGetString',
-    'glXGetCurrentDisplay',
-    'glXGetClientString',
-    'glXGetProcAddress',
-    'glXGetProcAddressARB',
-    'glXQueryVersion',
-    'glXGetVisualFromFBConfig',
-    'glXChooseFBConfig',
-    'glXCreateNewContext',
-    'glXMakeContextCurrent',
-    'glXQueryExtension',
-    'glXIsDirect',
-])
-
-
-def image_tag(html, image):
-    url = os.path.relpath(image, options.results)
-    html.write('        <td><a href="%s"><img src="%s"/></a></td>\n' % (url, url))
-
-
-def runtest(html, demo):
 
+def runtest(report, demo):
     app = os.path.join(options.mesa_demos, 'src', demo)
-
     dirname, basename = os.path.split(app)
+    name = demo.replace('/', '-')
+    args = [os.path.join('.', basename)]
 
-    trace = os.path.abspath(os.path.join(options.results, demo.replace('/', '-') + '.trace'))
-
-    env = os.environ.copy()
-    env['LD_PRELOAD'] = os.path.abspath(os.path.join(options.build, 'glxtrace.so'))
-    env['TRACE_FILE'] = trace
+    test = TestCase(
+        name = name,
+        args = args,
+        cwd = dirname,
+        build = options.build,
+        results = options.results,
+    )
+    test.run(report)
 
-    args = [os.path.join('.', basename)]
-    window_name = args[0]
-
-    p = popen(args, cwd=dirname)
-    for i in range(6):
-        time.sleep(0.5)
-        if p.poll() is not None:
-            break
-        if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
-            break
-    if p.returncode is None:
-        p.terminate()
-    elif p.returncode:
-        sys.stdout.write('SKIP (app)\n')
-        return
-
-    ref_image = os.path.join(options.results, demo.replace('/', '-') + '.ref.png')
-    p = popen(args, env=env, cwd=dirname)
-    try:
-        for i in range(10):
-            time.sleep(0.5)
-            if p.poll() is not None:
-                break
-            if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
-                break
-
-        if p.returncode is None:
-            subprocess.call("xwd -name '%s' | xwdtopnm | pnmtopng > '%s'" % (args[0], ref_image), shell=True)
-    finally:
-        if p.returncode is None:
-            p.terminate()
-            p.wait()
-        elif p.returncode:
-            print p.returncode
-            sys.stdout.write('FAIL (trace)\n')
-            return
-
-    p = popen([os.path.join(options.build, 'tracedump'), trace], stdout=subprocess.PIPE)
-    call_re = re.compile('^([0-9]+) (\w+)\(')
-    swapbuffers = 0
-    flushes = 0
-    for orig_line in p.stdout:
-        orig_line = orig_line.rstrip()
-        line = ansi_strip(orig_line)
-        mo = call_re.match(line)
-        if mo:
-            call_no = int(mo.group(1))
-            function_name = mo.group(2)
-            if function_name in ignored_function_names:
-                continue
-            if function_name == 'glXSwapBuffers':
-                swapbuffers += 1
-            if function_name in ('glFlush', 'glFinish'):
-                flushes += 1
-        #print orig_line
-    p.wait()
-    if p.returncode != 0:
-        sys.stdout.write('FAIL (tracedump)\n')
-        return
-
-    args = [os.path.join(options.build, 'glretrace')]
-    if swapbuffers:
-        args += ['-db']
-        frames = swapbuffers
-    else:
-        args += ['-sb']
-        frames = flushes
-    if os.path.exists(ref_image) and frames < 10:
-        snapshot_prefix = os.path.join(options.results, demo.replace('/', '-') + '.')
-        args += ['-s', snapshot_prefix]
-    else:
-        snapshot_prefix = None
-    args += [trace]
-    p = popen(args, stdout=subprocess.PIPE)
-    image_re = re.compile('^Wrote (.*\.png)$')
-    image = None
-    for line in p.stdout:
-        line = line.rstrip()
-        mo = image_re.match(line)
-        if mo:
-            image = mo.group(1)
-    p.wait()
-    if p.returncode != 0:
-        sys.stdout.write('FAIL (glretrace)\n')
-        return
-
-    if image:
-        delta_image = os.path.join(options.results, demo.replace('/', '-') + '.diff.png')
-        p = popen([
-            'compare',
-            '-alpha', 'opaque',
-            '-metric', 'AE',
-            '-fuzz', '5%',
-            '-dissimilarity-threshold', '1',
-            ref_image, image, delta_image
-        ], stderr = subprocess.PIPE)
-        _, stderr = p.communicate()
-
-        try:
-            ae = int(stderr)
-        except ValueError:
-            ae = 9999
-
-        if ae:
-            html.write('      <tr>\n')
-            image_tag(html, ref_image)
-            image_tag(html, image)
-            image_tag(html, delta_image)
-            html.write('      </tr>\n')
-            html.flush()
-
-            sys.stdout.write('FAIL (snapshot)\n')
-            return
-
-    sys.stdout.write('PASS\n')
 
 
 tests = [
@@ -387,6 +232,7 @@ tests = [
     'demos/clearspd',
     'demos/copypix',
     'demos/cubemap',
+    'demos/cuberender',
     'demos/dinoshade',
     'demos/dissolve',
     'demos/drawpix',
@@ -782,8 +628,6 @@ def main():
 
     if not options.mesa_demos:
         optparser.error('path to Mesa demos not specified')
-    if not os.path.exists(options.results):
-        os.makedirs(options.results)
 
     if args:
         testlist = []
@@ -797,16 +641,9 @@ def main():
     else:
         testlist = tests
 
-    html = open(os.path.join(options.results, 'index.html'), 'wt')
-    html.write('<html>\n')
-    html.write('  <body>\n')
-    html.write('    <table border="1">\n')
-    html.write('      <tr><th>Ref</th><th>Src</th><th>&Delta;</th></tr>\n')
+    report = Report(options.results)
     for test in testlist:
-       runtest(html, test)
-    html.write('    </table>\n')
-    html.write('  </body>\n')
-    html.write('</html>\n')
+       runtest(report, test)
 
 
 if __name__ == '__main__':
diff --git a/test.py b/test.py
new file mode 100755
index 0000000..3c96c12
--- /dev/null
+++ b/test.py
@@ -0,0 +1,282 @@
+#!/usr/bin/env python
+##########################################################################
+#
+# Copyright 2011 Jose Fonseca
+# All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+##########################################################################/
+
+'''Common test suite code.'''
+
+
+import os.path
+import optparse
+import sys
+import subprocess
+import time
+import re
+import signal
+
+
+ansi_re = re.compile('\x1b\[[0-9]{1,2}(;[0-9]{1,2}){0,2}m')
+
+
+def ansi_strip(s):
+    # http://www.theeggeadventure.com/wikimedia/index.php/Linux_Tips#Use_sed_to_remove_ANSI_colors
+    return ansi_re.sub('', s)
+
+
+def popen(command, *args, **kwargs):
+    if 'cwd' in kwargs:
+        sys.stdout.write('cd %s && ' % kwargs['cwd'])
+    if 'env' in kwargs:
+        for name, value in kwargs['env'].iteritems():
+            if value != os.environ.get(name, None):
+                sys.stdout.write('%s=%s ' % (name, value))
+    sys.stdout.write(' '.join(command) + '\n')
+    sys.stdout.flush()
+    return subprocess.Popen(command, *args, **kwargs)
+
+
+ignored_function_names = set([
+    'glGetString',
+    'glXGetCurrentDisplay',
+    'glXGetClientString',
+    'glXGetProcAddress',
+    'glXGetProcAddressARB',
+    'glXQueryVersion',
+    'glXGetVisualFromFBConfig',
+    'glXChooseFBConfig',
+    'glXCreateNewContext',
+    'glXMakeContextCurrent',
+    'glXQueryExtension',
+    'glXIsDirect',
+])
+
+
+class Report:
+
+    def __init__(self, basedir):
+        self.basedir = basedir
+        if not os.path.exists(basedir):
+            os.makedirs(basedir)
+        self.html = open(os.path.join(basedir, 'index.html'), 'wt')
+        self._header()
+
+    def _header(self):
+        self.html.write('<html>\n')
+        self.html.write('  <body>\n')
+        self.html.write('    <table border="1">\n')
+        self.html.write('      <tr><th>Ref</th><th>Src</th><th>&Delta;</th></tr>\n')
+
+    def _image_tag(self, image):
+        url = os.path.relpath(image, self.basedir)
+        self.html.write('        <td><a href="%s"><img src="%s"/></a></td>\n' % (url, url))
+
+    def add_snapshot(self, ref_image, src_image, diff_image):
+        self.html.write('      <tr>\n')
+        self._image_tag(ref_image)
+        self._image_tag(src_image)
+        self._image_tag(diff_image)
+        self.html.write('      </tr>\n')
+        self.html.flush()
+
+    def _footer(self):
+        self.html.write('    </table>\n')
+        self.html.write('  </body>\n')
+        self.html.write('</html>\n')
+
+    def __del__(self):
+        self._footer()
+        self.html.close()
+
+
+class TestCase:
+
+    def __init__(self, name, args, cwd=None, build = '.', results = '.'):
+        self.name = name
+        self.args = args
+        self.cwd = cwd
+        self.build = build
+        self.results = results
+
+        if not os.path.exists(results):
+            os.makedirs(results)
+
+    def run(self, report):
+
+        trace = os.path.abspath(os.path.join(self.results, self.name + '.trace'))
+
+        ld_preload = os.path.abspath(os.path.join(self.build, 'glxtrace.so'))
+        if not os.path.exists(ld_preload):
+            sys.stderr.write('error: could not find %s\n' % ld_preload)
+            sys.exit(1)
+
+        env = os.environ.copy()
+        env['LD_PRELOAD'] = ld_preload
+        env['TRACE_FILE'] = trace
+
+        window_name = self.args[0]
+
+        p = popen(self.args, cwd=self.cwd)
+        for i in range(3):
+            time.sleep(1)
+            if p.poll() is not None:
+                break
+            if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
+                break
+        if p.returncode is None:
+            p.terminate()
+        elif p.returncode:
+            sys.stdout.write('SKIP (app)\n')
+            return
+
+        ref_image = os.path.join(self.results, self.name + '.ref.png')
+        p = popen(self.args, env=env, cwd=self.cwd)
+        try:
+            for i in range(5):
+                time.sleep(1)
+                if p.poll() is not None:
+                    break
+                if subprocess.call(['xwininfo', '-name', window_name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0:
+                    break
+
+            if p.returncode is None:
+                subprocess.call("xwd -name '%s' | xwdtopnm | pnmtopng > '%s'" % (window_name, ref_image), shell=True)
+        finally:
+            if p.returncode is None:
+                p.terminate()
+                p.wait()
+            elif p.returncode:
+                print p.returncode
+                sys.stdout.write('FAIL (trace)\n')
+                return
+
+        p = popen([os.path.join(self.build, 'tracedump'), trace], stdout=subprocess.PIPE)
+        call_re = re.compile('^([0-9]+) (\w+)\(')
+        swapbuffers = 0
+        flushes = 0
+        for orig_line in p.stdout:
+            orig_line = orig_line.rstrip()
+            line = ansi_strip(orig_line)
+            mo = call_re.match(line)
+            if mo:
+                call_no = int(mo.group(1))
+                function_name = mo.group(2)
+                if function_name in ignored_function_names:
+                    continue
+                if function_name == 'glXSwapBuffers':
+                    swapbuffers += 1
+                if function_name in ('glFlush', 'glFinish'):
+                    flushes += 1
+            #print orig_line
+        p.wait()
+        if p.returncode != 0:
+            sys.stdout.write('FAIL (tracedump)\n')
+            return
+
+        args = [os.path.join(self.build, 'glretrace')]
+        if swapbuffers:
+            args += ['-db']
+            frames = swapbuffers
+        else:
+            args += ['-sb']
+            frames = flushes
+        if os.path.exists(ref_image) and frames < 10:
+            snapshot_prefix = os.path.join(self.results, self.name + '.')
+            args += ['-s', snapshot_prefix]
+        else:
+            snapshot_prefix = None
+        args += [trace]
+        p = popen(args, stdout=subprocess.PIPE)
+        image_re = re.compile('^Wrote (.*\.png)$')
+        image = None
+        for line in p.stdout:
+            line = line.rstrip()
+            mo = image_re.match(line)
+            if mo:
+                image = mo.group(1)
+        p.wait()
+        if p.returncode != 0:
+            sys.stdout.write('FAIL (glretrace)\n')
+            return
+
+        if image:
+            delta_image = os.path.join(self.results, self.name + '.diff.png')
+            p = popen([
+                'compare',
+                '-alpha', 'opaque',
+                '-metric', 'AE',
+                '-fuzz', '5%',
+                '-dissimilarity-threshold', '1',
+                ref_image, image, delta_image
+            ], stderr = subprocess.PIPE)
+            _, stderr = p.communicate()
+
+            try:
+                ae = int(stderr)
+            except ValueError:
+                ae = 9999
+
+            if ae:
+                report.add_snapshot(ref_image, image, delta_image)
+                sys.stdout.write('FAIL (snapshot)\n')
+                return
+
+        sys.stdout.write('PASS\n')
+
+
+def main():
+    # Parse command line options
+    optparser = optparse.OptionParser(
+        usage='\n\t%prog [options] -- program [args] ...',
+        version='%%prog')
+    optparser.add_option(
+        '--build', metavar='PATH',
+        type='string', dest='build', default='.',
+        help='path to apitrace build [default=%default]')
+    optparser.add_option(
+        '--cwd', metavar='PATH',
+        type='string', dest='cwd', default='.',
+        help='change directory [default=%default]')
+    optparser.add_option(
+        '--results', metavar='PATH',
+        type='string', dest='results', default='results',
+        help='results directory [default=%default]')
+
+    (options, args) = optparser.parse_args(sys.argv[1:])
+    if not args:
+        optparser.error('program must be specified')
+
+    test = TestCase(
+        name = os.path.basename(args[0]), 
+        args = args,
+        cwd = options.cwd,
+        build = options.build,
+        results = options.results,
+    )
+
+    report = Report(options.results)
+    test.run(report)
+
+
+if __name__ == '__main__':
+    main()