Add stress tests from "apitrace trim".

The trim_stress directory makes it simple to stress test "apitrace
trim" by simply dropping new trace files into this directory. The
stress testing ensures that creating a new trace by trimming to a
single frame still generates the same image as that frame from the
original trace. It performs this test exhaustively for each frame of
each trace.

This testing ensures that trimming doesn't break the rendered results
of any frame. It does not do anything to test that trimming actually
discards anything. So the existing cli tests are still
independendently useful for verifying that expected content is
discarded.

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 10961587b04c6fd91f4f3bc7348923e6a0be0a33..7c55f765740d27d08ab1ca961a23338755977aff 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -103,4 +103,5 @@ add_subdirectory (traces)
 # condition here.
 if (OPENGL_FOUND)
     add_subdirectory (cli)
+    add_subdirectory (trim_stress)
 endif ()

diff --git a/trim_stress/.gitignore b/trim_stress/.gitignore
new file mode 100644 (file)
index 0000000..2c3a0d6
--- /dev/null
+++ b/trim_stress/.gitignore
@@ -0,0 +1,5 @@
+!*.trace
+*-ref/
+*-out
+*-index.html
+*.trace.trim

diff --git a/trim_stress/CMakeLists.txt b/trim_stress/CMakeLists.txt
new file mode 100644 (file)
index 0000000..652590d
--- /dev/null
+++ b/trim_stress/CMakeLists.txt
@@ -0,0 +1,16 @@
+file (GLOB traces RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.trace)
+
+list (SORT traces)
+
+foreach (trace ${traces})
+    if (APITRACE_EXECUTABLE AND APITRACE_SOURCE_DIR)
+        add_test(
+            NAME trim-stress-${trace}
+            COMMAND
+            python ${CMAKE_SOURCE_DIR}/trim_stress_driver.py
+                --apitrace ${APITRACE_EXECUTABLE}
+                --apitrace-source ${APITRACE_SOURCE_DIR}
+                ${CMAKE_CURRENT_SOURCE_DIR}/${trace}
+        )
+    endif ()
+endforeach (trace)

diff --git a/trim_stress/README.markdown b/trim_stress/README.markdown
new file mode 100644 (file)
index 0000000..adab8dc
--- /dev/null
+++ b/trim_stress/README.markdown
@@ -0,0 +1,19 @@
+This directory performs stress testing of "apitrace trim".
+
+For each trace file in this directory the trim_stress.py test driver
+will perform the following operations:
+
+Given <program>.trace:
+
+    1. Generate snapshots of original trace in ./<program>-ref/
+
+    2. For each frame of the trace:
+
+        1. Use "apitrace trim" to trim <program>.trace to a
+           single-frame trace in <program>-trim.trace
+
+       2. Generate a snapshot of the trimmed trace to
+          ./<program>-out/<frame>.png
+
+    3. Use "apitrace diff-images" to compare all snapshots in
+       ./<program>-out with those in ./<program>-ref

diff --git a/trim_stress/glxsimple.trace b/trim_stress/glxsimple.trace
new file mode 100644 (file)
index 0000000..9c22592
Binary files /dev/null and b/trim_stress/glxsimple.trace differ

diff --git a/trim_stress_driver.py b/trim_stress_driver.py
new file mode 100644 (file)
index 0000000..5b10f14
--- /dev/null
+++ b/trim_stress_driver.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+'''Stress test driver for apitrace trimp.'''
+
+import os, errno, shutil, subprocess
+
+from base_driver import *
+
+def rm_and_mkdir(dir):
+
+    # Operate only on local directories
+    dir = './' + dir
+
+    # Failing to delete a directory that doesn't exist is no failure
+    def rmtree_onerror(function, path, excinfo):
+        if excinfo[0] == OSError and excinfo[1].errno != errno.ENOENT:
+            raise
+
+    shutil.rmtree(dir, onerror = rmtree_onerror)
+    os.makedirs(dir)
+
+class TrimStressDriver(Driver):
+
+    def trim_stress(self, trace_file):
+        "Execute an apitrace-trim stress test for the given trace."
+
+        (dir, file_name) = os.path.split(trace_file)
+        (name, extension) = file_name.rsplit('.', 1)
+
+        ref_dir = name + '-ref/'
+        out_dir = name + '-out/'
+        trim_file = name + '.trace.trim'
+
+        rm_and_mkdir(ref_dir)
+        rm_and_mkdir(out_dir)
+
+        subprocess.check_call([self.options.apitrace, "dump-images", "--call-nos=no", "--output=" + ref_dir, trace_file]);
+
+        # Count the number of frame snapshots generated
+        frames = 0
+        for dirpath, dirs, files in os.walk(ref_dir):
+            # Don't descend into any sub-directories (not that there
+            # should be any)
+            del dirs[:]
+            frames = len(files)
+
+        for frame in range(frames):
+            try:
+                subprocess.check_call([self.options.apitrace, "trim", "--frame=%d" % (frame), "--output=" + trim_file, trace_file])
+            except:
+                print "An error occurred while trimming frame %d from %s" % (frame, trace_file)
+                fail()
+            try:
+                subprocess.check_call([self.options.apitrace, "dump-images", "--call-nos=no", "--output=" + out_dir + "frame", trim_file])
+            except:
+                print "An error occurred replaying %s to generate a frame snapshot" % (trim_file)
+                fail()
+            os.rename("%s/frame0000000000.png" % (out_dir), "%s/%010d.png" % (out_dir, frame))
+
+        try:
+            subprocess.check_call([self.options.apitrace, "diff-images", "-v", ref_dir, out_dir])
+        except:
+            print "Trimmed frames did not match reference images. See " + name + "-index.html"
+            fail()
+        finally:
+            os.rename("index.html", name + "-index.html")
+
+    def run(self):
+        self.parseOptions()
+
+       self.trim_stress(self.args[0])
+
+        pass_()
+
+if __name__ == '__main__':
+    TrimStressDriver().run()