trim_stress: Fail test at first frame which doesn't match reference

When stress-testing large traces, it's inefficient to work through the
entire trace before reporting an error that occurred in one of the
first frames.

Instead, check each frame one at a time immediately as each rendered
frame becomes available. Then fail the test at the first mismatched
frame.

diff --git a/trim_stress/README.markdown b/trim_stress/README.markdown
index adab8dcb1b74117d924ba2e7d004633a8a6f2385..5678ceb807d570ed5cf5e2c2b09e02790e4116e1 100644 (file)
--- a/trim_stress/README.markdown
+++ b/trim_stress/README.markdown
@@ -5,15 +5,22 @@ will perform the following operations:
 
 Given <program>.trace:
 
-    1. Generate snapshots of original trace in ./<program>-ref/
+    1. Generate snapshots of original trace in ./<program>-ref/, (this
+       step is skipped if the <program>-ref directory already exists.
+       Simply delete this directory before "make test" to force new
+       generation of reference images.
 
     2. For each frame of the trace:
 
         1. Use "apitrace trim" to trim <program>.trace to a
            single-frame trace in <program>-trim.trace
 
-       2. Generate a snapshot of the trimmed trace to
-          ./<program>-out/<frame>.png
+       2. Generate a snapshot of the trimmed frame to
+          ./<program>-tmp-out/<frame>.png
 
-    3. Use "apitrace diff-images" to compare all snapshots in
-       ./<program>-out with those in ./<program>-ref
+        3. Use "apitrace diff-images" to compare this one frame with
+           the corresponding reference image in ./<program>-ref.  If
+           the images differ the entire process is interrupted here.
+
+        4. Move the frame snapshot from ./<program>-tmp-out to
+           ./<program>-out

diff --git a/trim_stress_driver.py b/trim_stress_driver.py
index 64863f82f0ba2b24b252d7f3ee1daf190cdb4cba..0feead391d888ee2bd08dff62594f622982e8812 100644 (file)
--- a/trim_stress_driver.py
+++ b/trim_stress_driver.py
@@ -50,13 +50,18 @@ class TrimStressDriver(Driver):
         (name, extension) = file_name.rsplit('.', 1)
 
         ref_dir = name + '-ref/'
+        tmp_out_dir = name + '-tmp-out/'
         out_dir = name + '-out/'
         trim_file = name + '.trace.trim'
 
-        rm_and_mkdir(ref_dir)
         rm_and_mkdir(out_dir)
+        rm_and_mkdir(tmp_out_dir)
 
-        subprocess.check_call([self.options.apitrace, "dump-images", "--call-nos=no", "--output=" + ref_dir, trace_file]);
+        if (not os.path.exists(ref_dir)):
+            rm_and_mkdir(ref_dir)
+            subprocess.check_call([self.options.apitrace,
+                                   "dump-images", "--call-nos=no",
+                                   "--output=" + ref_dir, trace_file]);
 
         # Count the number of frame snapshots generated
         frames = 0
@@ -68,21 +73,43 @@ class TrimStressDriver(Driver):
 
         for frame in range(frames):
             try:
-                subprocess.check_call([self.options.apitrace, "trim", "--auto", "--frame=%d" % (frame), "--output=" + trim_file, trace_file])
+                subprocess.check_call([self.options.apitrace,
+                                       "trim", "--auto",
+                                       "--frame=%d/frame" % (frame),
+                                       "--output=" + trim_file, trace_file])
             except:
-                print "An error occurred while trimming frame %d from %s" % (frame, trace_file)
+                print "An error occurred while trimming " + \
+                    "frame %d from %s" % (frame, trace_file)
                 fail()
             try:
-                subprocess.check_call([self.options.apitrace, "dump-images", "--call-nos=no", "--output=" + out_dir + "frame", trim_file])
+                subprocess.check_call([self.options.apitrace,
+                                       "dump-images", "--call-nos=no",
+                                       "--output=" + tmp_out_dir + "frame",
+                                       trim_file])
             except:
-                print "An error occurred replaying %s to generate a frame snapshot" % (trim_file)
+                print "An error occurred replaying " + \
+                    "%s to generate a frame snapshot" % (trim_file)
                 fail()
-            os.rename("%s/frame0000000000.png" % (out_dir), "%s/%010d.png" % (out_dir, frame))
-
+            os.rename("%s/frame0000000000.png" % (tmp_out_dir),
+                      "%s/%010d.png" % (tmp_out_dir, frame))
+            try:
+                subprocess.check_call([self.options.apitrace,
+                                       "diff-images", "-v",
+                                       ref_dir, tmp_out_dir])
+            except:
+                print "Trimmed frame did not match " + \
+                    "reference images. See " + name + "-index.html"
+                fail()
+            finally:
+                os.rename("index.html", name + "-index.html")
+            os.rename("%s/%010d.png" % (tmp_out_dir, frame),
+                      "%s/%010d.png" % (out_dir, frame))
         try:
-            subprocess.check_call([self.options.apitrace, "diff-images", "-v", ref_dir, out_dir])
+            subprocess.check_call([self.options.apitrace,
+                                   "diff-images", "-v", ref_dir, out_dir])
         except:
-            print "Trimmed frames did not match reference images. See " + name + "-index.html"
+            print "Trimmed frame did not match " + \
+                "reference images. See " + name + "-index.html"
             fail()
         finally:
             os.rename("index.html", name + "-index.html")