Add a test that "apitrace diff-images" flags mismatching images.

With this test we add a new EXPECT_FAILURE: attribute to the script
syntax to indicate that a particular apitrace command in the script is
expected to fail. This requires manually invoking the Popen
constructor rather than using subprocess.check_output, (which is
*close* to what we need but fails to assign the output when raising an
exception).

We also remove the globbing from the CMakeLists.txt file to get a
manual ordering of the tests (rather than executing in alphabetic
order).

diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 283f56fc055838bf53a94f569b4e5f211b6a5711..39a15c86ef55d454086f8242b5341c97e1143b5e 100644 (file)
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -1,15 +1,26 @@
-file (GLOB scripts RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.script)
+function (ADD_CLI_TEST)
+    cmake_parse_arguments(
+        TEST
+        # Options
+        ""
+        # One value args
+        "NAME"
+        # Multi value args
+        ""
+        ${ARGN}
+    )
 
-list (SORT scripts)
-
-foreach (script ${scripts})
     if (APITRACE_EXECUTABLE)
         add_test(
-            NAME ${script}
-            COMMAND python ${CMAKE_SOURCE_DIR}/cli_driver.py
-               --apitrace ${APITRACE_EXECUTABLE}
-               --apitrace-source ${APITRACE_SOURCE_DIR}
-               ${CMAKE_CURRENT_SOURCE_DIR}/${script}
+            NAME ${TEST_NAME}
+            COMMAND
+            python ${CMAKE_SOURCE_DIR}/cli_driver.py
+                --apitrace ${APITRACE_EXECUTABLE}
+                --apitrace-source ${APITRACE_SOURCE_DIR}
+                ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_NAME}
         )
     endif ()
-endforeach (script)
+endfunction ()
+
+add_cli_test(NAME "cli-diff-images.script")
+add_cli_test(NAME "cli-diff-images-mismatch.script")

diff --git a/cli/README.markdown b/cli/README.markdown
index 3d49c7c9a355758746de5bb0966eb55a262447f8..6be80a9790ef8aeaffc7354b54370b7bd7830272 100644 (file)
--- a/cli/README.markdown
+++ b/cli/README.markdown
@@ -10,8 +10,10 @@ test in the traces directory. Otherwise, a new test program can be
 written in this directory.
 
 The tests in this directory are found in files with names matching
-*.script. The scripts are simple line-based commands with the
-following meanings based on the first word of each line:
+*.script by convention. The scripts must be listed explicitly in the
+CMakeLists.txt file. Each script consists of simple line-based
+commands with the following meanings (based on the first word of each
+line):
 
   apitrace:     Execute the current apitrace executable being tested
                with the given arguments. If apitrace returns a
@@ -26,6 +28,10 @@ following meanings based on the first word of each line:
                 interpreted locally. If this fails for any reason
                 other than "file does not exist" the test will fail.
 
-If none of the commands in the script cause the test to fail, then the
-test will pass.
+Commands can be prefixed with "EXPECT_FAILURE:" to indicate that a
+command is expected to return a non-zero value. In this case, a return
+value of zero from the command will cause the test to fail.
+
+If none of the commands in the script cause the test to fail (as
+described above), then the test will pass.
 

diff --git a/cli/cli-diff-images-mismatch.script b/cli/cli-diff-images-mismatch.script
new file mode 100644 (file)
index 0000000..9c97fa3
--- /dev/null
+++ b/cli/cli-diff-images-mismatch.script
@@ -0,0 +1,4 @@
+rm_and_mkdir ./tri-out
+apitrace dump-images -o ./tri-out/tri tri.trace
+EXPECT_FAILURE: apitrace diff-images -v ./tri-ref-mismatch ./tri-out
+expect "Comparing ./tri-ref-mismatch/tri0000000027.png and ./tri-out/tri0000000027.png ... MISMATCH\n"

diff --git a/cli/tri-ref-mismatch/tri0000000027.png b/cli/tri-ref-mismatch/tri0000000027.png
new file mode 100644 (file)
index 0000000..e682610
Binary files /dev/null and b/cli/tri-ref-mismatch/tri0000000027.png differ

diff --git a/cli_driver.py b/cli_driver.py
index 6906b738e11c271629951b715a546d169ef23179..a7f4b3d5212a67821e636715140c8bc5e5d0c5f9 100644 (file)
--- a/cli_driver.py
+++ b/cli_driver.py
@@ -33,7 +33,17 @@ class CliDriver(Driver):
     def do_apitrace(self, args):
         cmd = [self.options.apitrace] + args[1:]
  
-        self.output = subprocess.check_output(cmd)
+        proc = subprocess.Popen(cmd, stdout = subprocess.PIPE)
+        self.output = proc.communicate()[0]
+
+        proc.wait()
+
+        if (self.expect_failure):
+            if (proc.returncode == 0):
+                fail("Command unexpectedly passed when expecting failure:\n    " + " ".join(cmd))
+        else:
+            if (proc.returncode != 0):
+                fail("Command failed (returned non-zero):\n    " + " ".join(cmd))
 
     def do_expect(self, args):
         expected = json.loads(" ".join(args[1:]))
@@ -68,6 +78,9 @@ class CliDriver(Driver):
         script = open(cli_script, 'rt')
 
         while True:
+
+            self.expect_failure = False
+
             line = script.readline()
 
             if (line == ''):
@@ -78,6 +91,10 @@ class CliDriver(Driver):
             if (len(cmd) == 0):
                 continue
 
+            if (cmd[0] == 'EXPECT_FAILURE:'):
+                self.expect_failure = True
+                cmd.pop(0)
+
             commands.get(cmd[0], self.unknown_command)(cmd)
 
     def run(self):