From: Carl Worth <cworth@cworth.org>
Date: Wed, 8 Aug 2012 23:18:21 +0000 (-0700)
Subject: Add a test that "apitrace diff-images" flags mismatching images.
X-Git-Url: https://git.cworth.org/git?p=apitrace-tests;a=commitdiff_plain;h=21e5859e0292d7adc067f98a87be1bb0fe5d11d4

Add a test that "apitrace diff-images" flags mismatching images.

With this test we add a new EXPECT_FAILURE: attribute to the script
syntax to indicate that a particular apitrace command in the script is
expected to fail. This requires manually invoking the Popen
constructor rather than using subprocess.check_output, (which is
*close* to what we need but fails to assign the output when raising an
exception).

We also remove the globbing from the CMakeLists.txt file to get a
manual ordering of the tests (rather than executing in alphabetic
order).
---

diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 283f56f..39a15c8 100644
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -1,15 +1,26 @@
-file (GLOB scripts RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.script)
+function (ADD_CLI_TEST)
+    cmake_parse_arguments(
+        TEST
+        # Options
+        ""
+        # One value args
+        "NAME"
+        # Multi value args
+        ""
+        ${ARGN}
+    )
 
-list (SORT scripts)
-
-foreach (script ${scripts})
     if (APITRACE_EXECUTABLE)
         add_test(
-            NAME ${script}
-            COMMAND python ${CMAKE_SOURCE_DIR}/cli_driver.py
-	        --apitrace ${APITRACE_EXECUTABLE}
-		--apitrace-source ${APITRACE_SOURCE_DIR}
-		${CMAKE_CURRENT_SOURCE_DIR}/${script}
+            NAME ${TEST_NAME}
+            COMMAND
+            python ${CMAKE_SOURCE_DIR}/cli_driver.py
+                --apitrace ${APITRACE_EXECUTABLE}
+                --apitrace-source ${APITRACE_SOURCE_DIR}
+                ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_NAME}
         )
     endif ()
-endforeach (script)
+endfunction ()
+
+add_cli_test(NAME "cli-diff-images.script")
+add_cli_test(NAME "cli-diff-images-mismatch.script")
diff --git a/cli/README.markdown b/cli/README.markdown
index 3d49c7c..6be80a9 100644
--- a/cli/README.markdown
+++ b/cli/README.markdown
@@ -10,8 +10,10 @@ test in the traces directory. Otherwise, a new test program can be
 written in this directory.
 
 The tests in this directory are found in files with names matching
-*.script. The scripts are simple line-based commands with the
-following meanings based on the first word of each line:
+*.script by convention. The scripts must be listed explicitly in the
+CMakeLists.txt file. Each script consists of simple line-based
+commands with the following meanings (based on the first word of each
+line):
 
   apitrace:     Execute the current apitrace executable being tested
   		with the given arguments. If apitrace returns a
@@ -26,6 +28,10 @@ following meanings based on the first word of each line:
                 interpreted locally. If this fails for any reason
                 other than "file does not exist" the test will fail.
 
-If none of the commands in the script cause the test to fail, then the
-test will pass.
+Commands can be prefixed with "EXPECT_FAILURE:" to indicate that a
+command is expected to return a non-zero value. In this case, a return
+value of zero from the command will cause the test to fail.
+
+If none of the commands in the script cause the test to fail (as
+described above), then the test will pass.
 
diff --git a/cli/cli-diff-images-mismatch.script b/cli/cli-diff-images-mismatch.script
new file mode 100644
index 0000000..9c97fa3
--- /dev/null
+++ b/cli/cli-diff-images-mismatch.script
@@ -0,0 +1,4 @@
+rm_and_mkdir ./tri-out
+apitrace dump-images -o ./tri-out/tri tri.trace
+EXPECT_FAILURE: apitrace diff-images -v ./tri-ref-mismatch ./tri-out
+expect "Comparing ./tri-ref-mismatch/tri0000000027.png and ./tri-out/tri0000000027.png ... MISMATCH\n"
diff --git a/cli/tri-ref-mismatch/tri0000000027.png b/cli/tri-ref-mismatch/tri0000000027.png
new file mode 100644
index 0000000..e682610
Binary files /dev/null and b/cli/tri-ref-mismatch/tri0000000027.png differ
diff --git a/cli_driver.py b/cli_driver.py
index 6906b73..a7f4b3d 100644
--- a/cli_driver.py
+++ b/cli_driver.py
@@ -33,7 +33,17 @@ class CliDriver(Driver):
     def do_apitrace(self, args):
         cmd = [self.options.apitrace] + args[1:]
  
-        self.output = subprocess.check_output(cmd)
+        proc = subprocess.Popen(cmd, stdout = subprocess.PIPE)
+        self.output = proc.communicate()[0]
+
+        proc.wait()
+
+        if (self.expect_failure):
+            if (proc.returncode == 0):
+                fail("Command unexpectedly passed when expecting failure:\n    " + " ".join(cmd))
+        else:
+            if (proc.returncode != 0):
+                fail("Command failed (returned non-zero):\n    " + " ".join(cmd))
 
     def do_expect(self, args):
         expected = json.loads(" ".join(args[1:]))
@@ -68,6 +78,9 @@ class CliDriver(Driver):
         script = open(cli_script, 'rt')
 
         while True:
+
+            self.expect_failure = False
+
             line = script.readline()
 
             if (line == ''):
@@ -78,6 +91,10 @@ class CliDriver(Driver):
             if (len(cmd) == 0):
                 continue
 
+            if (cmd[0] == 'EXPECT_FAILURE:'):
+                self.expect_failure = True
+                cmd.pop(0)
+
             commands.get(cmd[0], self.unknown_command)(cmd)
 
     def run(self):