Add a test that "apitrace diff-images" flags mismatching images.

With this test we add a new EXPECT_FAILURE: attribute to the script
syntax to indicate that a particular apitrace command in the script is
expected to fail. This requires manually invoking the Popen
constructor rather than using subprocess.check_output, (which is
*close* to what we need but fails to assign the output when raising an
exception).

We also remove the globbing from the CMakeLists.txt file to get a
manual ordering of the tests (rather than executing in alphabetic
order).

diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
index 283f56fc055838bf53a94f569b4e5f211b6a5711..39a15c86ef55d454086f8242b5341c97e1143b5e 100644 (file)
--- a/cli/CMakeLists.txt
+++ b/cli/CMakeLists.txt
@@ -1,15 +1,26 @@
-file (GLOB scripts RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.script)
+function (ADD_CLI_TEST)
+    cmake_parse_arguments(
+        TEST
+        # Options
+        ""
+        # One value args
+        "NAME"
+        # Multi value args
+        ""
+        ${ARGN}
+    )

 
 

-list (SORT scripts)
-
-foreach (script ${scripts})

     if (APITRACE_EXECUTABLE)
         add_test(
     if (APITRACE_EXECUTABLE)
         add_test(

-            NAME ${script}
-            COMMAND python ${CMAKE_SOURCE_DIR}/cli_driver.py
-               --apitrace ${APITRACE_EXECUTABLE}
-               --apitrace-source ${APITRACE_SOURCE_DIR}
-               ${CMAKE_CURRENT_SOURCE_DIR}/${script}
+            NAME ${TEST_NAME}
+            COMMAND
+            python ${CMAKE_SOURCE_DIR}/cli_driver.py
+                --apitrace ${APITRACE_EXECUTABLE}
+                --apitrace-source ${APITRACE_SOURCE_DIR}
+                ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_NAME}

         )
     endif ()
         )
     endif ()

-endforeach (script)
+endfunction ()
+
+add_cli_test(NAME "cli-diff-images.script")
+add_cli_test(NAME "cli-diff-images-mismatch.script")

diff --git a/cli/README.markdown b/cli/README.markdown
index 3d49c7c9a355758746de5bb0966eb55a262447f8..6be80a9790ef8aeaffc7354b54370b7bd7830272 100644 (file)
--- a/cli/README.markdown
+++ b/cli/README.markdown
@@ -10,8 +10,10 @@ test in the traces directory. Otherwise, a new test program can be
 written in this directory.
 
 The tests in this directory are found in files with names matching
 written in this directory.
 
 The tests in this directory are found in files with names matching

-*.script. The scripts are simple line-based commands with the
-following meanings based on the first word of each line:
+*.script by convention. The scripts must be listed explicitly in the
+CMakeLists.txt file. Each script consists of simple line-based
+commands with the following meanings (based on the first word of each
+line):

 
   apitrace:     Execute the current apitrace executable being tested
                with the given arguments. If apitrace returns a
 
   apitrace:     Execute the current apitrace executable being tested
                with the given arguments. If apitrace returns a


@@ -26,6 +28,10 @@ following meanings based on the first word of each line:
                 interpreted locally. If this fails for any reason
                 other than "file does not exist" the test will fail.
 
                 interpreted locally. If this fails for any reason
                 other than "file does not exist" the test will fail.
 

-If none of the commands in the script cause the test to fail, then the
-test will pass.
+Commands can be prefixed with "EXPECT_FAILURE:" to indicate that a
+command is expected to return a non-zero value. In this case, a return
+value of zero from the command will cause the test to fail.
+
+If none of the commands in the script cause the test to fail (as
+described above), then the test will pass.

 
 

diff --git a/cli/cli-diff-images-mismatch.script b/cli/cli-diff-images-mismatch.script
new file mode 100644 (file)
index 0000000..9c97fa3
--- /dev/null
+++ b/cli/cli-diff-images-mismatch.script
@@ -0,0 +1,4 @@
+rm_and_mkdir ./tri-out
+apitrace dump-images -o ./tri-out/tri tri.trace
+EXPECT_FAILURE: apitrace diff-images -v ./tri-ref-mismatch ./tri-out
+expect "Comparing ./tri-ref-mismatch/tri0000000027.png and ./tri-out/tri0000000027.png ... MISMATCH\n"

diff --git a/cli/tri-ref-mismatch/tri0000000027.png b/cli/tri-ref-mismatch/tri0000000027.png
new file mode 100644 (file)
index 0000000..e682610
Binary files /dev/null and b/cli/tri-ref-mismatch/tri0000000027.png differ

diff --git a/cli_driver.py b/cli_driver.py
index 6906b738e11c271629951b715a546d169ef23179..a7f4b3d5212a67821e636715140c8bc5e5d0c5f9 100644 (file)
--- a/cli_driver.py
+++ b/cli_driver.py
@@ -33,7 +33,17 @@ class CliDriver(Driver):
     def do_apitrace(self, args):
         cmd = [self.options.apitrace] + args[1:]
  
     def do_apitrace(self, args):
         cmd = [self.options.apitrace] + args[1:]
  

-        self.output = subprocess.check_output(cmd)
+        proc = subprocess.Popen(cmd, stdout = subprocess.PIPE)
+        self.output = proc.communicate()[0]
+
+        proc.wait()
+
+        if (self.expect_failure):
+            if (proc.returncode == 0):
+                fail("Command unexpectedly passed when expecting failure:\n    " + " ".join(cmd))
+        else:
+            if (proc.returncode != 0):
+                fail("Command failed (returned non-zero):\n    " + " ".join(cmd))

 
     def do_expect(self, args):
         expected = json.loads(" ".join(args[1:]))
 
     def do_expect(self, args):
         expected = json.loads(" ".join(args[1:]))


@@ -68,6 +78,9 @@ class CliDriver(Driver):
         script = open(cli_script, 'rt')
 
         while True:
         script = open(cli_script, 'rt')
 
         while True:

+
+            self.expect_failure = False
+

             line = script.readline()
 
             if (line == ''):
             line = script.readline()
 
             if (line == ''):


@@ -78,6 +91,10 @@ class CliDriver(Driver):
             if (len(cmd) == 0):
                 continue
 
             if (len(cmd) == 0):
                 continue
 

+            if (cmd[0] == 'EXPECT_FAILURE:'):
+                self.expect_failure = True
+                cmd.pop(0)
+

             commands.get(cmd[0], self.unknown_command)(cmd)
 
     def run(self):
             commands.get(cmd[0], self.unknown_command)(cmd)
 
     def run(self):