Add a new 'cli' subdirectory for higher-level testing of apitrace CLI.

As described in the README, this new directory is for tests that
perform higher-level testing of the apitrace command-line interface
than either of the existing traces or apps directories.

Compared to traces, these new tests can do higher-level things like
dumping images from traces, comparing images, etc.

Compared to apps, these tests are intended to be portable across
multiple targets. (This isn't actually the case yet because commands
like "apitrace dump-images" currently work only on OpenGL. Also, even
when that is fixed, we will also need to then extend these tests to
have traces for each supported target for testing.)

In this commit, the only new test exercises "apitrace dump-images" and
"apitrace diff-images" on a simple trace file with one frame,
(tri.trace grabbed from apps/gl).

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 47c38cb06b3bfb488b699269d309c93c2edc1a57..10961587b04c6fd91f4f3bc7348923e6a0be0a33 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -90,3 +90,17 @@ enable_testing()
 add_subdirectory (apps)
 add_subdirectory (traces)
 
+# FIXME: The tests in the cli directory are intended to be high-level
+# tests of the apitrace command-line interface which would ideally be
+# portable across all platforms. However, these tests all rely on
+# doing image comparisons and the current implementation of the
+# "apitrace dump-images" command relies on direct invocation of the
+# glretrace command.
+#
+# Someday, we should have more unified commands for replaying traces,
+# dumping images, etc. At that point these cli tests should be usable
+# with all targets so that we can drop the "if (OPENGL_FOUND)"
+# condition here.
+if (OPENGL_FOUND)
+    add_subdirectory (cli)
+endif ()

diff --git a/cli/.gitignore b/cli/.gitignore
new file mode 100644 (file)
index 0000000..5e96abe
--- /dev/null
+++ b/cli/.gitignore
@@ -0,0 +1,2 @@
+!*.trace
+tri-out

diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt
new file mode 100644 (file)
index 0000000..283f56f
--- /dev/null
+++ b/cli/CMakeLists.txt
@@ -0,0 +1,15 @@
+file (GLOB scripts RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.script)
+
+list (SORT scripts)
+
+foreach (script ${scripts})
+    if (APITRACE_EXECUTABLE)
+        add_test(
+            NAME ${script}
+            COMMAND python ${CMAKE_SOURCE_DIR}/cli_driver.py
+               --apitrace ${APITRACE_EXECUTABLE}
+               --apitrace-source ${APITRACE_SOURCE_DIR}
+               ${CMAKE_CURRENT_SOURCE_DIR}/${script}
+        )
+    endif ()
+endforeach (script)

diff --git a/cli/README.markdown b/cli/README.markdown
new file mode 100644 (file)
index 0000000..3d49c7c
--- /dev/null
+++ b/cli/README.markdown
@@ -0,0 +1,31 @@
+This directory tests high-level functionality of apitrace command-line
+interface (cli). See also the neighboring "traces" directory which
+also tests the cli, but in ways that can only be verified by doing
+comparisons of resulting dump files.
+
+For writing new tests, if the functionality can be exercised by a
+sequence of apitrace command invocations, and then a final comparison
+of "apitrace dump" output, then it's likely simplest to write a new
+test in the traces directory. Otherwise, a new test program can be
+written in this directory.
+
+The tests in this directory are found in files with names matching
+*.script. The scripts are simple line-based commands with the
+following meanings based on the first word of each line:
+
+  apitrace:     Execute the current apitrace executable being tested
+               with the given arguments. If apitrace returns a
+               non-zero status, the test will fail.
+
+  expect:       Compare the results of the previously-executed command
+                with the given (json-quoted) string. If the strings
+                are not identical, the test will fail.
+
+  rm_and_mkdir: Remove any existing directory of the given name and
+                then create it. The directory name is always
+                interpreted locally. If this fails for any reason
+                other than "file does not exist" the test will fail.
+
+If none of the commands in the script cause the test to fail, then the
+test will pass.
+

diff --git a/cli/cli-diff-images.script b/cli/cli-diff-images.script
new file mode 100644 (file)
index 0000000..cb4630f
--- /dev/null
+++ b/cli/cli-diff-images.script
@@ -0,0 +1,4 @@
+rm_and_mkdir ./tri-out
+apitrace dump-images -o ./tri-out/tri tri.trace
+apitrace diff-images -v ./tri-ref ./tri-out
+expect "Comparing ./tri-ref/tri0000000027.png and ./tri-out/tri0000000027.png ... MATCH\n"

diff --git a/cli/tri-ref/tri0000000027.png b/cli/tri-ref/tri0000000027.png
new file mode 100644 (file)
index 0000000..b7840bd
Binary files /dev/null and b/cli/tri-ref/tri0000000027.png differ

diff --git a/cli/tri.trace b/cli/tri.trace
new file mode 100644 (file)
index 0000000..f900d52
Binary files /dev/null and b/cli/tri.trace differ

diff --git a/cli_driver.py b/cli_driver.py
new file mode 100644 (file)
index 0000000..6906b73
--- /dev/null
+++ b/cli_driver.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+'''Test driver for scripts in the cli directory.'''
+
+import json, errno, shutil, subprocess
+
+from base_driver import *
+
+class CliDriver(Driver):
+
+    def do_apitrace(self, args):
+        cmd = [self.options.apitrace] + args[1:]
+ 
+        self.output = subprocess.check_output(cmd)
+
+    def do_expect(self, args):
+        expected = json.loads(" ".join(args[1:]))
+        if (self.output != expected):
+            fail("Unexpected output:\n    Expected: %s\n    Received: %s\n" % (expected, self.output))
+
+    def do_rm_and_mkdir(self, args):
+
+        # Operate only on local directories
+        dir = './' + args[1]
+
+        # Failing to delete a directory that doesn't exist is no failure
+        def rmtree_onerror(function, path, excinfo):
+            if excinfo[0] == OSError and excinfo[1].errno != errno.ENOENT:
+                raise
+
+        shutil.rmtree(dir, onerror = rmtree_onerror)
+        os.makedirs(dir)
+
+    def unknown_command(self, args):
+        fail('Broken test script: Unknown command: %s' % (args[0]))
+
+    def run_script(self, cli_script):
+        "Execute the commands in the given cli script."
+
+        commands = {
+            'apitrace': self.do_apitrace,
+            'expect': self.do_expect,
+            'rm_and_mkdir': self.do_rm_and_mkdir
+        }
+
+        script = open(cli_script, 'rt')
+
+        while True:
+            line = script.readline()
+
+            if (line == ''):
+                break
+
+            cmd = line.split()
+
+            if (len(cmd) == 0):
+                continue
+
+            commands.get(cmd[0], self.unknown_command)(cmd)
+
+    def run(self):
+        self.parseOptions()
+
+       self.run_script(self.args[0])
+
+        pass_()
+
+if __name__ == '__main__':
+    CliDriver().run()