Integrating Squish GUI Tester with Squish Coco

Last edited on

The code coverage tool Squish Coco does not only work with unit tests. It can also be used to analyze the coverage of GUI tests. It's easily possible to run tools alongside each other. That will give the C/C++ code coverage of the whole test suite. A more in-depth analysis will be possible when correlating each test case (and its results) with the respective coverage information. Especially since Squish Coco features the comparisons of individual executions including the calculation of an optimal order.

General Approach

The approach depicted below is based on the possibility to apply information about the name, the result and a free-form comment to each execution (stored in a *.csexe files). See the documentation section Execution Comment, Name And Status for more details.

As an example we'll use Squish for Qt's addressbook on a Unix-based system and a JavaScript test script.

  1. First we'll initialize the execution data with the name of the Squish test case that is being run:

    function main()
{

    var currentAUT = currentApplicationContext();
    var execution = currentAUT.cwd + "\\" + currentAUT.name + ".exe.csexe"
    var testCase = squishinfo.testCase;
    var testExecutionName = testCase.substr(testCase.lastIndexOf('/') + 1);
    var file = File.open(execution, "a");
    file.write("*" + testExecutionName + "\n");    
    file.close();

    var ctx = startApplication("addressbook");
...
    test.js

  2. Insert the main test script at this point

  3. After the main test script we'll log the result of the test for the coverage tool:

    ...
    // wait until AUT shutdown
    while (ctx.isRunning) {
        snooze(1); // increase time if not enough to dump coverage data
    }

    // test result summary and status
    var positive = test.resultCount("passes");
    var negative = test.resultCount("fails") +
               test.resultCount("errors") +
               test.resultCount("fatals");
    var msg = "TEST RESULTS - Passed: " + positive +  " | " + "Failed/Errored/Fatal: " + negative;
    var status = negative == 0 ? "PASSED" : "FAILED";

    var file = File.open(execution, "a");
    file.write("<html><body>" + msg + "</body></html>\n");
    file.write("!" + status + "\n")
    file.close();
}
    test.js continued

When you execute scripts containing these steps, the Squish Coco Execution Report loads with the test case name, status and execution summary in the execution details and comments.

Simplified for Reuse

  1. Create a file called squishCocoLogging.js in Test Suite Resources with the following functions:

    function getExecutionPath(){
    var currentAUT = currentApplicationContext();
    var execution = currentAUT.cwd + "\\" + currentAUT.name + ".exe.csexe"
    return execution;
}

function logTestNameToCocoReport(currentTestCase, execution){
    var testExecutionName = currentTestCase.substr(currentTestCase.lastIndexOf('\\') + 1);
    var file = File.open(execution, "a");
    file.write("*" + testExecutionName + "\n");
    file.close();
}

function logTestResultsToCocoReport(testInfo, execution){

    var currentAUT = currentApplicationContext();

    // wait until AUT shuts down
    while (currentAUT.isRunning)
      snooze(5);

    // collect test result summary and status
    var positive = testInfo.resultCount("passes");
    var negative = testInfo.resultCount("fails") + testInfo.resultCount("errors") +
        testInfo.resultCount("fatals");
    var msg = "TEST RESULTS - Passed: " + positive +  " | " + "Failed/Errored/Fatal: " + negative;
    var status = negative == 0 ? "PASSED" : "FAILED";

    // output results and status to Coco execution report file
    var file = File.open(execution, "a");
    file.write("<html><body>" + msg + "</body></html>\n");
    file.write("!" + status + "\n")
    file.close();
}
    JAVASCRIPT - squishCocoLogging.js
    import re

def getExecutionPath():
    currentAUT = currentApplicationContext()
    execution = "%(currAUTPath)s\\%(currAUTName)s.exe.csexe" % {"currAUTPath" : currentAUT.cwd, "currAUTName" : currentAUT.name}
    return execution

def logTestNameToCocoReport(currentTestCase, execution):
    testExecutionName = re.search(r'[^\\]\w*$', currentTestCase)
    testExecutionName = testExecutionName.group(0)
    file = open(execution, "a")
    file.write("*" + testExecutionName + "\n")
    file.close()

def logTestResultsToCocoReport(testInfo, execution):
    currentAUT = currentApplicationContext()
    # wait until AUT shuts down
    while (currentAUT.isRunning):
        snooze(5)

    # collect test result summary and status
    positive = testInfo.resultCount("passes")
    negative = testInfo.resultCount("fails") + testInfo.resultCount("errors") + testInfo.resultCount("fatals")
    msg = "TEST RESULTS - Passed: %(positive)s  |  Failed/Errored/Fatal: %(negative)s" % {"positive": positive, "negative": negative}
    if negative == 0:
        status = "PASSED"
    else:
        status = "FAILED"

    # output results and status to Coco execution report file
    file = open(execution, "a")
    file.write("<html><body>" + msg + "</body></html>\n")
    file.write("!" + status + "\n")
    file.close()
    PYTHON - squishCocoLogging.py

  2. Add the following function calls after startApplication() in the main test script:

    execution = getExecutionPath();
logTestNameToCocoReport(squishinfo.testCase, execution);
    JAVASCRIPT- test.js part a
    execution = getExecutionPath()
logTestNameToCocoReport(squishinfo.testCase, execution)
    PYTHON - test.py part a

  3. At the end of your script, after closing the AUT (for example with steps clicking File > Exit), call the following function:

    logTestResultsToCocoReport(test, execution);
    JAVASCRIPT- test.js part b
    logTestResultsToCocoReport(test, execution)
    PYTHON - test.py part b

  4. In the event your AUT closes unexpectedly, or a script error occurs, incorporating a try, catch, finally ensures your results still output to the Coco report file

Your main test script should appear similar to the following:

source(findFile("scripts","squishCocoLogging.JS"))

function main()
{
    startApplication("addressbook");
    execution = getExecutionPath();
    logTestNameToCocoReport(squishinfo.testCase, execution);

    try{
        // body of script
       }
       catch(e){
           test.fail('An unexpected error occurred', e.message)
       }
       finally{
           logTestResultsToCocoReport(test, execution)
       }
}
JAVASCRIPT - test.js example
source(findFile("scripts","squishCocoLogging.py"))

def main():
    startApplication("addressbook")
    execution = getExecutionPath()
    logTestNameToCocoReport(squishinfo.testCase, execution)

    try:
        try:
            # body of script
        except Exception, e:
            test.fail("test failed: ", e)
    finally:
        logTestResultsToCocoReport(test,execution)
PYTHON - test.js example