string silResultsDirectory = "_silUnitResults";
struct stringLow {
string value;
string lower;
}
struct suiteResults {
string name;
number total;
number passes;
number failures;
string resultXML;
}
/**
* Sorts an array of strings regardless of casing.
* Normal Sort: A,B,C,a,b,c
* This Sort: A,a,B,b,C,c
* @param array - The array to sort
* @return The sorted array
*/
function sortAlpha(string[] array) {
stringLow[] lowerResults;
for(string a in array) {
stringLow sl;
sl.value = a;
sl.lower = toLower(a);
lowerResults += sl;
}
try { lowerResults = arrayStructSort(lowerResults, "lower"); } catch {}
string[] result;
for(stringLow sl in lowerResults) { result += sl.value; }
return result;
}
/**
* Lists all the files in a given directory tree.
* @param dirName - The path of directory to search
* @param excludes - A list of directories or files to exclude from results
* @return The list of files found in the directory
*/
function directoryFiles(string dirName, string[] excludes) {
string[] dirScripts;
if(arrayElementExists(excludes, dirName)) { return dirScripts; }
string[] dirs = findDirectories(dirName, ".*");
dirs = sortAlpha(dirs);
for(string d in dirs) {
dirScripts += directoryFiles(d, excludes);
}
string[] dirFiles;
for(string df in findFiles(dirName, ".*\\.test.sil")) {
boolean toAdd = true;
for(string ex in excludes) {
if(startsWith(df, ex)) { toAdd = false; break; }
}
if(toAdd) { dirFiles += df; }
}
dirScripts += sortAlpha(dirFiles);
return dirScripts;
}
/**
* Creates a JUnit style XML tag for passing tests
* @param className - The name of current test file
* @param name - The name of the current test
* @return XML representation
**/
function passText(string className, string name) {
return " <testcase classname='"+className+"' name='"+name+"'/>";
}
/**
* Creates a JUnit style XML tag for failing tests
* @param className - The name of current test file
* @param name - The name of the current test
* @param type - The 'assert' type that caused the failure
* @param message - The error message printed in the error failure tag
* @return XML representation
**/
function failText(string className, string name, string type, string message) {
return " <testcase classname='"+className+"' name='"+name+"'>\n <failure type='"+type+"'>"+message+"</failure>\n </testcase>";
}
/**
* Creates a struct representing the results of a testcase
* @param name - The name of the testcase
* @param total - The total numbers of tests within the testcase
* @param pass - The number of passed tests within the testcase
* @param fail - The number of failed tests within the testcase
* @param resultXML - The path of the JUnit XML file
* @return The struct representation of the testcase results
**/
function processResult(string name, number total, number pass, number fail, string resultXML) {
suiteResults result;
result.name = name;
result.total = total;
result.passes = pass;
result.failures = fail;
result.resultXML = resultXML;
return result;
}
/**
* Creates a string representation of the testcase results
* @return String of information about testcase
**/
function printResult(suiteResults result) {
return result.name+" ---> Tests: "+result.total+" (P: "+result.passes+" - F: "+result.failures+")";
}
/**
* Retrieves a list functions within provided string, where the function is annoted with a specfic label
* @param testText - String to search for annotations
* @param annotation - Text used to annotate functions. Examples: TEST, BEFORE, AFTER, BEFOREEACH, AFTEREACH
* @return String array of function names containing annotation
**/
function getAnnotatedFunctions(string testText, string annotation) {
string[] results;
string regPre = "(?<=(\\/\\*\\*";
string regPost = "\\*\\*\\/(\n|.)function )).+?(?=\\()";
string regex = "("+regPre+"(?i)"+annotation+regPost+")|("+regPre+".(?i)"+annotation+"."+regPost+")"; // Creates a complex REGEX looking for variations in silUnit annotations
testText = replace(testText, "\r\n", "\n");
string current = testText;
// Finds all matched cases
while(matchEnd(current, regex) != -1) {
results += matchText(current, regex);
current = substring(current, matchEnd(current, regex), length(current));
}
return results;
}
/**
* Parses a provided test file, generates a 'runner' script based on annotations within test file, and executes the tests
* @param path - The path of a test file to evaluate
* @return String representation of test results
**/
function executeTest(string path) {
// Gets annotationed functions from test file
string testText = readFromTextFile(path);
string[] testNames = getAnnotatedFunctions(testText, "TEST");
string[] before = getAnnotatedFunctions(testText, "BEFORE");
string[] beforeEach = getAnnotatedFunctions(testText, "BEFOREEACH");
string[] after = getAnnotatedFunctions(testText, "AFTER");
string[] afterEach = getAnnotatedFunctions(testText, "AFTEREACH");
// Creates the 'runner' script
if(!directoryExists(silResultsDirectory)) { createDirectory(silResultsDirectory); }
string[] pathSplit = split(path, "/");
string newPath = silResultsDirectory+"/"+pathSplit[size(pathSplit)-1]+"Run.sil";
if(fileExists(newPath)) { deleteFile(newPath); }
createFile(newPath);
path = replace(path, silEnv("sil.home")+"/", "");
printInFile(newPath, "include \""+path+"\";");
printInFile(newPath, "string directory = \""+silResultsDirectory+"\";");
printInFile(newPath, "string testResults = directory+\"/"+replace(path, "/", "_")+"Results.xml\";");
printInFile(newPath, "number testNumbs = "+size(testNames)+";");
printInFile(newPath, "number passes = 0;");
printInFile(newPath, "number failures = 0;");
printInFile(newPath, " ");
printInFile(newPath, "if(!directoryExists(directory)) { createDirectory(directory); }");
printInFile(newPath, "if(fileExists(testResults)) { deleteFile(testResults); }");
printInFile(newPath, "createFile(testResults);");
printInFile(newPath, "printInFile(testResults, \"<testsuite tests='"+size(testNames)+"'>\");");
printInFile(newPath, " ");
for(string b in before) {
printInFile(newPath, "try { "+b+"(); } catch { logPrint(\"WARN\", \" - "+path+": "+b+"\"); }");
}
// Adds each test to the 'runner' script
for(string t in testNames) {
printInFile(newPath, "// -----------------------------------------------------------------------------");
for(string be in beforeEach) { printInFile(newPath, "try { logPrint(\"WARN\", \" - "+path+": "+be+"\"); "+be+"(); } catch { }"); }
printInFile(newPath, "try { ");
printInFile(newPath, " "+t+"();");
printInFile(newPath, " logPrint(\"WARN\", \" - "+path+": "+t+"\");");
printInFile(newPath, " passes++;");
printInFile(newPath, " printInFile(testResults, passText(\""+path+"\", \""+t+"\"));");
printInFile(newPath, "} catch string err {");
printInFile(newPath, " string[] messArr = split(err, \"-\");");
printInFile(newPath, " printInFile(testResults, failText(\""+path+"\", \""+t+"\", messArr[0], messArr[1]));");
printInFile(newPath, " logPrint(\"ERROR\", \" - "+path+": "+t+"-FAILURE: \"+messArr[0]+\"-\"+messArr[1]); ");
printInFile(newPath, " failures++;");
printInFile(newPath, "}");
for(string be in afterEach) { printInFile(newPath, "try { logPrint(\"WARN\", \" - "+path+": "+be+"\"); "+be+"(); } catch { }"); }
printInFile(newPath, " ");
}
for(string b in after) {
printInFile(newPath, "try { "+b+"(); } catch { logPrint(\"WARN\", \" - "+path+": "+b+"\"); }");
}
printInFile(newPath, "printInFile(testResults, \"</testsuite>\");");
printInFile(newPath, "return processResult(\""+path+"\", testNumbs, passes, failures, testResults);");
string resultString = call("", newPath, {});
// deleteFile(newPath);
return resultString;
}
/**
* Executes a series of testcases within a provided directory. Generates a global XML file containing all test results.
* @param testDirectory - A directory to search for scripts ending in '.test.sil'
**/
function executeSuite(string testDirectory) {
string outputName = replace(testDirectory, silEnv("sil.home"), "");
if(outputName == "") { outputName = "silprograms"; }
string suiteOutputFile = silResultsDirectory+outputName+".xml";
runnerLog("Result File: "+suiteOutputFile);
if(fileExists(suiteOutputFile)) { deleteFile(suiteOutputFile); }
createFile(suiteOutputFile);
printInFile(suiteOutputFile, "<testsuites>");
string[] files = directoryFiles(testDirectory, {});
runnerLog(" -- Test Files: "+size(files));
suiteResults rTotal = processResult("Total", 0, 0, 0, "");
for(string f in files) {
suiteResults r = executeTest(f);
printInFile(suiteOutputFile, trim(readFromTextFile(r.resultXML)));
deleteFile(r.resultXML);
runnerLog(printResult(r));
rTotal.total += r.total;
rTotal.passes += r.passes;
rTotal.failures += r.failures;
rTotal.resultXML += r.resultXML;
}
printInFile(suiteOutputFile, "</testsuites>");
return rTotal;
}
////////////////////////////////////////////////////////////////////////////////
// ASSERTS //
////////////////////////////////////////////////////////////////////////////////
function assertEquals(string expected, string result) {
if(toUpper(expected) != toUpper(result)) {
string resultString = "assertEquals-'"+result+"' does not equal the expected result '"+expected+"'";
throw resultString;
}
}
function assertNotEquals(string expected, string result) {
if(toUpper(expected) == toUpper(result)) {
string resultString = "assertNotEquals-'"+result+"' does equal the expected result '"+expected+"'";
throw resultString;
}
}
function assertNull(string result) {
if(isNotNull(result)) {
string resultString = "assertNull-'"+result+"' is not null";
throw resultString;
}
}
function assertNotNull(string result) {
if(isNull(result)) {
string resultString = "assertNotNull-Value is null";
throw resultString;
}
}
function assertTrue(boolean result) {
if(!result) {
string resultString = "assertTrue-Value is false";
throw resultString;
}
}
function assertFalse(boolean result) {
if(result) {
string resultString = "assertTrue-Value is true";
throw resultString;
}
} |