SDL  2.0
SDL_test_harness.c File Reference
#include "SDL_config.h"
#include "SDL_test.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
+ Include dependency graph for SDL_test_harness.c:

Go to the source code of this file.

Macros

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"
 
#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
 
#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"
 

Functions

char * SDLTest_GenerateRunSeed (const int length)
 Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z). More...
 
static Uint64 SDLTest_GenerateExecKey (const char *runSeed, char *suiteName, char *testName, int iteration)
 
static SDL_TimerID SDLTest_SetTestTimeout (int timeout, void(*callback)())
 Set timeout handler for test. More...
 
static SDL_NORETURN void SDLTest_BailOut ()
 Timeout handler. Aborts test run and exits harness process. More...
 
static int SDLTest_RunTest (SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
 Execute a test using the given execution key. More...
 
static float GetClock ()
 
int SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
 Execute a test suite using the given run seed and execution key. More...
 

Variables

static Uint32 SDLTest_TestCaseTimeout = 3600
 

Macro Definition Documentation

◆ SDLTEST_FINAL_RESULT_FORMAT

#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"

Definition at line 38 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

◆ SDLTEST_INVALID_NAME_FORMAT

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"

Definition at line 32 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites(), and SDLTest_RunTest().

◆ SDLTEST_LOG_SUMMARY_FORMAT

#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"

Definition at line 35 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

Function Documentation

◆ GetClock()

static float GetClock ( )
static

Definition at line 350 of file SDL_test_harness.c.

Referenced by SDLTest_RunSuites().

351 {
352  float currentClock = clock() / (float) CLOCKS_PER_SEC;
353  return currentClock;
354 }

◆ SDLTest_BailOut()

static SDL_NORETURN void SDLTest_BailOut ( )
static

Timeout handler. Aborts test run and exits harness process.

Definition at line 213 of file SDL_test_harness.c.

References SDLTest_LogError(), and TEST_ABORTED.

Referenced by SDLTest_RunTest().

214 {
215  SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
216  exit(TEST_ABORTED); /* bail out from the test */
217 }
#define TEST_ABORTED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103

◆ SDLTest_GenerateExecKey()

static Uint64 SDLTest_GenerateExecKey ( const char *  runSeed,
char *  suiteName,
char *  testName,
int  iteration 
)
static

Generates an execution key for the fuzzer.

Parameters
runSeedThe run seed to use
suiteNameThe name of the test suite
testNameThe name of the test
iterationThe iteration count
Returns
The generated execution key to initialize the fuzzer with.

Definition at line 101 of file SDL_test_harness.c.

References SDLTest_Md5Context::digest, NULL, SDL_ENOMEM, SDL_Error, SDL_free, SDL_malloc, SDL_memset, SDL_snprintf, SDL_strlen, SDLTest_LogError(), SDLTest_Md5Final(), SDLTest_Md5Init(), and SDLTest_Md5Update().

Referenced by SDLTest_RunSuites().

102 {
103  SDLTest_Md5Context md5Context;
104  Uint64 *keys;
105  char iterationString[16];
106  size_t runSeedLength;
107  size_t suiteNameLength;
108  size_t testNameLength;
109  size_t iterationStringLength;
110  size_t entireStringLength;
111  char *buffer;
112 
113  if (runSeed == NULL || runSeed[0] == '\0') {
114  SDLTest_LogError("Invalid runSeed string.");
115  return -1;
116  }
117 
118  if (suiteName == NULL || suiteName[0] == '\0') {
119  SDLTest_LogError("Invalid suiteName string.");
120  return -1;
121  }
122 
123  if (testName == NULL || testName[0] == '\0') {
124  SDLTest_LogError("Invalid testName string.");
125  return -1;
126  }
127 
128  if (iteration <= 0) {
129  SDLTest_LogError("Invalid iteration count.");
130  return -1;
131  }
132 
133  /* Convert iteration number into a string */
134  SDL_memset(iterationString, 0, sizeof(iterationString));
135  SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
136 
137  /* Combine the parameters into single string */
138  runSeedLength = SDL_strlen(runSeed);
139  suiteNameLength = SDL_strlen(suiteName);
140  testNameLength = SDL_strlen(testName);
141  iterationStringLength = SDL_strlen(iterationString);
142  entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
143  buffer = (char *)SDL_malloc(entireStringLength);
144  if (buffer == NULL) {
145  SDLTest_LogError("Failed to allocate buffer for execKey generation.");
147  return 0;
148  }
149  SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
150 
151  /* Hash string and use half of the digest as 64bit exec key */
152  SDLTest_Md5Init(&md5Context);
153  SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength);
154  SDLTest_Md5Final(&md5Context);
155  SDL_free(buffer);
156  keys = (Uint64 *)md5Context.digest;
157 
158  return keys[0];
159 }
#define SDL_Error
uint64_t Uint64
Definition: SDL_stdinc.h:216
static void iteration()
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
#define SDL_free
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
#define NULL
Definition: begin_code.h:164
GLuint buffer
#define SDL_strlen
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
#define SDL_snprintf
#define SDL_malloc
#define SDL_memset
unsigned char digest[16]
Definition: SDL_test_md5.h:75

◆ SDLTest_GenerateRunSeed()

char* SDLTest_GenerateRunSeed ( const int  length)

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters
lengthThe length of the seed string to generate
Returns
The generated seed string

Definition at line 54 of file SDL_test_harness.c.

References NULL, SDL_ENOMEM, SDL_Error, SDL_malloc, SDLTest_LogError(), SDLTest_Random(), and SDLTest_RandomInitTime().

Referenced by sdltest_generateRunSeed(), and SDLTest_RunSuites().

55 {
56  char *seed = NULL;
57  SDLTest_RandomContext randomContext;
58  int counter;
59 
60  /* Sanity check input */
61  if (length <= 0) {
62  SDLTest_LogError("The length of the harness seed must be >0.");
63  return NULL;
64  }
65 
66  /* Allocate output buffer */
67  seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68  if (seed == NULL) {
69  SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71  return NULL;
72  }
73 
74  /* Generate a random string of alphanumeric characters */
75  SDLTest_RandomInitTime(&randomContext);
76  for (counter = 0; counter < length; counter++) {
77  unsigned int number = SDLTest_Random(&randomContext);
78  char ch = (char) (number % (91 - 48)) + 48;
79  if (ch >= 58 && ch <= 64) {
80  ch = 65;
81  }
82  seed[counter] = ch;
83  }
84  seed[length] = '\0';
85 
86  return seed;
87 }
#define SDL_Error
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
#define NULL
Definition: begin_code.h:164
GLuint counter
#define SDL_malloc
GLuint GLsizei GLsizei * length

◆ SDLTest_RunSuites()

int SDLTest_RunSuites ( SDLTest_TestSuiteReference testSuites[],
const char *  userRunSeed,
Uint64  userExecKey,
const char *  filter,
int  testIterations 
)

Execute a test suite using the given run seed and execution key.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters
testSuitesSuites containing the test case.
userRunSeedCustom run seed provided by user, or NULL to autogenerate one.
userExecKeyCustom execution key provided by user, or 0 to autogenerate one.
filterFilter specification. NULL disables. Case sensitive.
testIterationsNumber of iterations to run each test case.
Returns
Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 370 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free, SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, and SDLTest_TestSuiteReference::testCases.

Referenced by main().

371 {
372  int totalNumberOfTests = 0;
373  int failedNumberOfTests = 0;
374  int suiteCounter;
375  int testCounter;
376  int iterationCounter;
377  SDLTest_TestSuiteReference *testSuite;
378  const SDLTest_TestCaseReference *testCase;
379  const char *runSeed = NULL;
380  char *currentSuiteName;
381  char *currentTestName;
382  Uint64 execKey;
383  float runStartSeconds;
384  float suiteStartSeconds;
385  float testStartSeconds;
386  float runEndSeconds;
387  float suiteEndSeconds;
388  float testEndSeconds;
389  float runtime;
390  int suiteFilter = 0;
391  char *suiteFilterName = NULL;
392  int testFilter = 0;
393  char *testFilterName = NULL;
394  SDL_bool forceTestRun = SDL_FALSE;
395  int testResult = 0;
396  int runResult = 0;
397  Uint32 totalTestFailedCount = 0;
398  Uint32 totalTestPassedCount = 0;
399  Uint32 totalTestSkippedCount = 0;
400  Uint32 testFailedCount = 0;
401  Uint32 testPassedCount = 0;
402  Uint32 testSkippedCount = 0;
403  Uint32 countSum = 0;
404  const SDLTest_TestCaseReference **failedTests;
405 
406  /* Sanitize test iterations */
407  if (testIterations < 1) {
408  testIterations = 1;
409  }
410 
411  /* Generate run see if we don't have one already */
412  if (userRunSeed == NULL || userRunSeed[0] == '\0') {
413  runSeed = SDLTest_GenerateRunSeed(16);
414  if (runSeed == NULL) {
415  SDLTest_LogError("Generating a random seed failed");
416  return 2;
417  }
418  } else {
419  runSeed = userRunSeed;
420  }
421 
422 
423  /* Reset per-run counters */
424  totalTestFailedCount = 0;
425  totalTestPassedCount = 0;
426  totalTestSkippedCount = 0;
427 
428  /* Take time - run start */
429  runStartSeconds = GetClock();
430 
431  /* Log run with fuzzer parameters */
432  SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
433 
434  /* Count the total number of tests */
435  suiteCounter = 0;
436  while (testSuites[suiteCounter]) {
437  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
438  suiteCounter++;
439  testCounter = 0;
440  while (testSuite->testCases[testCounter])
441  {
442  testCounter++;
443  totalNumberOfTests++;
444  }
445  }
446 
447  /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
448  failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
449  if (failedTests == NULL) {
450  SDLTest_LogError("Unable to allocate cache for failed tests");
452  return -1;
453  }
454 
455  /* Initialize filtering */
456  if (filter != NULL && filter[0] != '\0') {
457  /* Loop over all suites to check if we have a filter match */
458  suiteCounter = 0;
459  while (testSuites[suiteCounter] && suiteFilter == 0) {
460  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
461  suiteCounter++;
462  if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
463  /* Matched a suite name */
464  suiteFilter = 1;
465  suiteFilterName = testSuite->name;
466  SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
467  break;
468  }
469 
470  /* Within each suite, loop over all test cases to check if we have a filter match */
471  testCounter = 0;
472  while (testSuite->testCases[testCounter] && testFilter == 0)
473  {
474  testCase = testSuite->testCases[testCounter];
475  testCounter++;
476  if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
477  /* Matched a test name */
478  suiteFilter = 1;
479  suiteFilterName = testSuite->name;
480  testFilter = 1;
481  testFilterName = testCase->name;
482  SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
483  break;
484  }
485  }
486  }
487 
488  if (suiteFilter == 0 && testFilter == 0) {
489  SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
490  SDLTest_Log("Exit code: 2");
491  SDL_free((void *) failedTests);
492  return 2;
493  }
494  }
495 
496  /* Loop over all suites */
497  suiteCounter = 0;
498  while(testSuites[suiteCounter]) {
499  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
500  currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
501  suiteCounter++;
502 
503  /* Filter suite if flag set and we have a name */
504  if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
505  SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
506  /* Skip suite */
507  SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
508  suiteCounter,
509  currentSuiteName);
510  } else {
511 
512  /* Reset per-suite counters */
513  testFailedCount = 0;
514  testPassedCount = 0;
515  testSkippedCount = 0;
516 
517  /* Take time - suite start */
518  suiteStartSeconds = GetClock();
519 
520  /* Log suite started */
521  SDLTest_Log("===== Test Suite %i: '%s' started\n",
522  suiteCounter,
523  currentSuiteName);
524 
525  /* Loop over all test cases */
526  testCounter = 0;
527  while(testSuite->testCases[testCounter])
528  {
529  testCase = testSuite->testCases[testCounter];
530  currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
531  testCounter++;
532 
533  /* Filter tests if flag set and we have a name */
534  if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
535  SDL_strcmp(testFilterName, testCase->name) != 0) {
536  /* Skip test */
537  SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
538  suiteCounter,
539  testCounter,
540  currentTestName);
541  } else {
542  /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
543  if (testFilter == 1 && !testCase->enabled) {
544  SDLTest_Log("Force run of disabled test since test filter was set");
545  forceTestRun = SDL_TRUE;
546  }
547 
548  /* Take time - test start */
549  testStartSeconds = GetClock();
550 
551  /* Log test started */
552  SDLTest_Log("----- Test Case %i.%i: '%s' started",
553  suiteCounter,
554  testCounter,
555  currentTestName);
556  if (testCase->description != NULL && testCase->description[0] != '\0') {
557  SDLTest_Log("Test Description: '%s'",
558  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
559  }
560 
561  /* Loop over all iterations */
562  iterationCounter = 0;
563  while(iterationCounter < testIterations)
564  {
565  iterationCounter++;
566 
567  if (userExecKey != 0) {
568  execKey = userExecKey;
569  } else {
570  execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
571  }
572 
573  SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
574  testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
575 
576  if (testResult == TEST_RESULT_PASSED) {
577  testPassedCount++;
578  totalTestPassedCount++;
579  } else if (testResult == TEST_RESULT_SKIPPED) {
580  testSkippedCount++;
581  totalTestSkippedCount++;
582  } else {
583  testFailedCount++;
584  totalTestFailedCount++;
585  }
586  }
587 
588  /* Take time - test end */
589  testEndSeconds = GetClock();
590  runtime = testEndSeconds - testStartSeconds;
591  if (runtime < 0.0f) runtime = 0.0f;
592 
593  if (testIterations > 1) {
594  /* Log test runtime */
595  SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
596  SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
597  } else {
598  /* Log test runtime */
599  SDLTest_Log("Total Test runtime: %.1f sec", runtime);
600  }
601 
602  /* Log final test result */
603  switch (testResult) {
604  case TEST_RESULT_PASSED:
605  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
606  break;
607  case TEST_RESULT_FAILED:
608  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
609  break;
611  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
612  break;
613  }
614 
615  /* Collect failed test case references for repro-step display */
616  if (testResult == TEST_RESULT_FAILED) {
617  failedTests[failedNumberOfTests] = testCase;
618  failedNumberOfTests++;
619  }
620  }
621  }
622 
623  /* Take time - suite end */
624  suiteEndSeconds = GetClock();
625  runtime = suiteEndSeconds - suiteStartSeconds;
626  if (runtime < 0.0f) runtime = 0.0f;
627 
628  /* Log suite runtime */
629  SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
630 
631  /* Log summary and final Suite result */
632  countSum = testPassedCount + testFailedCount + testSkippedCount;
633  if (testFailedCount == 0)
634  {
635  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
636  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
637  }
638  else
639  {
640  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
641  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
642  }
643 
644  }
645  }
646 
647  /* Take time - run end */
648  runEndSeconds = GetClock();
649  runtime = runEndSeconds - runStartSeconds;
650  if (runtime < 0.0f) runtime = 0.0f;
651 
652  /* Log total runtime */
653  SDLTest_Log("Total Run runtime: %.1f sec", runtime);
654 
655  /* Log summary and final run result */
656  countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
657  if (totalTestFailedCount == 0)
658  {
659  runResult = 0;
660  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
661  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
662  }
663  else
664  {
665  runResult = 1;
666  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
667  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
668  }
669 
670  /* Print repro steps for failed tests */
671  if (failedNumberOfTests > 0) {
672  SDLTest_Log("Harness input to repro failures:");
673  for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
674  SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
675  }
676  }
677  SDL_free((void *) failedTests);
678 
679  SDLTest_Log("Exit code: %d", runResult);
680  return runResult;
681 }
#define SDLTEST_INVALID_NAME_FORMAT
char * SDLTest_GenerateRunSeed(const int length)
Generates a random run seed string for the harness. The generated seed will contain alphanumeric char...
#define SDLTEST_FINAL_RESULT_FORMAT
GLfloat f
#define SDL_Error
uint64_t Uint64
Definition: SDL_stdinc.h:216
const SDLTest_TestCaseReference ** testCases
GLuint const GLchar * name
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
#define SDL_free
static float GetClock()
#define TEST_RESULT_NO_ASSERT
#define NULL
Definition: begin_code.h:164
SDL_bool
Definition: SDL_stdinc.h:161
#define TEST_RESULT_PASSED
uint32_t Uint32
Definition: SDL_stdinc.h:203
#define SDLTEST_LOG_SUMMARY_FORMAT
#define SDL_PRIu64
Definition: SDL_stdinc.h:238
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
static Uint64 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
#define SDL_malloc
#define SDL_strcmp
#define TEST_RESULT_SKIPPED
static int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter

◆ SDLTest_RunTest()

static int SDLTest_RunTest ( SDLTest_TestSuiteReference testSuite,
const SDLTest_TestCaseReference testCase,
Uint64  execKey,
SDL_bool  forceTestRun 
)
static

Execute a test using the given execution key.

Parameters
testSuiteSuite containing the test case.
testCaseCase to execute.
execKeyExecution key for the fuzzer.
forceTestRunForce test to run even if test was disabled in suite.
Returns
Test case result.

Definition at line 230 of file SDL_test_harness.c.

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_FALSE, SDL_RemoveTimer, SDLTest_AssertSummaryToTestResult(), SDLTest_BailOut(), SDLTEST_FINAL_RESULT_FORMAT, SDLTest_FuzzerInit(), SDLTest_GetFuzzerInvocationCount(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTest_LogAssertSummary(), SDLTest_LogError(), SDLTest_ResetAssertSummary(), SDLTest_SetTestTimeout(), SDLTest_TestCaseTimeout, TEST_ABORTED, TEST_RESULT_FAILED, TEST_RESULT_SETUP_FAILURE, TEST_RESULT_SKIPPED, TEST_SKIPPED, TEST_STARTED, SDLTest_TestCaseReference::testCase, SDLTest_TestSuiteReference::testCases, SDLTest_TestSuiteReference::testSetUp, testSuites, and SDLTest_TestSuiteReference::testTearDown.

Referenced by SDLTest_RunSuites().

231 {
232  SDL_TimerID timer = 0;
233  int testCaseResult = 0;
234  int testResult = 0;
235  int fuzzerCount;
236 
237  if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
238  {
239  SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
241  }
242 
243  if (!testCase->enabled && forceTestRun == SDL_FALSE)
244  {
245  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
246  return TEST_RESULT_SKIPPED;
247  }
248 
249  /* Initialize fuzzer */
250  SDLTest_FuzzerInit(execKey);
251 
252  /* Reset assert tracker */
254 
255  /* Set timeout timer */
257 
258  /* Maybe run suite initalizer function */
259  if (testSuite->testSetUp) {
260  testSuite->testSetUp(0x0);
262  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
264  }
265  }
266 
267  /* Run test case function */
268  testCaseResult = testCase->testCase(0x0);
269 
270  /* Convert test execution result into harness result */
271  if (testCaseResult == TEST_SKIPPED) {
272  /* Test was programatically skipped */
273  testResult = TEST_RESULT_SKIPPED;
274  } else if (testCaseResult == TEST_STARTED) {
275  /* Test did not return a TEST_COMPLETED value; assume it failed */
276  testResult = TEST_RESULT_FAILED;
277  } else if (testCaseResult == TEST_ABORTED) {
278  /* Test was aborted early; assume it failed */
279  testResult = TEST_RESULT_FAILED;
280  } else {
281  /* Perform failure analysis based on asserts */
282  testResult = SDLTest_AssertSummaryToTestResult();
283  }
284 
285  /* Maybe run suite cleanup function (ignore failed asserts) */
286  if (testSuite->testTearDown) {
287  testSuite->testTearDown(0x0);
288  }
289 
290  /* Cancel timeout timer */
291  if (timer) {
292  SDL_RemoveTimer(timer);
293  }
294 
295  /* Report on asserts and fuzzer usage */
296  fuzzerCount = SDLTest_GetFuzzerInvocationCount();
297  if (fuzzerCount > 0) {
298  SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
299  }
300 
301  /* Final log based on test execution result */
302  if (testCaseResult == TEST_SKIPPED) {
303  /* Test was programatically skipped */
304  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
305  } else if (testCaseResult == TEST_STARTED) {
306  /* Test did not return a TEST_COMPLETED value; assume it failed */
307  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
308  } else if (testCaseResult == TEST_ABORTED) {
309  /* Test was aborted early; assume it failed */
310  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
311  } else {
313  }
314 
315  return testResult;
316 }
#define TEST_ABORTED
SDLTest_TestCaseTearDownFp testTearDown
#define SDLTEST_FINAL_RESULT_FORMAT
static Uint32 SDLTest_TestCaseTimeout
SDLTest_TestCaseSetUpFp testSetUp
int SDLTest_GetFuzzerInvocationCount(void)
#define TEST_RESULT_FAILED
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
static SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
void SDLTest_ResetAssertSummary(void)
Resets the assert summary counters to zero.
#define TEST_RESULT_SETUP_FAILURE
void SDLTest_LogAssertSummary(void)
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
#define TEST_STARTED
int SDLTest_AssertSummaryToTestResult(void)
Converts the current assert summary state to a test result.
#define NULL
Definition: begin_code.h:164
GLuint GLfloat x0
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
static SDL_NORETURN void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
void SDLTest_FuzzerInit(Uint64 execKey)
#define TEST_RESULT_SKIPPED
SDLTest_TestCaseFp testCase
#define TEST_SKIPPED
#define SDL_RemoveTimer
int SDL_TimerID
Definition: SDL_timer.h:86

◆ SDLTest_SetTestTimeout()

static SDL_TimerID SDLTest_SetTestTimeout ( int  timeout,
void(*)()  callback 
)
static

Set timeout handler for test.

Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.

Parameters
timeoutTimeout interval in seconds.
callbackFunction that will be called after timeout has elapsed.
Returns
Timer id or -1 on failure.

Definition at line 172 of file SDL_test_harness.c.

References callback(), NULL, SDL_AddTimer, SDL_GetError, SDL_INIT_TIMER, SDL_InitSubSystem, SDL_NORETURN, SDL_WasInit, and SDLTest_LogError().

Referenced by SDLTest_RunTest().

173 {
174  Uint32 timeoutInMilliseconds;
175  SDL_TimerID timerID;
176 
177  if (callback == NULL) {
178  SDLTest_LogError("Timeout callback can't be NULL");
179  return -1;
180  }
181 
182  if (timeout < 0) {
183  SDLTest_LogError("Timeout value must be bigger than zero.");
184  return -1;
185  }
186 
187  /* Init SDL timer if not initialized before */
188  if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
190  SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
191  return -1;
192  }
193  }
194 
195  /* Set timer */
196  timeoutInMilliseconds = timeout * 1000;
197  timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
198  if (timerID == 0) {
199  SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
200  return -1;
201  }
202 
203  return timerID;
204 }
#define SDL_GetError
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
#define SDL_InitSubSystem
#define SDL_AddTimer
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
#define NULL
Definition: begin_code.h:164
#define SDL_INIT_TIMER
Definition: SDL.h:77
GLbitfield GLuint64 timeout
uint32_t Uint32
Definition: SDL_stdinc.h:203
GLuint GLfloat x0
#define SDL_WasInit
int SDL_TimerID
Definition: SDL_timer.h:86

Variable Documentation

◆ SDLTest_TestCaseTimeout

Uint32 SDLTest_TestCaseTimeout = 3600
static

Definition at line 41 of file SDL_test_harness.c.

Referenced by SDLTest_RunTest().