SDL  2.0
SDL_test_harness.c File Reference
#include "SDL_config.h"
#include "SDL_test.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
+ Include dependency graph for SDL_test_harness.c:

Go to the source code of this file.

Macros

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"
 
#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"
 
#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"
 

Functions

char * SDLTest_GenerateRunSeed (const int length)
 Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z). More...
 
static Uint64 SDLTest_GenerateExecKey (const char *runSeed, char *suiteName, char *testName, int iteration)
 
static SDL_TimerID SDLTest_SetTestTimeout (int timeout, void(*callback)())
 Set timeout handler for test. More...
 
static SDL_NORETURN void SDLTest_BailOut ()
 Timeout handler. Aborts test run and exits harness process. More...
 
static int SDLTest_RunTest (SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
 Execute a test using the given execution key. More...
 
static float GetClock ()
 
int SDLTest_RunSuites (SDLTest_TestSuiteReference *testSuites[], const char *userRunSeed, Uint64 userExecKey, const char *filter, int testIterations)
 Execute a test suite using the given run seed and execution key. More...
 

Variables

static Uint32 SDLTest_TestCaseTimeout = 3600
 

Macro Definition Documentation

◆ SDLTEST_FINAL_RESULT_FORMAT

#define SDLTEST_FINAL_RESULT_FORMAT   ">>> %s '%s': %s\n"

Definition at line 38 of file SDL_test_harness.c.

◆ SDLTEST_INVALID_NAME_FORMAT

#define SDLTEST_INVALID_NAME_FORMAT   "(Invalid)"

Definition at line 32 of file SDL_test_harness.c.

◆ SDLTEST_LOG_SUMMARY_FORMAT

#define SDLTEST_LOG_SUMMARY_FORMAT   "%s Summary: Total=%d Passed=%d Failed=%d Skipped=%d"

Definition at line 35 of file SDL_test_harness.c.

Function Documentation

◆ GetClock()

static float GetClock ( )
static

Definition at line 350 of file SDL_test_harness.c.

351 {
352  float currentClock = clock() / (float) CLOCKS_PER_SEC;
353  return currentClock;
354 }

Referenced by SDLTest_RunSuites().

◆ SDLTest_BailOut()

static SDL_NORETURN void SDLTest_BailOut ( )
static

Timeout handler. Aborts test run and exits harness process.

Definition at line 213 of file SDL_test_harness.c.

214 {
215  SDLTest_LogError("TestCaseTimeout timer expired. Aborting test run.");
216  exit(TEST_ABORTED); /* bail out from the test */
217 }

References SDLTest_LogError(), and TEST_ABORTED.

Referenced by SDLTest_RunTest().

◆ SDLTest_GenerateExecKey()

static Uint64 SDLTest_GenerateExecKey ( const char *  runSeed,
char *  suiteName,
char *  testName,
int  iteration 
)
static

Generates an execution key for the fuzzer.

Parameters
runSeedThe run seed to use
suiteNameThe name of the test suite
testNameThe name of the test
iterationThe iteration count
Returns
The generated execution key to initialize the fuzzer with.

Definition at line 101 of file SDL_test_harness.c.

102 {
103  SDLTest_Md5Context md5Context;
104  Uint64 *keys;
105  char iterationString[16];
106  size_t runSeedLength;
107  size_t suiteNameLength;
108  size_t testNameLength;
109  size_t iterationStringLength;
110  size_t entireStringLength;
111  char *buffer;
112 
113  if (runSeed == NULL || runSeed[0] == '\0') {
114  SDLTest_LogError("Invalid runSeed string.");
115  return -1;
116  }
117 
118  if (suiteName == NULL || suiteName[0] == '\0') {
119  SDLTest_LogError("Invalid suiteName string.");
120  return -1;
121  }
122 
123  if (testName == NULL || testName[0] == '\0') {
124  SDLTest_LogError("Invalid testName string.");
125  return -1;
126  }
127 
128  if (iteration <= 0) {
129  SDLTest_LogError("Invalid iteration count.");
130  return -1;
131  }
132 
133  /* Convert iteration number into a string */
134  SDL_memset(iterationString, 0, sizeof(iterationString));
135  SDL_snprintf(iterationString, sizeof(iterationString) - 1, "%d", iteration);
136 
137  /* Combine the parameters into single string */
138  runSeedLength = SDL_strlen(runSeed);
139  suiteNameLength = SDL_strlen(suiteName);
140  testNameLength = SDL_strlen(testName);
141  iterationStringLength = SDL_strlen(iterationString);
142  entireStringLength = runSeedLength + suiteNameLength + testNameLength + iterationStringLength + 1;
143  buffer = (char *)SDL_malloc(entireStringLength);
144  if (buffer == NULL) {
145  SDLTest_LogError("Failed to allocate buffer for execKey generation.");
147  return 0;
148  }
149  SDL_snprintf(buffer, entireStringLength, "%s%s%s%d", runSeed, suiteName, testName, iteration);
150 
151  /* Hash string and use half of the digest as 64bit exec key */
152  SDLTest_Md5Init(&md5Context);
153  SDLTest_Md5Update(&md5Context, (unsigned char *)buffer, (unsigned int) entireStringLength);
154  SDLTest_Md5Final(&md5Context);
155  SDL_free(buffer);
156  keys = (Uint64 *)md5Context.digest;
157 
158  return keys[0];
159 }

References SDLTest_Md5Context::digest, iteration(), NULL, SDL_ENOMEM, SDL_Error, SDL_free, SDL_malloc, SDL_memset, SDL_snprintf, SDL_strlen, SDLTest_LogError(), SDLTest_Md5Final(), SDLTest_Md5Init(), and SDLTest_Md5Update().

Referenced by SDLTest_RunSuites().

◆ SDLTest_GenerateRunSeed()

char* SDLTest_GenerateRunSeed ( const int  length)

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Generates a random run seed string for the harness. The generated seed will contain alphanumeric characters (0-9A-Z).

Note: The returned string needs to be deallocated by the caller.

Parameters
lengthThe length of the seed string to generate
Returns
The generated seed string

Definition at line 54 of file SDL_test_harness.c.

55 {
56  char *seed = NULL;
57  SDLTest_RandomContext randomContext;
58  int counter;
59 
60  /* Sanity check input */
61  if (length <= 0) {
62  SDLTest_LogError("The length of the harness seed must be >0.");
63  return NULL;
64  }
65 
66  /* Allocate output buffer */
67  seed = (char *)SDL_malloc((length + 1) * sizeof(char));
68  if (seed == NULL) {
69  SDLTest_LogError("SDL_malloc for run seed output buffer failed.");
71  return NULL;
72  }
73 
74  /* Generate a random string of alphanumeric characters */
75  SDLTest_RandomInitTime(&randomContext);
76  for (counter = 0; counter < length; counter++) {
77  unsigned int number = SDLTest_Random(&randomContext);
78  char ch = (char) (number % (91 - 48)) + 48;
79  if (ch >= 58 && ch <= 64) {
80  ch = 65;
81  }
82  seed[counter] = ch;
83  }
84  seed[length] = '\0';
85 
86  return seed;
87 }

References NULL, SDL_ENOMEM, SDL_Error, SDL_malloc, SDLTest_LogError(), SDLTest_Random(), and SDLTest_RandomInitTime().

Referenced by sdltest_generateRunSeed(), and SDLTest_RunSuites().

◆ SDLTest_RunSuites()

int SDLTest_RunSuites ( SDLTest_TestSuiteReference testSuites[],
const char *  userRunSeed,
Uint64  userExecKey,
const char *  filter,
int  testIterations 
)

Execute a test suite using the given run seed and execution key.

The filter string is matched to the suite name (full comparison) to select a single suite, or if no suite matches, it is matched to the test names (full comparison) to select a single test.

Parameters
testSuitesSuites containing the test case.
userRunSeedCustom run seed provided by user, or NULL to autogenerate one.
userExecKeyCustom execution key provided by user, or 0 to autogenerate one.
filterFilter specification. NULL disables. Case sensitive.
testIterationsNumber of iterations to run each test case.
Returns
Test run result; 0 when all tests passed, 1 if any tests failed.

Definition at line 370 of file SDL_test_harness.c.

371 {
372  int totalNumberOfTests = 0;
373  int failedNumberOfTests = 0;
374  int suiteCounter;
375  int testCounter;
376  int iterationCounter;
377  SDLTest_TestSuiteReference *testSuite;
378  const SDLTest_TestCaseReference *testCase;
379  const char *runSeed = NULL;
380  char *currentSuiteName;
381  char *currentTestName;
382  Uint64 execKey;
383  float runStartSeconds;
384  float suiteStartSeconds;
385  float testStartSeconds;
386  float runEndSeconds;
387  float suiteEndSeconds;
388  float testEndSeconds;
389  float runtime;
390  int suiteFilter = 0;
391  char *suiteFilterName = NULL;
392  int testFilter = 0;
393  char *testFilterName = NULL;
394  SDL_bool forceTestRun = SDL_FALSE;
395  int testResult = 0;
396  int runResult = 0;
397  Uint32 totalTestFailedCount = 0;
398  Uint32 totalTestPassedCount = 0;
399  Uint32 totalTestSkippedCount = 0;
400  Uint32 testFailedCount = 0;
401  Uint32 testPassedCount = 0;
402  Uint32 testSkippedCount = 0;
403  Uint32 countSum = 0;
404  const SDLTest_TestCaseReference **failedTests;
405 
406  /* Sanitize test iterations */
407  if (testIterations < 1) {
408  testIterations = 1;
409  }
410 
411  /* Generate run see if we don't have one already */
412  if (userRunSeed == NULL || userRunSeed[0] == '\0') {
413  runSeed = SDLTest_GenerateRunSeed(16);
414  if (runSeed == NULL) {
415  SDLTest_LogError("Generating a random seed failed");
416  return 2;
417  }
418  } else {
419  runSeed = userRunSeed;
420  }
421 
422 
423  /* Reset per-run counters */
424  totalTestFailedCount = 0;
425  totalTestPassedCount = 0;
426  totalTestSkippedCount = 0;
427 
428  /* Take time - run start */
429  runStartSeconds = GetClock();
430 
431  /* Log run with fuzzer parameters */
432  SDLTest_Log("::::: Test Run /w seed '%s' started\n", runSeed);
433 
434  /* Count the total number of tests */
435  suiteCounter = 0;
436  while (testSuites[suiteCounter]) {
437  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
438  suiteCounter++;
439  testCounter = 0;
440  while (testSuite->testCases[testCounter])
441  {
442  testCounter++;
443  totalNumberOfTests++;
444  }
445  }
446 
447  /* Pre-allocate an array for tracking failed tests (potentially all test cases) */
448  failedTests = (const SDLTest_TestCaseReference **)SDL_malloc(totalNumberOfTests * sizeof(SDLTest_TestCaseReference *));
449  if (failedTests == NULL) {
450  SDLTest_LogError("Unable to allocate cache for failed tests");
452  return -1;
453  }
454 
455  /* Initialize filtering */
456  if (filter != NULL && filter[0] != '\0') {
457  /* Loop over all suites to check if we have a filter match */
458  suiteCounter = 0;
459  while (testSuites[suiteCounter] && suiteFilter == 0) {
460  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
461  suiteCounter++;
462  if (testSuite->name != NULL && SDL_strcmp(filter, testSuite->name) == 0) {
463  /* Matched a suite name */
464  suiteFilter = 1;
465  suiteFilterName = testSuite->name;
466  SDLTest_Log("Filtering: running only suite '%s'", suiteFilterName);
467  break;
468  }
469 
470  /* Within each suite, loop over all test cases to check if we have a filter match */
471  testCounter = 0;
472  while (testSuite->testCases[testCounter] && testFilter == 0)
473  {
474  testCase = testSuite->testCases[testCounter];
475  testCounter++;
476  if (testCase->name != NULL && SDL_strcmp(filter, testCase->name) == 0) {
477  /* Matched a test name */
478  suiteFilter = 1;
479  suiteFilterName = testSuite->name;
480  testFilter = 1;
481  testFilterName = testCase->name;
482  SDLTest_Log("Filtering: running only test '%s' in suite '%s'", testFilterName, suiteFilterName);
483  break;
484  }
485  }
486  }
487 
488  if (suiteFilter == 0 && testFilter == 0) {
489  SDLTest_LogError("Filter '%s' did not match any test suite/case.", filter);
490  SDLTest_Log("Exit code: 2");
491  SDL_free((void *) failedTests);
492  return 2;
493  }
494  }
495 
496  /* Loop over all suites */
497  suiteCounter = 0;
498  while(testSuites[suiteCounter]) {
499  testSuite=(SDLTest_TestSuiteReference *)testSuites[suiteCounter];
500  currentSuiteName = (char *)((testSuite->name) ? testSuite->name : SDLTEST_INVALID_NAME_FORMAT);
501  suiteCounter++;
502 
503  /* Filter suite if flag set and we have a name */
504  if (suiteFilter == 1 && suiteFilterName != NULL && testSuite->name != NULL &&
505  SDL_strcmp(suiteFilterName, testSuite->name) != 0) {
506  /* Skip suite */
507  SDLTest_Log("===== Test Suite %i: '%s' skipped\n",
508  suiteCounter,
509  currentSuiteName);
510  } else {
511 
512  /* Reset per-suite counters */
513  testFailedCount = 0;
514  testPassedCount = 0;
515  testSkippedCount = 0;
516 
517  /* Take time - suite start */
518  suiteStartSeconds = GetClock();
519 
520  /* Log suite started */
521  SDLTest_Log("===== Test Suite %i: '%s' started\n",
522  suiteCounter,
523  currentSuiteName);
524 
525  /* Loop over all test cases */
526  testCounter = 0;
527  while(testSuite->testCases[testCounter])
528  {
529  testCase = testSuite->testCases[testCounter];
530  currentTestName = (char *)((testCase->name) ? testCase->name : SDLTEST_INVALID_NAME_FORMAT);
531  testCounter++;
532 
533  /* Filter tests if flag set and we have a name */
534  if (testFilter == 1 && testFilterName != NULL && testCase->name != NULL &&
535  SDL_strcmp(testFilterName, testCase->name) != 0) {
536  /* Skip test */
537  SDLTest_Log("===== Test Case %i.%i: '%s' skipped\n",
538  suiteCounter,
539  testCounter,
540  currentTestName);
541  } else {
542  /* Override 'disabled' flag if we specified a test filter (i.e. force run for debugging) */
543  if (testFilter == 1 && !testCase->enabled) {
544  SDLTest_Log("Force run of disabled test since test filter was set");
545  forceTestRun = SDL_TRUE;
546  }
547 
548  /* Take time - test start */
549  testStartSeconds = GetClock();
550 
551  /* Log test started */
552  SDLTest_Log("----- Test Case %i.%i: '%s' started",
553  suiteCounter,
554  testCounter,
555  currentTestName);
556  if (testCase->description != NULL && testCase->description[0] != '\0') {
557  SDLTest_Log("Test Description: '%s'",
558  (testCase->description) ? testCase->description : SDLTEST_INVALID_NAME_FORMAT);
559  }
560 
561  /* Loop over all iterations */
562  iterationCounter = 0;
563  while(iterationCounter < testIterations)
564  {
565  iterationCounter++;
566 
567  if (userExecKey != 0) {
568  execKey = userExecKey;
569  } else {
570  execKey = SDLTest_GenerateExecKey(runSeed, testSuite->name, testCase->name, iterationCounter);
571  }
572 
573  SDLTest_Log("Test Iteration %i: execKey %" SDL_PRIu64, iterationCounter, execKey);
574  testResult = SDLTest_RunTest(testSuite, testCase, execKey, forceTestRun);
575 
576  if (testResult == TEST_RESULT_PASSED) {
577  testPassedCount++;
578  totalTestPassedCount++;
579  } else if (testResult == TEST_RESULT_SKIPPED) {
580  testSkippedCount++;
581  totalTestSkippedCount++;
582  } else {
583  testFailedCount++;
584  totalTestFailedCount++;
585  }
586  }
587 
588  /* Take time - test end */
589  testEndSeconds = GetClock();
590  runtime = testEndSeconds - testStartSeconds;
591  if (runtime < 0.0f) runtime = 0.0f;
592 
593  if (testIterations > 1) {
594  /* Log test runtime */
595  SDLTest_Log("Runtime of %i iterations: %.1f sec", testIterations, runtime);
596  SDLTest_Log("Average Test runtime: %.5f sec", runtime / (float)testIterations);
597  } else {
598  /* Log test runtime */
599  SDLTest_Log("Total Test runtime: %.1f sec", runtime);
600  }
601 
602  /* Log final test result */
603  switch (testResult) {
604  case TEST_RESULT_PASSED:
605  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Passed");
606  break;
607  case TEST_RESULT_FAILED:
608  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", currentTestName, "Failed");
609  break;
611  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT,"Test", currentTestName, "No Asserts");
612  break;
613  }
614 
615  /* Collect failed test case references for repro-step display */
616  if (testResult == TEST_RESULT_FAILED) {
617  failedTests[failedNumberOfTests] = testCase;
618  failedNumberOfTests++;
619  }
620  }
621  }
622 
623  /* Take time - suite end */
624  suiteEndSeconds = GetClock();
625  runtime = suiteEndSeconds - suiteStartSeconds;
626  if (runtime < 0.0f) runtime = 0.0f;
627 
628  /* Log suite runtime */
629  SDLTest_Log("Total Suite runtime: %.1f sec", runtime);
630 
631  /* Log summary and final Suite result */
632  countSum = testPassedCount + testFailedCount + testSkippedCount;
633  if (testFailedCount == 0)
634  {
635  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
636  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Passed");
637  }
638  else
639  {
640  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Suite", countSum, testPassedCount, testFailedCount, testSkippedCount);
641  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite", currentSuiteName, "Failed");
642  }
643 
644  }
645  }
646 
647  /* Take time - run end */
648  runEndSeconds = GetClock();
649  runtime = runEndSeconds - runStartSeconds;
650  if (runtime < 0.0f) runtime = 0.0f;
651 
652  /* Log total runtime */
653  SDLTest_Log("Total Run runtime: %.1f sec", runtime);
654 
655  /* Log summary and final run result */
656  countSum = totalTestPassedCount + totalTestFailedCount + totalTestSkippedCount;
657  if (totalTestFailedCount == 0)
658  {
659  runResult = 0;
660  SDLTest_Log(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
661  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Passed");
662  }
663  else
664  {
665  runResult = 1;
666  SDLTest_LogError(SDLTEST_LOG_SUMMARY_FORMAT, "Run", countSum, totalTestPassedCount, totalTestFailedCount, totalTestSkippedCount);
667  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Run /w seed", runSeed, "Failed");
668  }
669 
670  /* Print repro steps for failed tests */
671  if (failedNumberOfTests > 0) {
672  SDLTest_Log("Harness input to repro failures:");
673  for (testCounter = 0; testCounter < failedNumberOfTests; testCounter++) {
674  SDLTest_Log(" --seed %s --filter %s", runSeed, failedTests[testCounter]->name);
675  }
676  }
677  SDL_free((void *) failedTests);
678 
679  SDLTest_Log("Exit code: %d", runResult);
680  return runResult;
681 }

References SDLTest_TestCaseReference::description, SDLTest_TestCaseReference::enabled, GetClock(), SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_ENOMEM, SDL_Error, SDL_FALSE, SDL_free, SDL_malloc, SDL_PRIu64, SDL_strcmp, SDL_TRUE, SDLTEST_FINAL_RESULT_FORMAT, SDLTest_GenerateExecKey(), SDLTest_GenerateRunSeed(), SDLTEST_INVALID_NAME_FORMAT, SDLTest_Log(), SDLTEST_LOG_SUMMARY_FORMAT, SDLTest_LogError(), SDLTest_RunTest(), TEST_RESULT_FAILED, TEST_RESULT_NO_ASSERT, TEST_RESULT_PASSED, TEST_RESULT_SKIPPED, SDLTest_TestSuiteReference::testCases, and testSuites.

Referenced by main().

◆ SDLTest_RunTest()

static int SDLTest_RunTest ( SDLTest_TestSuiteReference testSuite,
const SDLTest_TestCaseReference testCase,
Uint64  execKey,
SDL_bool  forceTestRun 
)
static

Execute a test using the given execution key.

Parameters
testSuiteSuite containing the test case.
testCaseCase to execute.
execKeyExecution key for the fuzzer.
forceTestRunForce test to run even if test was disabled in suite.
Returns
Test case result.

Definition at line 230 of file SDL_test_harness.c.

231 {
232  SDL_TimerID timer = 0;
233  int testCaseResult = 0;
234  int testResult = 0;
235  int fuzzerCount;
236 
237  if (testSuite==NULL || testCase==NULL || testSuite->name==NULL || testCase->name==NULL)
238  {
239  SDLTest_LogError("Setup failure: testSuite or testCase references NULL");
241  }
242 
243  if (!testCase->enabled && forceTestRun == SDL_FALSE)
244  {
245  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Disabled)");
246  return TEST_RESULT_SKIPPED;
247  }
248 
249  /* Initialize fuzzer */
250  SDLTest_FuzzerInit(execKey);
251 
252  /* Reset assert tracker */
254 
255  /* Set timeout timer */
257 
258  /* Maybe run suite initalizer function */
259  if (testSuite->testSetUp) {
260  testSuite->testSetUp(0x0);
262  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Suite Setup", testSuite->name, "Failed");
264  }
265  }
266 
267  /* Run test case function */
268  testCaseResult = testCase->testCase(0x0);
269 
270  /* Convert test execution result into harness result */
271  if (testCaseResult == TEST_SKIPPED) {
272  /* Test was programatically skipped */
273  testResult = TEST_RESULT_SKIPPED;
274  } else if (testCaseResult == TEST_STARTED) {
275  /* Test did not return a TEST_COMPLETED value; assume it failed */
276  testResult = TEST_RESULT_FAILED;
277  } else if (testCaseResult == TEST_ABORTED) {
278  /* Test was aborted early; assume it failed */
279  testResult = TEST_RESULT_FAILED;
280  } else {
281  /* Perform failure analysis based on asserts */
282  testResult = SDLTest_AssertSummaryToTestResult();
283  }
284 
285  /* Maybe run suite cleanup function (ignore failed asserts) */
286  if (testSuite->testTearDown) {
287  testSuite->testTearDown(0x0);
288  }
289 
290  /* Cancel timeout timer */
291  if (timer) {
292  SDL_RemoveTimer(timer);
293  }
294 
295  /* Report on asserts and fuzzer usage */
296  fuzzerCount = SDLTest_GetFuzzerInvocationCount();
297  if (fuzzerCount > 0) {
298  SDLTest_Log("Fuzzer invocations: %d", fuzzerCount);
299  }
300 
301  /* Final log based on test execution result */
302  if (testCaseResult == TEST_SKIPPED) {
303  /* Test was programatically skipped */
304  SDLTest_Log(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Skipped (Programmatically)");
305  } else if (testCaseResult == TEST_STARTED) {
306  /* Test did not return a TEST_COMPLETED value; assume it failed */
307  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (test started, but did not return TEST_COMPLETED)");
308  } else if (testCaseResult == TEST_ABORTED) {
309  /* Test was aborted early; assume it failed */
310  SDLTest_LogError(SDLTEST_FINAL_RESULT_FORMAT, "Test", testCase->name, "Failed (Aborted)");
311  } else {
313  }
314 
315  return testResult;
316 }

References SDLTest_TestCaseReference::enabled, SDLTest_TestCaseReference::name, SDLTest_TestSuiteReference::name, NULL, SDL_FALSE, SDL_RemoveTimer, SDLTest_AssertSummaryToTestResult(), SDLTest_BailOut(), SDLTEST_FINAL_RESULT_FORMAT, SDLTest_FuzzerInit(), SDLTest_GetFuzzerInvocationCount(), SDLTest_Log(), SDLTest_LogAssertSummary(), SDLTest_LogError(), SDLTest_ResetAssertSummary(), SDLTest_SetTestTimeout(), SDLTest_TestCaseTimeout, TEST_ABORTED, TEST_RESULT_FAILED, TEST_RESULT_SETUP_FAILURE, TEST_RESULT_SKIPPED, TEST_SKIPPED, TEST_STARTED, SDLTest_TestCaseReference::testCase, SDLTest_TestSuiteReference::testSetUp, and SDLTest_TestSuiteReference::testTearDown.

Referenced by SDLTest_RunSuites().

◆ SDLTest_SetTestTimeout()

static SDL_TimerID SDLTest_SetTestTimeout ( int  timeout,
void(*)()  callback 
)
static

Set timeout handler for test.

Note: SDL_Init(SDL_INIT_TIMER) will be called if it wasn't done so before.

Parameters
timeoutTimeout interval in seconds.
callbackFunction that will be called after timeout has elapsed.
Returns
Timer id or -1 on failure.

Definition at line 172 of file SDL_test_harness.c.

173 {
174  Uint32 timeoutInMilliseconds;
175  SDL_TimerID timerID;
176 
177  if (callback == NULL) {
178  SDLTest_LogError("Timeout callback can't be NULL");
179  return -1;
180  }
181 
182  if (timeout < 0) {
183  SDLTest_LogError("Timeout value must be bigger than zero.");
184  return -1;
185  }
186 
187  /* Init SDL timer if not initialized before */
188  if (SDL_WasInit(SDL_INIT_TIMER) == 0) {
190  SDLTest_LogError("Failed to init timer subsystem: %s", SDL_GetError());
191  return -1;
192  }
193  }
194 
195  /* Set timer */
196  timeoutInMilliseconds = timeout * 1000;
197  timerID = SDL_AddTimer(timeoutInMilliseconds, (SDL_TimerCallback)callback, 0x0);
198  if (timerID == 0) {
199  SDLTest_LogError("Creation of SDL timer failed: %s", SDL_GetError());
200  return -1;
201  }
202 
203  return timerID;
204 }

References callback(), NULL, SDL_AddTimer, SDL_GetError, SDL_INIT_TIMER, SDL_InitSubSystem, SDL_WasInit, and SDLTest_LogError().

Referenced by SDLTest_RunTest().

Variable Documentation

◆ SDLTest_TestCaseTimeout

Uint32 SDLTest_TestCaseTimeout = 3600
static

Definition at line 41 of file SDL_test_harness.c.

Referenced by SDLTest_RunTest().

TEST_RESULT_FAILED
#define TEST_RESULT_FAILED
Definition: SDL_test_harness.h:58
TEST_STARTED
#define TEST_STARTED
Definition: SDL_test_harness.h:52
SDLTest_TestCaseTimeout
static Uint32 SDLTest_TestCaseTimeout
Definition: SDL_test_harness.c:41
SDL_memset
#define SDL_memset
Definition: SDL_dynapi_overrides.h:386
SDL_GetError
#define SDL_GetError
Definition: SDL_dynapi_overrides.h:113
Uint32
uint32_t Uint32
Definition: SDL_stdinc.h:203
SDLTest_TestSuiteReference::name
char * name
Definition: SDL_test_harness.h:91
SDLTest_TestCaseReference::testCase
SDLTest_TestCaseFp testCase
Definition: SDL_test_harness.h:77
SDLTest_Md5Context
Definition: SDL_test_md5.h:71
SDLTest_RunTest
static int SDLTest_RunTest(SDLTest_TestSuiteReference *testSuite, const SDLTest_TestCaseReference *testCase, Uint64 execKey, SDL_bool forceTestRun)
Execute a test using the given execution key.
Definition: SDL_test_harness.c:230
NULL
#define NULL
Definition: begin_code.h:167
timeout
GLbitfield GLuint64 timeout
Definition: SDL_opengl_glext.h:1483
SDLTest_Md5Final
void SDLTest_Md5Final(SDLTest_Md5Context *mdContext)
complete digest computation
Definition: SDL_test_md5.c:180
TEST_RESULT_SETUP_FAILURE
#define TEST_RESULT_SETUP_FAILURE
Definition: SDL_test_harness.h:61
SDLTest_Md5Update
void SDLTest_Md5Update(SDLTest_Md5Context *mdContext, unsigned char *inBuf, unsigned int inLen)
update digest from variable length data
Definition: SDL_test_md5.c:131
SDLTest_Log
void SDLTest_Log(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and INFO priority.
Definition: SDL_test_log.c:85
SDL_ENOMEM
@ SDL_ENOMEM
Definition: SDL_error.h:57
SDL_TimerID
int SDL_TimerID
Definition: SDL_timer.h:86
x0
GLuint GLfloat x0
Definition: SDL_opengl_glext.h:8583
SDLTest_TestCaseReference::description
char * description
Definition: SDL_test_harness.h:81
GetClock
static float GetClock()
Definition: SDL_test_harness.c:350
SDL_PRIu64
#define SDL_PRIu64
Definition: SDL_stdinc.h:238
callback
static Uint32 callback(Uint32 interval, void *param)
Definition: testtimer.c:34
SDLTEST_FINAL_RESULT_FORMAT
#define SDLTEST_FINAL_RESULT_FORMAT
Definition: SDL_test_harness.c:38
SDLTest_RandomContext
Definition: SDL_test_random.h:59
SDLTest_SetTestTimeout
static SDL_TimerID SDLTest_SetTestTimeout(int timeout, void(*callback)())
Set timeout handler for test.
Definition: SDL_test_harness.c:172
SDL_InitSubSystem
#define SDL_InitSubSystem
Definition: SDL_dynapi_overrides.h:55
length
GLuint GLsizei GLsizei * length
Definition: SDL_opengl_glext.h:669
TEST_RESULT_PASSED
#define TEST_RESULT_PASSED
Definition: SDL_test_harness.h:57
SDLTest_GenerateRunSeed
char * SDLTest_GenerateRunSeed(const int length)
Generates a random run seed string for the harness. The generated seed will contain alphanumeric char...
Definition: SDL_test_harness.c:54
filter
GLint GLint GLint GLint GLint GLint GLint GLbitfield GLenum filter
Definition: SDL_opengl_glext.h:1184
iteration
static void iteration()
Definition: testaudiohotplug.c:84
SDLTEST_LOG_SUMMARY_FORMAT
#define SDLTEST_LOG_SUMMARY_FORMAT
Definition: SDL_test_harness.c:35
SDL_AddTimer
#define SDL_AddTimer
Definition: SDL_dynapi_overrides.h:487
SDL_Error
#define SDL_Error
Definition: SDL_dynapi_overrides.h:115
SDL_FALSE
@ SDL_FALSE
Definition: SDL_stdinc.h:163
buffer
GLuint buffer
Definition: SDL_opengl_glext.h:533
counter
GLuint counter
Definition: SDL_opengl_glext.h:4992
testSuites
SDLTest_TestSuiteReference * testSuites[]
Definition: testautomation_suites.h:32
SDLTest_TestSuiteReference::testTearDown
SDLTest_TestCaseTearDownFp testTearDown
Definition: SDL_test_harness.h:97
SDL_free
#define SDL_free
Definition: SDL_dynapi_overrides.h:377
f
GLfloat f
Definition: SDL_opengl_glext.h:1870
TEST_ABORTED
#define TEST_ABORTED
Definition: SDL_test_harness.h:51
SDLTest_Md5Context::digest
unsigned char digest[16]
Definition: SDL_test_md5.h:75
name
GLuint const GLchar * name
Definition: SDL_opengl_glext.h:660
SDL_TimerCallback
Uint32(* SDL_TimerCallback)(Uint32 interval, void *param)
Definition: SDL_timer.h:81
SDLTest_TestCaseReference
Definition: SDL_test_harness.h:75
SDLTest_TestSuiteReference::testSetUp
SDLTest_TestCaseSetUpFp testSetUp
Definition: SDL_test_harness.h:93
SDLTest_Md5Init
void SDLTest_Md5Init(SDLTest_Md5Context *mdContext)
initialize the context
Definition: SDL_test_md5.c:110
SDLTest_ResetAssertSummary
void SDLTest_ResetAssertSummary(void)
Resets the assert summary counters to zero.
Definition: SDL_test_assert.c:113
TEST_RESULT_SKIPPED
#define TEST_RESULT_SKIPPED
Definition: SDL_test_harness.h:60
SDLTest_TestSuiteReference::testCases
const SDLTest_TestCaseReference ** testCases
Definition: SDL_test_harness.h:95
TEST_SKIPPED
#define TEST_SKIPPED
Definition: SDL_test_harness.h:54
SDLTest_LogAssertSummary
void SDLTest_LogAssertSummary(void)
Logs summary of all assertions (total, pass, fail) since last reset as INFO or ERROR.
Definition: SDL_test_assert.c:123
SDLTest_GenerateExecKey
static Uint64 SDLTest_GenerateExecKey(const char *runSeed, char *suiteName, char *testName, int iteration)
Definition: SDL_test_harness.c:101
SDLTest_BailOut
static SDL_NORETURN void SDLTest_BailOut()
Timeout handler. Aborts test run and exits harness process.
Definition: SDL_test_harness.c:213
SDL_TRUE
@ SDL_TRUE
Definition: SDL_stdinc.h:164
SDLTest_RandomInitTime
void SDLTest_RandomInitTime(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
Definition: SDL_test_random.c:65
SDL_INIT_TIMER
#define SDL_INIT_TIMER
Definition: SDL.h:77
SDL_snprintf
#define SDL_snprintf
Definition: SDL_dynapi_overrides.h:40
Uint64
uint64_t Uint64
Definition: SDL_stdinc.h:216
SDLTest_LogError
void SDLTest_LogError(SDL_PRINTF_FORMAT_STRING const char *fmt,...) SDL_PRINTF_VARARG_FUNC(1)
Prints given message with a timestamp in the TEST category and the ERROR priority.
Definition: SDL_test_log.c:103
TEST_RESULT_NO_ASSERT
#define TEST_RESULT_NO_ASSERT
Definition: SDL_test_harness.h:59
SDL_strlen
#define SDL_strlen
Definition: SDL_dynapi_overrides.h:393
SDLTest_AssertSummaryToTestResult
int SDLTest_AssertSummaryToTestResult(void)
Converts the current assert summary state to a test result.
Definition: SDL_test_assert.c:139
SDL_malloc
#define SDL_malloc
Definition: SDL_dynapi_overrides.h:374
SDLTest_TestCaseReference::name
char * name
Definition: SDL_test_harness.h:79
SDLTest_FuzzerInit
void SDLTest_FuzzerInit(Uint64 execKey)
Definition: SDL_test_fuzzer.c:63
SDL_strcmp
#define SDL_strcmp
Definition: SDL_dynapi_overrides.h:417
SDLTest_GetFuzzerInvocationCount
int SDLTest_GetFuzzerInvocationCount(void)
Definition: SDL_test_fuzzer.c:73
SDL_WasInit
#define SDL_WasInit
Definition: SDL_dynapi_overrides.h:57
SDLTEST_INVALID_NAME_FORMAT
#define SDLTEST_INVALID_NAME_FORMAT
Definition: SDL_test_harness.c:32
SDLTest_TestSuiteReference
Definition: SDL_test_harness.h:89
SDLTest_Random
unsigned int SDLTest_Random(SDLTest_RandomContext *rndContext)
Initialize random number generator based on current system time.
Definition: SDL_test_random.c:80
SDL_RemoveTimer
#define SDL_RemoveTimer
Definition: SDL_dynapi_overrides.h:488
SDL_bool
SDL_bool
Definition: SDL_stdinc.h:161
SDLTest_TestCaseReference::enabled
int enabled
Definition: SDL_test_harness.h:83