Support arbitrary tagging and selection of testcases.

A testcase can optionally have a list of tags associated with it.
Srunner can be run with an optional include list of tags and an optional
exclude list of tags. These will have the effect of filtering testcases
that would otherwise be run.

Author: Crispin Dent-Young

doc/check.texi

@@ -974,6 +974,7 @@ easier for the developer to write, run, and analyze tests.
 * Test Fixtures::               
 * Multiple Suites in one SRunner::  
 * Selective Running of Tests::
+* Selecting Tests Based on Arbitrary Tags::
 * Testing Signal Handling and Exit Values::  
 * Looping Tests::               
 * Test Timeouts::               
@@ -1301,11 +1302,12 @@ srunner_add_suite (sr, make_pack_suite ());
 @end verbatim
 @end example
 
-@node Selective Running of Tests, Testing Signal Handling and Exit Values, Multiple Suites in one SRunner, Advanced Features
+@node Selective Running of Tests, Selecting Tests Based on Arbitrary Tags, Multiple Suites in one SRunner, Advanced Features
 @section Selective Running of Tests
 
 @vindex CK_RUN_SUITE
 @vindex CK_RUN_CASE
+
 After adding a couple of suites and some test cases in each, it is
 sometimes practical to be able to run only one suite, or one
 specific test case, without recompiling the test code. There are
@@ -1314,8 +1316,70 @@ two environment variables available that offers this ability,
 the name of the suite and/or test case you want to run. These
 environment variables can also be a good integration tool for
 running specific tests from within another tool, e.g. an IDE.
-  
-@node Testing Signal Handling and Exit Values, Looping Tests, Selective Running of Tests, Advanced Features
+
+@node Selecting Tests Based on Arbitrary Tags, Testing Signal Handling and Exit Values, Selective Running of Tests, Advanced Features
+@section Selecting Tests Based on Arbitrary Tags
+
+@vindex CK_INCLUDE_TAGS
+@vindex CK_EXCLUDE_TAGS
+
+It can also be useful to be able to dynamically select or exclude
+groups of tests to be run based on criteria other than the suite or
+testcase name. For example certain test cases can be tagged as
+requiring a long run time and so quick sanity tests can be run that
+exclude all test cases with such a tag. Alternately tags may be used
+where tests cover multiple functional areas in order to indicate which
+areas a test case covers. Tests can then be run that include all test
+cases for a given set of areas.
+
+A tag is a string of characters excluding spaces.
+
+Tags are applied to a testcase by passing a space separated list of
+tags to @code{tcase_set_tags} once it has been created. For example in
+function @code{make_tagged_suite()} in @file{check_check_tags.c} - 
+
+@example
+@verbatim
+    Suite *s;
+
+    TCase *red, *blue, *purple, *yellow, *black;
+
+    s = suite_create("Check Tag Filtering");
+
+    red = tcase_create("Red");
+    tcase_set_tags(red, "Red");
+    suite_add_tcase (s, red);
+    tcase_add_test(red, red_test1);
+
+    blue = tcase_create("Blue");
+    tcase_set_tags(blue, "Blue");
+    suite_add_tcase (s, blue);
+    tcase_add_test(blue, blue_test1);
+
+    purple = tcase_create("Purple");
+    tcase_set_tags(purple, "Red Blue");
+    suite_add_tcase (s, purple);
+    tcase_add_test(purple, purple_test1);
+
+@end verbatim
+@end example
+
+Once test cases are tagged then there are two environment variables
+available for selecting testcases based on these tags
+@code{CK_INCLUDE_TAGS} and @code{CK_EXCLUDE_TAGS}. These can be set to
+a space separated list of tag names. If @code{CK_INCLUDE_TAGS} is set
+then only testcases which include at least one tag in common with
+@code{CK_INCLUDE_TAGS} will be run. If @code{CK_EXCLUDE_TAGS} is set
+then testcases with one tag in common with @code{CK_EXCLUDE_TAGS} will
+not be run. In cases where both @code{CK_INCLUDE_TAGS} and
+@code{CK_EXCLUDE_TAGS} match then the test will be excluded.
+
+Both @code{CK_INCLUDE_TAGS} and @code{CK_EXCLUDE_TAGS} can be
+specified in conjunction with @code{CK_RUN_SUITE} or even
+@code{CK_RUN_CASE} in which case they will have the effect of further
+narrowing the selection.
+
+@node Testing Signal Handling and Exit Values, Looping Tests, Selecting Tests Based on Arbitrary Tags, Advanced Features
 @section Testing Signal Handling and Exit Values
 
 @findex tcase_add_test_raise_signal
@@ -1975,6 +2039,10 @@ CK_RUN_CASE: Name of a test case, runs only that test. See section @ref{Selectiv
 
 CK_RUN_SUITE: Name of a test suite, runs only that suite. See section @ref{Selective Running of Tests}.
 
+CK_INCLUDE_TAGS: String of space separated tags, runs only test cases associated with at least one of the tags, See section @ref{Selecting Tests Based on Arbitrary Tags}.
+
+CK_EXCLUDE_TAGS: String of space separated tags, runs only test cases not associated with any of the tags, See section @ref{Selecting Tests Based on Arbitrary Tags}.
+ 
 CK_VERBOSITY: How much output to emit, accepts: ``silent'', ``minimal'', ``normal'', ``subunit'', or ``verbose''.  See section @ref{SRunner Output}.
 
 CK_FORK: Set to ``no'' to disable using fork() to run unit tests in their own process. This is useful for debugging segmentation faults.  See section @ref{No Fork Mode}.

src/check.c

@@ -103,6 +103,7 @@ static void suite_free(Suite * s)
     free(s);
 }
 
+
 TCase *tcase_create(const char *name)
 {
     char *env;
@@ -149,10 +150,49 @@ TCase *tcase_create(const char *name)
     tc->ch_sflst = check_list_create();
     tc->unch_tflst = check_list_create();
     tc->ch_tflst = check_list_create();
+    tc->tags = check_list_create();
 
     return tc;
 }
 
+/*
+ * Helper function to create a list of tags from
+ * a space separated string.
+ */
+List *tag_string_to_list(const char *tags_string)
+{
+    List *list;
+    char *tags;
+    char *tag;
+
+    list = check_list_create();
+
+    if (NULL == tags_string)
+    {
+	return list;
+    }
+
+    tags = strdup(tags_string);
+    tag = strtok(tags, " ");
+    while (tag)
+    {
+	check_list_add_end(list, strdup(tag));
+	tag = strtok(NULL, " ");
+    }
+    free(tags);
+    return list;
+}
+
+void tcase_set_tags(TCase * tc, const char *tags_orig)
+{
+    /* replace any pre-existing list */
+    if (tc->tags)
+    {
+	check_list_apply(tc->tags, free);
+	check_list_free(tc->tags);
+    }
+    tc->tags = tag_string_to_list(tags_orig);
+}
 
 static void tcase_free(TCase * tc)
 {
@@ -161,15 +201,40 @@ static void tcase_free(TCase * tc)
     check_list_apply(tc->ch_sflst, free);
     check_list_apply(tc->unch_tflst, free);
     check_list_apply(tc->ch_tflst, free);
+    check_list_apply(tc->tags, free);
     check_list_free(tc->tflst);
     check_list_free(tc->unch_sflst);
     check_list_free(tc->ch_sflst);
     check_list_free(tc->unch_tflst);
     check_list_free(tc->ch_tflst);
-
+    check_list_free(tc->tags);
     free(tc);
 }
 
+unsigned int tcase_matching_tag(TCase *tc, List *check_for)
+{
+
+    if (NULL == check_for)
+    {
+	return 0;
+    }
+
+    for(check_list_front(check_for); !check_list_at_end(check_for);
+        check_list_advance(check_for))
+    {
+	for(check_list_front(tc->tags); !check_list_at_end(tc->tags);
+	    check_list_advance(tc->tags))
+	{
+	    if (0 == strcmp((const char *)check_list_val(tc->tags),
+			    (const char *)check_list_val(check_for)))
+	    {
+		return 1;
+	    }
+	}
+    }
+    return 0;
+}
+
 void suite_add_tcase(Suite * s, TCase * tc)
 {
     if(s == NULL || tc == NULL || check_list_contains(s->tclst, tc))

src/check.h.in

@@ -172,6 +172,19 @@ CK_DLL_EXP void CK_EXPORT suite_add_tcase(Suite * s, TCase * tc);
  * */
 CK_DLL_EXP TCase *CK_EXPORT tcase_create(const char *name);
 
+/**
+ * Associate a test case with certain tags.
+ * Replaces any existing tags with the new set.
+ *
+ * @param tc the test case
+ *
+ * @param tags string containing arbitrary tags separated by spaces.
+ *        This will be copied. Passing NULL clears all tags.
+ *
+ * @since 0.11.0
+ * */
+CK_DLL_EXP void CK_EXPORT tcase_set_tags(TCase * tc,
+					 const char *tags);
 /**
  * Add a test function to a test case
  *
@@ -986,6 +999,33 @@ CK_DLL_EXP void CK_EXPORT srunner_run(SRunner * sr, const char *sname,
                                       enum print_output print_mode);
 
 
+/**
+ * Run a specific suite or test case or testcases with specific tags
+ * from a suite runner, printing results to stdout as specified by the
+ * print_mode.
+ *
+ * In addition to running any applicable suites or test cases, if the
+ * suite runner has been configured to output to a log, that is also
+ * performed.
+ *
+ * @param sr suite runner where the given suite or test case must be
+ * @param sname suite name to run. A NULL means "any suite".
+ * @param tcname test case name to run. A NULL means "any test case"
+ * @param include_tags space separate list of tags. Only run test cases
+ * that share one of these tags. A NULL means "any test case".
+ * @param exclude_tags space separate list of tags. Only run test cases
+ * that do not share one of these tags. A NULL means "any test case".
+ * Overrides any include criteria.
+ * @param print_mode the verbosity in which to report results to stdout
+ *
+ * @since 0.11.0
+ */
+CK_DLL_EXP void CK_EXPORT srunner_run_tagged(SRunner * sr, const char *sname,
+					     const char *tcname,
+					     const char *include_tags,
+					     const char *exclude_tags,
+					     enum print_output print_mode);
+
 /**
  * Retrieve the number of failed tests executed by a suite runner.
  *

src/check_impl.h

@@ -65,6 +65,7 @@ struct TCase
     List *unch_tflst;
     List *ch_sflst;
     List *ch_tflst;
+    List *tags;
 };
 
 typedef struct TestStats
@@ -134,4 +135,7 @@ enum fork_status cur_fork_status(void);
 
 clockid_t check_get_clockid(void);
 
+unsigned int tcase_matching_tag(TCase *tc, List *check_for);
+List *tag_string_to_list(const char *tags_string);
+
 #endif /* CHECK_IMPL_H */

src/check_run.c

@@ -60,6 +60,8 @@ static void srunner_run_init(SRunner * sr, enum print_output print_mode);
 static void srunner_run_end(SRunner * sr, enum print_output print_mode);
 static void srunner_iterate_suites(SRunner * sr,
                                    const char *sname, const char *tcname,
+				   const char *include_tags,
+				   const char *exclude_tags,
                                    enum print_output print_mode);
 static void srunner_iterate_tcase_tfuns(SRunner * sr, TCase * tc);
 static void srunner_add_failure(SRunner * sr, TestResult * tf);
@@ -160,15 +162,22 @@ static void srunner_run_end(SRunner * sr,
 
 static void srunner_iterate_suites(SRunner * sr,
                                    const char *sname, const char *tcname,
+				   const char *include_tags,
+				   const char *exclude_tags,
                                    enum print_output CK_ATTRIBUTE_UNUSED
                                    print_mode)
 {
+    List *include_tag_lst;
+    List *exclude_tag_lst;
     List *slst;
     List *tcl;
     TCase *tc;
 
     slst = sr->slst;
 
+    include_tag_lst = tag_string_to_list(include_tags);
+    exclude_tag_lst = tag_string_to_list(exclude_tags);
+
     for(check_list_front(slst); !check_list_at_end(slst);
         check_list_advance(slst))
     {
@@ -191,12 +200,31 @@ static void srunner_iterate_suites(SRunner * sr,
             {
                 continue;
             }
+	    if (include_tags != NULL)
+	    {
+		if (!tcase_matching_tag(tc, include_tag_lst))
+		{
+		    continue;
+		}
+	    }
+	    if (exclude_tags != NULL)
+	    {
+		if (tcase_matching_tag(tc, exclude_tag_lst))
+		{
+		    continue;
+		}
+	    }
 
             srunner_run_tcase(sr, tc);
         }
 
         log_suite_end(sr, s);
     }
+
+    check_list_apply(include_tag_lst, free);
+    check_list_apply(exclude_tag_lst, free);
+    check_list_free(include_tag_lst);
+    check_list_free(exclude_tag_lst);
 }
 
 static void srunner_iterate_tcase_tfuns(SRunner * sr, TCase * tc)
@@ -741,8 +769,9 @@ void srunner_run_all(SRunner * sr, enum print_output print_mode)
                 print_mode);
 }
 
-void srunner_run(SRunner * sr, const char *sname, const char *tcname,
-                 enum print_output print_mode)
+void srunner_run_tagged(SRunner * sr, const char *sname, const char *tcname,
+			const char *include_tags, const char *exclude_tags,
+			enum print_output print_mode)
 {
 #if defined(HAVE_SIGACTION) && defined(HAVE_FORK)
     static struct sigaction sigalarm_old_action;
@@ -756,7 +785,11 @@ void srunner_run(SRunner * sr, const char *sname, const char *tcname,
     if(!tcname)
         tcname = getenv("CK_RUN_CASE");
     if(!sname)
-        sname = getenv("CK_RUN_SUITE");
+	sname = getenv("CK_RUN_SUITE");
+    if(!include_tags)
+	include_tags = getenv("CK_INCLUDE_TAGS");
+    if(!exclude_tags)
+	exclude_tags = getenv("CK_EXCLUDE_TAGS");
 
     if(sr == NULL)
         return;
@@ -779,7 +812,8 @@ void srunner_run(SRunner * sr, const char *sname, const char *tcname,
     sigaction(SIGTERM, &sigterm_new_action, &sigterm_old_action);
 #endif /* HAVE_SIGACTION && HAVE_FORK */
     srunner_run_init(sr, print_mode);
-    srunner_iterate_suites(sr, sname, tcname, print_mode);
+    srunner_iterate_suites(sr, sname, tcname, include_tags, exclude_tags,
+			   print_mode);
     srunner_run_end(sr, print_mode);
 #if defined(HAVE_SIGACTION) && defined(HAVE_FORK)
     sigaction(SIGALRM, &sigalarm_old_action, NULL);
@@ -788,6 +822,12 @@ void srunner_run(SRunner * sr, const char *sname, const char *tcname,
 #endif /* HAVE_SIGACTION && HAVE_FORK */
 }
 
+void srunner_run(SRunner * sr, const char *sname, const char *tcname,
+                 enum print_output print_mode)
+{
+    srunner_run_tagged(sr, sname, tcname, NULL, NULL, print_mode);
+}
+
 pid_t check_fork(void)
 {
 #if defined(HAVE_FORK) && HAVE_FORK==1

tests/CMakeLists.txt

@@ -53,6 +53,7 @@ set(CHECK_CHECK_SOURCES
   check_check_pack.c
   check_check_selective.c
   check_check_sub.c
+  check_check_tags.c
   check_list.c)
 set(CHECK_CHECK_HEADERS check_check.h)
 add_executable(check_check ${CHECK_CHECK_HEADERS} ${CHECK_CHECK_SOURCES})