Merge pull request #15565 from LabNConsulting/chopps/code-cover

tests: enable code coverage reporting with topotests

2 files changed, 104 insertions, 34 deletions

diff --git a/tests/topotests/conftest.py b/tests/topotests/conftest.py
index 23eab68db4..c6f038b7f6 100755
--- a/tests/topotests/conftest.py
+++ b/tests/topotests/conftest.py
@@ -68,6 +68,10 @@ def log_handler(basename, logpath):
         topolog.logfinish(basename, logpath)
 
 
+def is_main_runner():
+    return "PYTEST_XDIST_WORKER" not in os.environ
+
+
 def pytest_addoption(parser):
     """
     Add topology-only option to the topology tester. This option makes pytest
@@ -86,6 +90,17 @@ def pytest_addoption(parser):
     )
 
     parser.addoption(
+        "--cov-topotest",
+        action="store_true",
+        help="Enable reporting of coverage",
+    )
+
+    parser.addoption(
+        "--cov-frr-build-dir",
+        help="Dir of coverage-enable build being run, default is the source dir",
+    )
+
+    parser.addoption(
         "--gdb-breakpoints",
         metavar="SYMBOL[,SYMBOL...]",
         help="Comma-separated list of functions to set gdb breakpoints on",
@@ -456,6 +471,37 @@ def pytest_assertrepr_compare(op, left, right):
     return json_result.gen_report()
 
 
+def setup_coverage(config):
+    commander = Commander("pytest")
+    if config.option.cov_frr_build_dir:
+        bdir = Path(config.option.cov_frr_build_dir).resolve()
+        output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
+    else:
+        # Support build sub-directory of main source dir
+        bdir = Path(__file__).resolve().parent.parent.parent
+        output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
+    m = re.match(f"({bdir}.*)/zebra/zebra_nb.gcno", output)
+    if not m:
+        logger.warning(
+            "No coverage data files (*.gcno) found, try specifying --cov-frr-build-dir"
+        )
+        return
+
+    bdir = Path(m.group(1))
+    # Save so we can get later from g_pytest_config
+    rundir = Path(config.option.rundir).resolve()
+    gcdadir = rundir / "gcda"
+    os.environ["FRR_BUILD_DIR"] = str(bdir)
+    os.environ["GCOV_PREFIX_STRIP"] = str(len(bdir.parts) - 1)
+    os.environ["GCOV_PREFIX"] = str(gcdadir)
+
+    if is_main_runner():
+        commander.cmd_raises(f"find {bdir} -name '*.gc??' -exec chmod o+rw {{}} +")
+        commander.cmd_raises(f"mkdir -p {gcdadir}")
+        commander.cmd_raises(f"chown -R root:frr {gcdadir}")
+        commander.cmd_raises(f"chmod 2775 {gcdadir}")
+
+
 def pytest_configure(config):
     """
     Assert that the environment is correctly configured, and get extra config.
@@ -556,8 +602,6 @@ def pytest_configure(config):
     if config.option.topology_only and is_xdist:
         pytest.exit("Cannot use --topology-only with distributed test mode")
 
-        pytest.exit("Cannot use --topology-only with distributed test mode")
-
     # Check environment now that we have config
     if not diagnose_env(rundir):
         pytest.exit("environment has errors, please read the logs in %s" % rundir)
@@ -572,27 +616,25 @@ def pytest_configure(config):
         if "TOPOTESTS_CHECK_STDERR" in os.environ:
             del os.environ["TOPOTESTS_CHECK_STDERR"]
 
+    if config.option.cov_topotest:
+        setup_coverage(config)
+
 
 @pytest.fixture(autouse=True, scope="session")
-def setup_session_auto():
+def session_autouse():
     # Aligns logs nicely
     logging.addLevelName(logging.WARNING, " WARN")
     logging.addLevelName(logging.INFO, " INFO")
 
-    if "PYTEST_TOPOTEST_WORKER" not in os.environ:
-        is_worker = False
-    elif not os.environ["PYTEST_TOPOTEST_WORKER"]:
-        is_worker = False
-    else:
-        is_worker = True
+    is_main = is_main_runner()
 
-    logger.debug("Before the run (is_worker: %s)", is_worker)
-    if not is_worker:
+    logger.debug("Before the run (is_main: %s)", is_main)
+    if is_main:
         cleanup_previous()
     yield
-    if not is_worker:
+    if is_main:
         cleanup_current()
-    logger.debug("After the run (is_worker: %s)", is_worker)
+    logger.debug("After the run (is_main: %s)", is_main)
 
 
 def pytest_runtest_setup(item):
@@ -719,6 +761,42 @@ def pytest_runtest_makereport(item, call):
         pause_test()
 
 
+def coverage_finish(terminalreporter, config):
+    commander = Commander("pytest")
+    rundir = Path(config.option.rundir).resolve()
+    bdir = Path(os.environ["FRR_BUILD_DIR"])
+    gcdadir = Path(os.environ["GCOV_PREFIX"])
+
+    # Get the data files into the build directory
+    logger.info("Copying gcda files from '%s' to '%s'", gcdadir, bdir)
+    user = os.environ.get("SUDO_USER", os.environ["USER"])
+    commander.cmd_raises(f"chmod -R ugo+r {gcdadir}")
+    commander.cmd_raises(
+        f"tar -C {gcdadir} -cf - . | su {user} -c 'tar -C {bdir} -xf -'"
+    )
+
+    # Get the results into a summary file
+    data_file = rundir / "coverage.info"
+    logger.info("Gathering coverage data into: %s", data_file)
+    commander.cmd_raises(f"lcov --directory {bdir} --capture --output-file {data_file}")
+
+    # Get coverage info filtered to a specific set of files
+    report_file = rundir / "coverage.info"
+    logger.debug("Generating coverage summary from: %s\n%s", report_file)
+    output = commander.cmd_raises(f"lcov --summary {data_file}")
+    logger.info("\nCOVERAGE-SUMMARY-START\n%s\nCOVERAGE-SUMMARY-END", output)
+    terminalreporter.write(
+        f"\nCOVERAGE-SUMMARY-START\n{output}\nCOVERAGE-SUMMARY-END\n"
+    )
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    # Only run if we are the top level test runner
+    is_xdist_worker = "PYTEST_XDIST_WORKER" in os.environ
+    if config.option.cov_topotest and not is_xdist_worker:
+        coverage_finish(terminalreporter, config)
+
+
 #
 # Add common fixtures available to all tests as parameters
 #
diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py
index 0a5be970b8..2f69f7364b 100644
--- a/tests/topotests/lib/topotest.py
+++ b/tests/topotests/lib/topotest.py
@@ -27,6 +27,7 @@ import time
 import logging
 from collections.abc import Mapping
 from copy import deepcopy
+from pathlib import Path
 
 import lib.topolog as topolog
 from lib.micronet_compat import Node
@@ -1262,8 +1263,8 @@ def rlimit_atleast(rname, min_value, raises=False):
 
 def fix_netns_limits(ns):
     # Maximum read and write socket buffer sizes
-    sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2 ** 20])
-    sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2 ** 20])
+    sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2**20])
+    sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2**20])
 
     sysctl_assure(ns, "net.ipv4.conf.all.rp_filter", 0)
     sysctl_assure(ns, "net.ipv4.conf.default.rp_filter", 0)
@@ -1322,8 +1323,8 @@ def fix_host_limits():
     sysctl_atleast(None, "net.core.netdev_max_backlog", 4 * 1024)
 
     # Maximum read and write socket buffer sizes
-    sysctl_atleast(None, "net.core.rmem_max", 16 * 2 ** 20)
-    sysctl_atleast(None, "net.core.wmem_max", 16 * 2 ** 20)
+    sysctl_atleast(None, "net.core.rmem_max", 16 * 2**20)
+    sysctl_atleast(None, "net.core.wmem_max", 16 * 2**20)
 
     # Garbage Collection Settings for ARP and Neighbors
     sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh2", 4 * 1024)
@@ -1523,7 +1524,7 @@ class Router(Node):
                 pass
         return ret
 
-    def stopRouter(self, assertOnError=True, minErrorVersion="5.1"):
+    def stopRouter(self, assertOnError=True):
         # Stop Running FRR Daemons
         running = self.listDaemons()
         if not running:
@@ -1570,9 +1571,6 @@ class Router(Node):
             )
 
         errors = self.checkRouterCores(reportOnce=True)
-        if self.checkRouterVersion("<", minErrorVersion):
-            # ignore errors in old versions
-            errors = ""
         if assertOnError and (errors is not None) and len(errors) > 0:
             assert "Errors found - details follow:" == 0, errors
         return errors
@@ -1803,6 +1801,8 @@ class Router(Node):
         "Starts FRR daemons for this router."
 
         asan_abort = bool(g_pytest_config.option.asan_abort)
+        cov_option = bool(g_pytest_config.option.cov_topotest)
+        cov_dir = Path(g_pytest_config.option.rundir) / "gcda"
         gdb_breakpoints = g_pytest_config.get_option_list("--gdb-breakpoints")
         gdb_daemons = g_pytest_config.get_option_list("--gdb-daemons")
         gdb_routers = g_pytest_config.get_option_list("--gdb-routers")
@@ -1836,13 +1836,6 @@ class Router(Node):
         # Re-enable to allow for report per run
         self.reportCores = True
 
-        # XXX: glue code forward ported from removed function.
-        if self.version is None:
-            self.version = self.cmd(
-                os.path.join(self.daemondir, "bgpd") + " -v"
-            ).split()[2]
-            logger.info("{}: running version: {}".format(self.name, self.version))
-
         perfds = {}
         perf_options = g_pytest_config.get_option("--perf-options", "-g")
         for perf in g_pytest_config.get_option("--perf", []):
@@ -1928,6 +1921,10 @@ class Router(Node):
                     self.logdir, self.name, daemon
                 )
 
+                if cov_option:
+                    scount = os.environ["GCOV_PREFIX_STRIP"]
+                    cmdenv += f"GCOV_PREFIX_STRIP={scount} GCOV_PREFIX={cov_dir}"
+
                 if valgrind_memleaks:
                     this_dir = os.path.dirname(
                         os.path.abspath(os.path.realpath(__file__))
@@ -2277,9 +2274,7 @@ class Router(Node):
         rc, o, e = self.cmd_status("kill -0 " + str(pid), warn=False)
         return rc == 0 or "No such process" not in e
 
-    def killRouterDaemons(
-        self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1"
-    ):
+    def killRouterDaemons(self, daemons, wait=True, assertOnError=True):
         # Kill Running FRR
         # Daemons(user specified daemon only) using SIGKILL
         rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype)
@@ -2339,9 +2334,6 @@ class Router(Node):
                         self.cmd("rm -- {}".format(daemonpidfile))
                     if wait:
                         errors = self.checkRouterCores(reportOnce=True)
-                        if self.checkRouterVersion("<", minErrorVersion):
-                            # ignore errors in old versions
-                            errors = ""
                         if assertOnError and len(errors) > 0:
                             assert "Errors found - details follow:" == 0, errors
             else: