To specify additional arguments for ``rr record``, one can use the
``--rr-options``.
+.. _code_coverage:
+
+Code coverage
+"""""""""""""
+Code coverage reporting requires installation of the ``gcov`` and ``lcov``
+packages.
+
+Code coverage can automatically be gathered for any topotest run. To support
+this FRR must first be compiled with the ``--enable-gcov`` configure option.
+This will cause *.gnco files to be created during the build. When topotests are
+run the statistics are generated and stored in *.gcda files. Topotest
+infrastructure will gather these files, capture the information into a
+``coverage.info`` ``lcov`` file and also report the coverage summary.
+
+To enable code coverage support pass the ``--cov-topotest`` argument to pytest.
+If you build your FRR in a directory outside of the FRR source directory you
+will also need to pass the ``--cov-frr-build-dir`` argument specifying the build
+directory location.
+
+During the topotest run the *.gcda files are generated into a ``gcda``
+sub-directory of the top-level run directory (i.e., normally
+``/tmp/topotests/gcda``). These files will then be copied at the end of the
+topotest run into the FRR build directory where the ``gcov`` and ``lcov``
+utilities expect to find them. This is done to deal with the various different
+file ownership and permissions.
+
+At the end of the run ``lcov`` will be run to capture all of the coverage data
+into a ``coverage.info`` file. This file will be located in the top-level run
+directory (i.e., normally ``/tmp/topotests/coverage.info``).
+
+The ``coverage.info`` file can then be used to generate coverage reports or file
+markup (e.g., using the ``genhtml`` utility) or enable markup within your
+IDE/editor if supported (e.g., the emacs ``cov-mode`` package)
+
+NOTE: the *.gcda files in ``/tmp/topotests/gcda`` are cumulative so if you do
+not remove them they will aggregate data across multiple topotest runs.
+
+
.. _topotests_docker:
Running Tests with Docker
topolog.logfinish(basename, logpath)
+def is_main_runner():
+ return "PYTEST_XDIST_WORKER" not in os.environ
+
+
def pytest_addoption(parser):
"""
Add topology-only option to the topology tester. This option makes pytest
help="Mininet cli on test failure",
)
+ parser.addoption(
+ "--cov-topotest",
+ action="store_true",
+ help="Enable reporting of coverage",
+ )
+
+ parser.addoption(
+ "--cov-frr-build-dir",
+ help="Dir of coverage-enable build being run, default is the source dir",
+ )
+
parser.addoption(
"--gdb-breakpoints",
metavar="SYMBOL[,SYMBOL...]",
return json_result.gen_report()
+def setup_coverage(config):
+ commander = Commander("pytest")
+ if config.option.cov_frr_build_dir:
+ bdir = Path(config.option.cov_frr_build_dir).resolve()
+ output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
+ else:
+ # Support build sub-directory of main source dir
+ bdir = Path(__file__).resolve().parent.parent.parent
+ output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
+ m = re.match(f"({bdir}.*)/zebra/zebra_nb.gcno", output)
+ if not m:
+ logger.warning(
+ "No coverage data files (*.gcno) found, try specifying --cov-frr-build-dir"
+ )
+ return
+
+ bdir = Path(m.group(1))
+ # Save so we can get later from g_pytest_config
+ rundir = Path(config.option.rundir).resolve()
+ gcdadir = rundir / "gcda"
+ os.environ["FRR_BUILD_DIR"] = str(bdir)
+ os.environ["GCOV_PREFIX_STRIP"] = str(len(bdir.parts) - 1)
+ os.environ["GCOV_PREFIX"] = str(gcdadir)
+
+ if is_main_runner():
+ commander.cmd_raises(f"find {bdir} -name '*.gc??' -exec chmod o+rw {{}} +")
+ commander.cmd_raises(f"mkdir -p {gcdadir}")
+ commander.cmd_raises(f"chown -R root:frr {gcdadir}")
+ commander.cmd_raises(f"chmod 2775 {gcdadir}")
+
+
def pytest_configure(config):
"""
Assert that the environment is correctly configured, and get extra config.
if config.option.topology_only and is_xdist:
pytest.exit("Cannot use --topology-only with distributed test mode")
- pytest.exit("Cannot use --topology-only with distributed test mode")
-
# Check environment now that we have config
if not diagnose_env(rundir):
pytest.exit("environment has errors, please read the logs in %s" % rundir)
if "TOPOTESTS_CHECK_STDERR" in os.environ:
del os.environ["TOPOTESTS_CHECK_STDERR"]
+ if config.option.cov_topotest:
+ setup_coverage(config)
+
@pytest.fixture(autouse=True, scope="session")
-def setup_session_auto():
+def session_autouse():
# Aligns logs nicely
logging.addLevelName(logging.WARNING, " WARN")
logging.addLevelName(logging.INFO, " INFO")
- if "PYTEST_TOPOTEST_WORKER" not in os.environ:
- is_worker = False
- elif not os.environ["PYTEST_TOPOTEST_WORKER"]:
- is_worker = False
- else:
- is_worker = True
+ is_main = is_main_runner()
- logger.debug("Before the run (is_worker: %s)", is_worker)
- if not is_worker:
+ logger.debug("Before the run (is_main: %s)", is_main)
+ if is_main:
cleanup_previous()
yield
- if not is_worker:
+ if is_main:
cleanup_current()
- logger.debug("After the run (is_worker: %s)", is_worker)
+ logger.debug("After the run (is_main: %s)", is_main)
def pytest_runtest_setup(item):
pause_test()
+def coverage_finish(terminalreporter, config):
+ commander = Commander("pytest")
+ rundir = Path(config.option.rundir).resolve()
+ bdir = Path(os.environ["FRR_BUILD_DIR"])
+ gcdadir = Path(os.environ["GCOV_PREFIX"])
+
+ # Get the data files into the build directory
+ logger.info("Copying gcda files from '%s' to '%s'", gcdadir, bdir)
+ user = os.environ.get("SUDO_USER", os.environ["USER"])
+ commander.cmd_raises(f"chmod -R ugo+r {gcdadir}")
+ commander.cmd_raises(
+ f"tar -C {gcdadir} -cf - . | su {user} -c 'tar -C {bdir} -xf -'"
+ )
+
+ # Get the results into a summary file
+ data_file = rundir / "coverage.info"
+ logger.info("Gathering coverage data into: %s", data_file)
+ commander.cmd_raises(f"lcov --directory {bdir} --capture --output-file {data_file}")
+
+ # Get coverage info filtered to a specific set of files
+ report_file = rundir / "coverage.info"
+ logger.debug("Generating coverage summary from: %s\n%s", report_file)
+ output = commander.cmd_raises(f"lcov --summary {data_file}")
+ logger.info("\nCOVERAGE-SUMMARY-START\n%s\nCOVERAGE-SUMMARY-END", output)
+ terminalreporter.write(
+ f"\nCOVERAGE-SUMMARY-START\n{output}\nCOVERAGE-SUMMARY-END\n"
+ )
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+ # Only run if we are the top level test runner
+ is_xdist_worker = "PYTEST_XDIST_WORKER" in os.environ
+ if config.option.cov_topotest and not is_xdist_worker:
+ coverage_finish(terminalreporter, config)
+
+
#
# Add common fixtures available to all tests as parameters
#
import logging
from collections.abc import Mapping
from copy import deepcopy
+from pathlib import Path
import lib.topolog as topolog
from lib.micronet_compat import Node
pass
return ret
- def stopRouter(self, assertOnError=True, minErrorVersion="5.1"):
+ def stopRouter(self, assertOnError=True):
# Stop Running FRR Daemons
running = self.listDaemons()
if not running:
)
errors = self.checkRouterCores(reportOnce=True)
- if self.checkRouterVersion("<", minErrorVersion):
- # ignore errors in old versions
- errors = ""
if assertOnError and (errors is not None) and len(errors) > 0:
assert "Errors found - details follow:" == 0, errors
return errors
"Starts FRR daemons for this router."
asan_abort = bool(g_pytest_config.option.asan_abort)
+ cov_option = bool(g_pytest_config.option.cov_topotest)
+ cov_dir = Path(g_pytest_config.option.rundir) / "gcda"
gdb_breakpoints = g_pytest_config.get_option_list("--gdb-breakpoints")
gdb_daemons = g_pytest_config.get_option_list("--gdb-daemons")
gdb_routers = g_pytest_config.get_option_list("--gdb-routers")
# Re-enable to allow for report per run
self.reportCores = True
- # XXX: glue code forward ported from removed function.
- if self.version is None:
- self.version = self.cmd(
- os.path.join(self.daemondir, "bgpd") + " -v"
- ).split()[2]
- logger.info("{}: running version: {}".format(self.name, self.version))
-
perfds = {}
perf_options = g_pytest_config.get_option("--perf-options", "-g")
for perf in g_pytest_config.get_option("--perf", []):
self.logdir, self.name, daemon
)
+ if cov_option:
+ scount = os.environ["GCOV_PREFIX_STRIP"]
+ cmdenv += f"GCOV_PREFIX_STRIP={scount} GCOV_PREFIX={cov_dir}"
+
if valgrind_memleaks:
this_dir = os.path.dirname(
os.path.abspath(os.path.realpath(__file__))
rc, o, e = self.cmd_status("kill -0 " + str(pid), warn=False)
return rc == 0 or "No such process" not in e
- def killRouterDaemons(
- self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1"
- ):
+ def killRouterDaemons(self, daemons, wait=True, assertOnError=True):
# Kill Running FRR
# Daemons(user specified daemon only) using SIGKILL
rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype)
self.cmd("rm -- {}".format(daemonpidfile))
if wait:
errors = self.checkRouterCores(reportOnce=True)
- if self.checkRouterVersion("<", minErrorVersion):
- # ignore errors in old versions
- errors = ""
if assertOnError and len(errors) > 0:
assert "Errors found - details follow:" == 0, errors
else:
projects / matthieu / frr.git / commitdiff