From 0ba1d257bedebcd01dffa4dbfb406bc3a2243d60 Mon Sep 17 00:00:00 2001 From: Christian Hopps Date: Fri, 9 Jul 2021 05:22:51 -0400 Subject: [PATCH] tests: add triage features: strace, asan-abort, docker exec TMUX and Screen support when running topotests inside docker. This allows the gdb, shell and vtysh features to correctly work even when running the tests inside docker. Add options: --asan-abort :: aborts the process on ASAN errors --strace-daemons :: strace some or all daemons Signed-off-by: Christian Hopps --- tests/topotests/conftest.py | 23 ++++++++++++++++++++++ tests/topotests/docker/frr-topotests.sh | 10 ++++++++++ tests/topotests/lib/topogen.py | 4 ++-- tests/topotests/lib/topotest.py | 26 ++++++++++++++++++++++--- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/tests/topotests/conftest.py b/tests/topotests/conftest.py index e57db7471c..d119b0931b 100755 --- a/tests/topotests/conftest.py +++ b/tests/topotests/conftest.py @@ -25,6 +25,12 @@ def pytest_addoption(parser): Add topology-only option to the topology tester. This option makes pytest only run the setup_module() to setup the topology without running any tests. """ + parser.addoption( + "--asan-abort", + action="store_true", + help="Configure address sanitizer to abort process on error", + ) + parser.addoption( "--gdb-breakpoints", metavar="SYMBOL[,SYMBOL...]", @@ -67,6 +73,12 @@ def pytest_addoption(parser): help="Spawn shell on all routers on test failure", ) + parser.addoption( + "--strace-daemons", + metavar="DAEMON[,DAEMON...]", + help="Comma-separated list of daemons to strace, or 'all'", + ) + parser.addoption( "--topology-only", action="store_true", @@ -167,6 +179,9 @@ def pytest_configure(config): if not diagnose_env(): pytest.exit("environment has errors, please read the logs") + asan_abort = config.getoption("--asan-abort") + topotest_extra_config["asan_abort"] = asan_abort + gdb_routers = config.getoption("--gdb-routers") gdb_routers = gdb_routers.split(",") if gdb_routers else [] topotest_extra_config["gdb_routers"] = gdb_routers @@ -185,6 +200,9 @@ def pytest_configure(config): shell = config.getoption("--shell") topotest_extra_config["shell"] = shell.split(",") if shell else [] + strace = config.getoption("--strace-daemons") + topotest_extra_config["strace_daemons"] = strace.split(",") if strace else [] + pause_after = config.getoption("--pause-after") shell_on_error = config.getoption("--shell-on-error") @@ -244,6 +262,11 @@ def pytest_runtest_makereport(item, call): ) ) + # We want to pause, if requested, on any error not just test cases + # (e.g., call.when == "setup") + if not pause: + pause = topotest_extra_config["pause_after"] + # (topogen) Set topology error to avoid advancing in the test. tgen = get_topogen() if tgen is not None: diff --git a/tests/topotests/docker/frr-topotests.sh b/tests/topotests/docker/frr-topotests.sh index 9ef59b3bbc..1eaaea2971 100755 --- a/tests/topotests/docker/frr-topotests.sh +++ b/tests/topotests/docker/frr-topotests.sh @@ -145,7 +145,15 @@ if [ "${TOPOTEST_PULL:-1}" = "1" ]; then docker pull frrouting/topotests:latest fi +if [[ -n "$TMUX" ]]; then + TMUX_OPTIONS="-v $(dirname $TMUX):$(dirname $TMUX) -e TMUX=$TMUX -e TMUX_PANE=$TMUX_PANE" +fi + +if [[ -n "$STY" ]]; then + SCREEN_OPTIONS="-v /run/screen:/run/screen -e STY=$STY" +fi set -- --rm -i \ + -v "$HOME:$HOME:ro" \ -v "$TOPOTEST_LOGS:/tmp" \ -v "$TOPOTEST_FRR:/root/host-frr:ro" \ -v "$TOPOTEST_BUILDCACHE:/root/persist" \ @@ -154,6 +162,8 @@ set -- --rm -i \ -e "TOPOTEST_DOC=$TOPOTEST_DOC" \ -e "TOPOTEST_SANITIZER=$TOPOTEST_SANITIZER" \ --privileged \ + $SCREEN_OPTINS \ + $TMUX_OPTIONS \ $TOPOTEST_OPTIONS \ frrouting/topotests:latest "$@" diff --git a/tests/topotests/lib/topogen.py b/tests/topotests/lib/topogen.py index ade5933504..b998878118 100644 --- a/tests/topotests/lib/topogen.py +++ b/tests/topotests/lib/topogen.py @@ -801,8 +801,8 @@ class TopoRouter(TopoGear): try: return json.loads(output) - except ValueError: - logger.warning("vtysh_cmd: failed to convert json output") + except ValueError as error: + logger.warning("vtysh_cmd: %s: failed to convert json output: %s: %s", self.name, str(output), str(error)) return {} def vtysh_multicmd(self, commands, pretty_output=True, daemon=None): diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index d1f60bfe0d..23dcced2bf 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -1152,6 +1152,18 @@ class Router(Node): self.reportCores = True self.version = None + self.ns_cmd = "sudo nsenter -m -n -t {} ".format(self.pid) + try: + # Allow escaping from running inside docker + cgroup = open("/proc/1/cgroup").read() + m = re.search("[0-9]+:cpuset:/docker/([a-f0-9]+)", cgroup) + if m: + self.ns_cmd = "docker exec -it {} ".format(m.group(1)) + self.ns_cmd + except IOError: + pass + else: + logger.debug("CMD to enter {}: {}".format(self.name, self.ns_cmd)) + def _config_frr(self, **params): "Configure FRR binaries" self.daemondir = params.get("frrdir") @@ -1350,7 +1362,7 @@ class Router(Node): term = topo_terminal if topo_terminal else "xterm" makeTerm(self, title=title if title else cmd, term=term, cmd=cmd) else: - nscmd = "sudo nsenter -m -n -t {} {}".format(self.pid, cmd) + nscmd = self.ns_cmd + cmd if "TMUX" in os.environ: self.cmd("tmux select-layout main-horizontal") wcmd = "tmux split-window -h" @@ -1451,11 +1463,13 @@ class Router(Node): def startRouterDaemons(self, daemons=None): "Starts all FRR daemons for this router." + asan_abort = g_extra_config["asan_abort"] gdb_breakpoints = g_extra_config["gdb_breakpoints"] gdb_daemons = g_extra_config["gdb_daemons"] gdb_routers = g_extra_config["gdb_routers"] valgrind_extra = g_extra_config["valgrind_extra"] valgrind_memleaks = g_extra_config["valgrind_memleaks"] + strace_daemons = g_extra_config["strace_daemons"] bundle_data = "" @@ -1482,7 +1496,6 @@ class Router(Node): os.path.join(self.daemondir, "bgpd") + " -v" ).split()[2] logger.info("{}: running version: {}".format(self.name, self.version)) - # If `daemons` was specified then some upper API called us with # specific daemons, otherwise just use our own configuration. daemons_list = [] @@ -1506,13 +1519,20 @@ class Router(Node): else: binary = os.path.join(self.daemondir, daemon) - cmdenv = "ASAN_OPTIONS=log_path={0}.asan".format(daemon) + cmdenv = "ASAN_OPTIONS=" + if asan_abort: + cmdenv = "abort_on_error=1:" + cmdenv += "log_path={0}/{1}.{2}.asan ".format(self.logdir, self.name, daemon) + if valgrind_memleaks: this_dir = os.path.dirname(os.path.abspath(os.path.realpath(__file__))) supp_file = os.path.abspath(os.path.join(this_dir, "../../../tools/valgrind.supp")) cmdenv += " /usr/bin/valgrind --num-callers=50 --log-file={1}/{2}.valgrind.{0}.%p --leak-check=full --suppressions={3}".format(daemon, self.logdir, self.name, supp_file) if valgrind_extra: cmdenv += "--gen-suppressions=all --expensive-definedness-checks=yes" + elif daemon in strace_daemons or "all" in strace_daemons: + cmdenv = "strace -f -D -o {1}/{2}.strace.{0} ".format(daemon, self.logdir, self.name) + cmdopt = "{} --log file:{}.log --log-level debug".format( daemon_opts, daemon ) -- 2.39.5