tests: add triage features: strace, asan-abort, docker exec

TMUX and Screen support when running topotests inside docker. This
allows the gdb, shell and vtysh features to correctly work even when
running the tests inside docker.

Add options:
--asan-abort :: aborts the process on ASAN errors
--strace-daemons :: strace some or all daemons

Signed-off-by: Christian Hopps <chopps@labn.net>

diff --git a/tests/topotests/conftest.py b/tests/topotests/conftest.py
index e57db7471c9905f83061ae30417d1260434e8602..d119b0931bdceec8cee35217344de844a497d42e 100755 (executable)
--- a/tests/topotests/conftest.py
+++ b/tests/topotests/conftest.py
@@ -25,6 +25,12 @@ def pytest_addoption(parser):
     Add topology-only option to the topology tester. This option makes pytest
     only run the setup_module() to setup the topology without running any tests.
     """
+    parser.addoption(
+        "--asan-abort",
+        action="store_true",
+        help="Configure address sanitizer to abort process on error",
+    )
+
     parser.addoption(
         "--gdb-breakpoints",
         metavar="SYMBOL[,SYMBOL...]",
@@ -67,6 +73,12 @@ def pytest_addoption(parser):
         help="Spawn shell on all routers on test failure",
     )
 
+    parser.addoption(
+        "--strace-daemons",
+        metavar="DAEMON[,DAEMON...]",
+        help="Comma-separated list of daemons to strace, or 'all'",
+    )
+
     parser.addoption(
         "--topology-only",
         action="store_true",
@@ -167,6 +179,9 @@ def pytest_configure(config):
     if not diagnose_env():
         pytest.exit("environment has errors, please read the logs")
 
+    asan_abort = config.getoption("--asan-abort")
+    topotest_extra_config["asan_abort"] = asan_abort
+
     gdb_routers = config.getoption("--gdb-routers")
     gdb_routers = gdb_routers.split(",") if gdb_routers else []
     topotest_extra_config["gdb_routers"] = gdb_routers
@@ -185,6 +200,9 @@ def pytest_configure(config):
     shell = config.getoption("--shell")
     topotest_extra_config["shell"] = shell.split(",") if shell else []
 
+    strace = config.getoption("--strace-daemons")
+    topotest_extra_config["strace_daemons"] = strace.split(",") if strace else []
+
     pause_after = config.getoption("--pause-after")
 
     shell_on_error = config.getoption("--shell-on-error")
@@ -244,6 +262,11 @@ def pytest_runtest_makereport(item, call):
                 )
             )
 
+            # We want to pause, if requested, on any error not just test cases
+            # (e.g., call.when == "setup")
+            if not pause:
+                pause = topotest_extra_config["pause_after"]
+
             # (topogen) Set topology error to avoid advancing in the test.
             tgen = get_topogen()
             if tgen is not None:

diff --git a/tests/topotests/docker/frr-topotests.sh b/tests/topotests/docker/frr-topotests.sh
index 9ef59b3bbcd6e1db6b6a81cb0b80c6faf61bb5de..1eaaea2971134861ed1dbfc786c2bbfa44c45d74 100755 (executable)
--- a/tests/topotests/docker/frr-topotests.sh
+++ b/tests/topotests/docker/frr-topotests.sh
@@ -145,7 +145,15 @@ if [ "${TOPOTEST_PULL:-1}" = "1" ]; then
        docker pull frrouting/topotests:latest
 fi
 
+if [[ -n "$TMUX" ]]; then
+    TMUX_OPTIONS="-v $(dirname $TMUX):$(dirname $TMUX) -e TMUX=$TMUX -e TMUX_PANE=$TMUX_PANE"
+fi
+
+if [[ -n "$STY" ]]; then
+    SCREEN_OPTIONS="-v /run/screen:/run/screen -e STY=$STY"
+fi
 set -- --rm -i \
+        -v "$HOME:$HOME:ro" \
        -v "$TOPOTEST_LOGS:/tmp" \
        -v "$TOPOTEST_FRR:/root/host-frr:ro" \
        -v "$TOPOTEST_BUILDCACHE:/root/persist" \
@@ -154,6 +162,8 @@ set -- --rm -i \
        -e "TOPOTEST_DOC=$TOPOTEST_DOC" \
        -e "TOPOTEST_SANITIZER=$TOPOTEST_SANITIZER" \
        --privileged \
+        $SCREEN_OPTINS \
+        $TMUX_OPTIONS \
        $TOPOTEST_OPTIONS \
        frrouting/topotests:latest "$@"
 

diff --git a/tests/topotests/lib/topogen.py b/tests/topotests/lib/topogen.py
index ade5933504ac1a554dadd16109538021acd2ff66..b9988781187517d3831765407e140f5fba5ce6e5 100644 (file)
--- a/tests/topotests/lib/topogen.py
+++ b/tests/topotests/lib/topogen.py
@@ -801,8 +801,8 @@ class TopoRouter(TopoGear):
 
         try:
             return json.loads(output)
-        except ValueError:
-            logger.warning("vtysh_cmd: failed to convert json output")
+        except ValueError as error:
+            logger.warning("vtysh_cmd: %s: failed to convert json output: %s: %s", self.name, str(output), str(error))
             return {}
 
     def vtysh_multicmd(self, commands, pretty_output=True, daemon=None):

diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py
index d1f60bfe0dbc1ce938ac0299ecfd40131956fa8b..23dcced2bf125c867569be5e18ac70f5d8ba5b5a 100644 (file)
--- a/tests/topotests/lib/topotest.py
+++ b/tests/topotests/lib/topotest.py
@@ -1152,6 +1152,18 @@ class Router(Node):
         self.reportCores = True
         self.version = None
 
+        self.ns_cmd = "sudo nsenter -m -n -t {} ".format(self.pid)
+        try:
+            # Allow escaping from running inside docker
+            cgroup = open("/proc/1/cgroup").read()
+            m = re.search("[0-9]+:cpuset:/docker/([a-f0-9]+)", cgroup)
+            if m:
+                self.ns_cmd = "docker exec -it {} ".format(m.group(1)) + self.ns_cmd
+        except IOError:
+            pass
+        else:
+            logger.debug("CMD to enter {}: {}".format(self.name, self.ns_cmd))
+
     def _config_frr(self, **params):
         "Configure FRR binaries"
         self.daemondir = params.get("frrdir")
@@ -1350,7 +1362,7 @@ class Router(Node):
             term = topo_terminal if topo_terminal else "xterm"
             makeTerm(self, title=title if title else cmd, term=term, cmd=cmd)
         else:
-            nscmd = "sudo nsenter -m -n -t {} {}".format(self.pid, cmd)
+            nscmd = self.ns_cmd + cmd
             if "TMUX" in os.environ:
                 self.cmd("tmux select-layout main-horizontal")
                 wcmd = "tmux split-window -h"
@@ -1451,11 +1463,13 @@ class Router(Node):
     def startRouterDaemons(self, daemons=None):
         "Starts all FRR daemons for this router."
 
+        asan_abort = g_extra_config["asan_abort"]
         gdb_breakpoints = g_extra_config["gdb_breakpoints"]
         gdb_daemons = g_extra_config["gdb_daemons"]
         gdb_routers = g_extra_config["gdb_routers"]
         valgrind_extra = g_extra_config["valgrind_extra"]
         valgrind_memleaks = g_extra_config["valgrind_memleaks"]
+        strace_daemons = g_extra_config["strace_daemons"]
 
         bundle_data = ""
 
@@ -1482,7 +1496,6 @@ class Router(Node):
                 os.path.join(self.daemondir, "bgpd") + " -v"
             ).split()[2]
             logger.info("{}: running version: {}".format(self.name, self.version))
-
         # If `daemons` was specified then some upper API called us with
         # specific daemons, otherwise just use our own configuration.
         daemons_list = []
@@ -1506,13 +1519,20 @@ class Router(Node):
             else:
                 binary = os.path.join(self.daemondir, daemon)
 
-                cmdenv = "ASAN_OPTIONS=log_path={0}.asan".format(daemon)
+                cmdenv = "ASAN_OPTIONS="
+                if asan_abort:
+                    cmdenv = "abort_on_error=1:"
+                cmdenv += "log_path={0}/{1}.{2}.asan ".format(self.logdir, self.name, daemon)
+
                 if valgrind_memleaks:
                     this_dir = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
                     supp_file = os.path.abspath(os.path.join(this_dir, "../../../tools/valgrind.supp"))
                     cmdenv += " /usr/bin/valgrind --num-callers=50 --log-file={1}/{2}.valgrind.{0}.%p --leak-check=full --suppressions={3}".format(daemon, self.logdir, self.name, supp_file)
                     if valgrind_extra:
                         cmdenv += "--gen-suppressions=all --expensive-definedness-checks=yes"
+                elif daemon in strace_daemons or "all" in strace_daemons:
+                    cmdenv = "strace -f -D -o {1}/{2}.strace.{0} ".format(daemon, self.logdir, self.name)
+
                 cmdopt = "{} --log file:{}.log --log-level debug".format(
                     daemon_opts, daemon
                 )