tests: Generate support bundle/dump data on tests failures

1. It will generate support bundle/sump data on test failures
2. It used /usr/lib/frr/generate_support_bundle.py utility to dump the data

Signed-off-by: Kuldeep Kashyap <kashyapk@vmware.com>

diff --git a/tests/topotests/lib/common_config.py b/tests/topotests/lib/common_config.py
index 1f2acb7f887684ee6030d4ce3debe0270edee477..21ed47fc4bd99c191fd2a80dc0180bd7ae6a754a 100644 (file)
--- a/tests/topotests/lib/common_config.py
+++ b/tests/topotests/lib/common_config.py
@@ -39,10 +39,9 @@ import socket
 import ipaddr
 
 from lib.topolog import logger, logger_config
-from lib.topogen import TopoRouter
+from lib.topogen import TopoRouter, get_topogen
 from lib.topotest import interface_set_status
 
-
 FRRCFG_FILE = "frr_json.conf"
 FRRCFG_BKUP_FILE = "frr_json_initial.conf"
 
@@ -475,25 +474,19 @@ def reset_config_on_routers(tgen, routerName=None):
 
         for line in t_delta.split("\n"):
             line = line.strip()
-            if (
-                line == "Lines To Delete"
-                or line == "==============="
-                or not line
-            ):
+            if line == "Lines To Delete" or line == "===============" or not line:
                 continue
 
-            if (line == "Lines To Add"):
+            if line == "Lines To Add":
                 check_debug = False
                 continue
 
-            if (line == "============"
-                or not line
-            ):
+            if line == "============" or not line:
                 continue
 
             # Leave debugs and log output alone
             if check_debug:
-                if ('debug' in line or 'log file' in line):
+                if "debug" in line or "log file" in line:
                     continue
 
             delta.write(line)
@@ -648,6 +641,33 @@ def get_frr_ipv6_linklocal(tgen, router, intf=None, vrf=None):
         return errormsg
 
 
+def generate_support_bundle():
+    """
+    API to generate support bundle on any verification ste failure.
+    it runs a python utility, /usr/lib/frr/generate_support_bundle.py,
+    which basically runs defined CLIs and dumps the data to specified location
+    """
+
+    tgen = get_topogen()
+    router_list = tgen.routers()
+    test_name = sys._getframe(2).f_code.co_name
+    TMPDIR = os.path.join(LOGDIR, tgen.modname)
+
+    for rname, rnode in router_list.iteritems():
+        logger.info("Generating support bundle for {}".format(rname))
+        rnode.run("mkdir -p /var/log/frr")
+        bundle_log = rnode.run("python2 /usr/lib/frr/generate_support_bundle.py")
+        logger.info(bundle_log)
+
+        dst_bundle = "{}/{}/support_bundles/{}".format(TMPDIR, rname, test_name)
+        src_bundle = "/var/log/frr"
+        rnode.run("rm -rf {}".format(dst_bundle))
+        rnode.run("mkdir -p {}".format(dst_bundle))
+        rnode.run("mv -f {}/* {}".format(src_bundle, dst_bundle))
+
+    return True
+
+
 def start_topology(tgen):
     """
     Starting topology, create tmp files which are loaded to routers
@@ -1202,9 +1222,11 @@ def retry(attempts=3, wait=2, return_is_str=True, initial_wait=0, return_is_dict
                         return ret
 
                     if _attempts == i:
+                        generate_support_bundle()
                         return ret
                 except Exception as err:
                     if _attempts == i:
+                        generate_support_bundle()
                         logger.info("Max number of attempts (%r) reached", _attempts)
                         raise
                     else:
@@ -1278,9 +1300,11 @@ def create_interfaces_cfg(tgen, topo, build=False):
 
                 # Include vrf if present
                 if "vrf" in data:
-                    interface_data.append("interface {} vrf {}".format(
-                        str(interface_name),
-                        str(data["vrf"])))
+                    interface_data.append(
+                        "interface {} vrf {}".format(
+                            str(interface_name), str(data["vrf"])
+                        )
+                    )
                 else:
                     interface_data.append("interface {}".format(str(interface_name)))
 

diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py
index b65acad23e2b762b010e2cf7e2268c057876da85..22ed4b4d0f21760b221280a90e3a4e4ee8dca3a7 100644 (file)
--- a/tests/topotests/lib/topotest.py
+++ b/tests/topotests/lib/topotest.py
@@ -1168,7 +1168,6 @@ class Router(Node):
 
         return self.startRouterDaemons()
 
-
     def getStdErr(self, daemon):
         return self.getLog("err", daemon)
 
@@ -1181,6 +1180,13 @@ class Router(Node):
     def startRouterDaemons(self, daemons=None):
         "Starts all FRR daemons for this router."
 
+        bundle_data = subprocess.check_output(
+            ["cat /etc/frr/support_bundle_commands.conf"], shell=True
+        )
+        self.cmd(
+            "echo '{}' > /etc/frr/support_bundle_commands.conf".format(bundle_data)
+        )
+
         # Starts actual daemons without init (ie restart)
         # cd to per node directory
         self.cmd("cd {}/{}".format(self.logdir, self.name))
@@ -1192,7 +1198,7 @@ class Router(Node):
         # XXX: glue code forward ported from removed function.
         if self.version == None:
             self.version = self.cmd(
-                os.path.join(self.daemondir, 'bgpd') + ' -v'
+                os.path.join(self.daemondir, "bgpd") + " -v"
             ).split()[2]
             logger.info("{}: running version: {}".format(self.name, self.version))
 
@@ -1206,7 +1212,7 @@ class Router(Node):
                     daemons_list.append(daemon)
 
         # Start Zebra first
-        if 'zebra' in daemons_list:
+        if "zebra" in daemons_list:
             zebra_path = os.path.join(self.daemondir, "zebra")
             zebra_option = self.daemons_options["zebra"]
             self.cmd(
@@ -1217,11 +1223,11 @@ class Router(Node):
             logger.debug("{}: {} zebra started".format(self, self.routertype))
 
             # Remove `zebra` so we don't attempt to start it again.
-            while 'zebra' in daemons_list:
-                daemons_list.remove('zebra')
+            while "zebra" in daemons_list:
+                daemons_list.remove("zebra")
 
         # Start staticd next if required
-        if 'staticd' in daemons_list:
+        if "staticd" in daemons_list:
             staticd_path = os.path.join(self.daemondir, "staticd")
             staticd_option = self.daemons_options["staticd"]
             self.cmd(
@@ -1232,8 +1238,8 @@ class Router(Node):
             logger.debug("{}: {} staticd started".format(self, self.routertype))
 
             # Remove `staticd` so we don't attempt to start it again.
-            while 'staticd' in daemons_list:
-                daemons_list.remove('staticd')
+            while "staticd" in daemons_list:
+                daemons_list.remove("staticd")
 
         # Fix Link-Local Addresses
         # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this
@@ -1256,18 +1262,18 @@ class Router(Node):
             logger.debug("{}: {} {} started".format(self, self.routertype, daemon))
 
         # Check if daemons are running.
-        rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
+        rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype)
         if re.search(r"No such file or directory", rundaemons):
             return "Daemons are not running"
 
         return ""
 
-
-    def killRouterDaemons(self, daemons, wait=True, assertOnError=True,
-                          minErrorVersion='5.1'):
+    def killRouterDaemons(
+        self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1"
+    ):
         # Kill Running Quagga or FRR specific
         # Daemons(user specified daemon only) using SIGKILL
-        rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype)
+        rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype)
         errors = ""
         daemonsNotRunning = []
         if re.search(r"No such file or directory", rundaemons):
@@ -1277,45 +1283,54 @@ class Router(Node):
                 numRunning = 0
                 for d in StringIO.StringIO(rundaemons):
                     if re.search(r"%s" % daemon, d):
-                        daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip()
-                        if (daemonpid.isdigit() and pid_exists(int(daemonpid))):
-                            logger.info('{}: killing {}'.format(
-                               self.name,
-                               os.path.basename(d.rstrip().rsplit(".", 1)[0])
-                            ))
-                            self.cmd('kill -9 %s' % daemonpid)
+                        daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip()
+                        if daemonpid.isdigit() and pid_exists(int(daemonpid)):
+                            logger.info(
+                                "{}: killing {}".format(
+                                    self.name,
+                                    os.path.basename(d.rstrip().rsplit(".", 1)[0]),
+                                )
+                            )
+                            self.cmd("kill -9 %s" % daemonpid)
                             self.waitOutput()
                             if pid_exists(int(daemonpid)):
                                 numRunning += 1
                         if wait and numRunning > 0:
-                            sleep(2, '{}: waiting for {} daemon to be stopped'.\
-                                format(self.name, daemon))
+                            sleep(
+                                2,
+                                "{}: waiting for {} daemon to be stopped".format(
+                                    self.name, daemon
+                                ),
+                            )
                             # 2nd round of kill if daemons didn't exit
                             for d in StringIO.StringIO(rundaemons):
                                 if re.search(r"%s" % daemon, d):
-                                    daemonpid = \
-                                        self.cmd('cat %s' % d.rstrip()).rstrip()
-                                    if (daemonpid.isdigit() and pid_exists(
-                                        int(daemonpid))):
-                                        logger.info('{}: killing {}'.format(
-                                            self.name,
-                                            os.path.basename(d.rstrip().\
-                                                rsplit(".", 1)[0])
-                                        ))
-                                        self.cmd('kill -9 %s' % daemonpid)
+                                    daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip()
+                                    if daemonpid.isdigit() and pid_exists(
+                                        int(daemonpid)
+                                    ):
+                                        logger.info(
+                                            "{}: killing {}".format(
+                                                self.name,
+                                                os.path.basename(
+                                                    d.rstrip().rsplit(".", 1)[0]
+                                                ),
+                                            )
+                                        )
+                                        self.cmd("kill -9 %s" % daemonpid)
                                         self.waitOutput()
-                                    self.cmd('rm -- {}'.format(d.rstrip()))
+                                    self.cmd("rm -- {}".format(d.rstrip()))
                     if wait:
                         errors = self.checkRouterCores(reportOnce=True)
-                        if self.checkRouterVersion('<', minErrorVersion):
-                            #ignore errors in old versions
+                        if self.checkRouterVersion("<", minErrorVersion):
+                            # ignore errors in old versions
                             errors = ""
                         if assertOnError and len(errors) > 0:
                             assert "Errors found - details follow:" == 0, errors
             else:
                 daemonsNotRunning.append(daemon)
         if len(daemonsNotRunning) > 0:
-            errors = errors+"Daemons are not running", daemonsNotRunning
+            errors = errors + "Daemons are not running", daemonsNotRunning
 
         return errors