ospf6-topo1: Enable coredumps for Quagga Daemons and add logic to skip remaining...

diff --git a/tests/topotests/ospf6-topo1/test_ospf6_topo1.py b/tests/topotests/ospf6-topo1/test_ospf6_topo1.py
index e7d0b1ab9f867edaf2a183a3a2ffd85cfc3e972b..12bcbe332c8fedab7fa63e4aa913e09e2b359463 100755 (executable)
--- a/tests/topotests/ospf6-topo1/test_ospf6_topo1.py
+++ b/tests/topotests/ospf6-topo1/test_ospf6_topo1.py
@@ -75,6 +75,8 @@ import re
 import sys
 import difflib
 import StringIO
+import glob
+import subprocess
 
 from mininet.topo import Topo
 from mininet.net import Mininet
@@ -87,6 +89,8 @@ from time import sleep
 
 import pytest
 
+fatal_error = ""
+
 def int2dpid(dpid):
     "Converting Integer to DPID"
 
@@ -123,6 +127,12 @@ class QuaggaRouter(Node):
         # Enable forwarding on the router
         self.cmd('sysctl net.ipv4.ip_forward=1')
         self.cmd('sysctl net.ipv6.conf.all.forwarding=1')
+        # Enable coredumps
+        self.cmd('sysctl kernel.core_uses_pid=1')
+        self.cmd('sysctl fs.suid_dumpable=2')
+        self.cmd("sysctl kernel.core_pattern=/tmp/%s_%%e_core-sig_%%s-pid_%%p.dmp" % self.name)
+        self.cmd('ulimit -c unlimited')
+        # Set ownership of config files
         self.cmd('chown quagga:quaggavty /etc/quagga')
         self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0,
                         'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0}
@@ -162,6 +172,10 @@ class QuaggaRouter(Node):
         with open("/etc/quagga/vtysh.conf", "w") as vtyshfile:
             vtyshfile.write('no service integrated-vtysh-config')
         self.cmd('chown quagga:quaggavty /etc/quagga/vtysh.conf')
+        # Try to find relevant old logfiles in /tmp and delete them
+        map(os.remove, glob.glob("/tmp/*%s*.log" % self.name))
+        # Remove old core files
+        map(os.remove, glob.glob("/tmp/%s*.dmp" % self.name))
         # Remove IP addresses from OS first - we have them in zebra.conf
         self.removeIPs()
         # Start Zebra first
@@ -180,10 +194,27 @@ class QuaggaRouter(Node):
                 self.waitOutput()
                 print('%s: %s started' % (self, daemon))
     def checkQuaggaRunning(self):
+        global fatal_error
+
         daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"')
         for daemon in self.daemons:
-            if (self.daemons[daemon] == 1):
-                assert daemon in daemonsRunning, "Daemon %s not running" % daemon
+            if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning):
+                sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon))
+                # Look for core file
+                corefiles = glob.glob("/tmp/%s_%s_core*.dmp" % (self.name, daemon))
+                if (len(corefiles) > 0):
+                    backtrace = subprocess.check_output(["gdb /usr/lib/quagga/%s %s --batch -ex bt 2> /dev/null"  % (daemon, corefiles[0])], shell=True)
+                    sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon))
+                    sys.stderr.write("%s\n" % backtrace)
+                else:
+                    # No core found - If we find matching logfile in /tmp, then print last 20 lines from it.
+                    if os.path.isfile("/tmp/%s-%s.log" % (self.name, daemon)):
+                        log_tail = subprocess.check_output(["tail -n20 /tmp/%s-%s.log 2> /dev/null"  % (self.name, daemon)], shell=True)
+                        sys.stderr.write("\nFrom quagga %s %s log file:\n" % (self.name, daemon))
+                        sys.stderr.write("%s\n" % log_tail)
+
+                fatal_error = "%s: Daemon %s not running" % (self.name, daemon)
+                assert False, "%s: Daemon %s not running" % (self.name, daemon)
 
 
 class LegacySwitch(OVSSwitch):
@@ -281,8 +312,13 @@ def teardown_module(module):
 
 
 def test_quagga_running():
+    global fatal_error
     global net
 
+    # Skip if previous fatal error condition is raised
+    if (fatal_error != ""):
+        pytest.skip(fatal_error)
+
     print("\n\n** Check if Quagga is running on each Router node")
     print("******************************************\n")
     sleep(5)
@@ -293,8 +329,13 @@ def test_quagga_running():
 
 
 def test_ospf6_converged():
+    global fatal_error
     global net
 
+    # Skip if previous fatal error condition is raised
+    if (fatal_error != ""):
+        pytest.skip(fatal_error)
+
     # Wait for OSPF6 to converge  (All Neighbors in either Full or TwoWay State)
     print("\n\n** Verify for OSPF6 daemons to converge")
     print("******************************************\n")
@@ -319,7 +360,7 @@ def test_ospf6_converged():
     else:
         # Bail out with error if a router fails to converge
         ospfStatus = net['r%s' % i].cmd('vtysh -c "show ipv6 ospf neigh"')
-
+        fatal_error = "OSPFv6 did not converge"
         assert False, "OSPFv6 did not converge:\n%s" % ospfStatus
 
     print("OSPFv3 converged.")
@@ -330,8 +371,13 @@ def test_ospf6_converged():
         sleep(15)
 
 def test_ospf6_routingTable():
+    global fatal_error
     global net
 
+    # Skip if previous fatal error condition is raised
+    if (fatal_error != ""):
+        pytest.skip(fatal_error)
+
     thisDir = os.path.dirname(os.path.realpath(__file__))
 
     # Verify OSPFv3 Routing Table
@@ -367,9 +413,14 @@ def test_ospf6_routingTable():
 
             assert failures == 0, "Routing Table verification failed for router r%s:\n%s" % (i, diff)
 
+    # For debugging after starting Quagga daemons, uncomment the next line
+    # CLI(net)
+
 
 if __name__ == '__main__':
 
     setLogLevel('info')
+    # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli
+    # retval = pytest.main(["-s", "--tb=no"])
     retval = pytest.main(["-s"])
     sys.exit(retval)