From a99a6a026ecd45d986274f4e3efca31183ebd182 Mon Sep 17 00:00:00 2001 From: Martin Winter Date: Sat, 26 Nov 2016 14:50:57 -0800 Subject: [PATCH] ospf6-topo1: Enable coredumps for Quagga Daemons and add logic to skip remaining test in case of fatal error --- .../topotests/ospf6-topo1/test_ospf6_topo1.py | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/tests/topotests/ospf6-topo1/test_ospf6_topo1.py b/tests/topotests/ospf6-topo1/test_ospf6_topo1.py index e7d0b1ab9f..12bcbe332c 100755 --- a/tests/topotests/ospf6-topo1/test_ospf6_topo1.py +++ b/tests/topotests/ospf6-topo1/test_ospf6_topo1.py @@ -75,6 +75,8 @@ import re import sys import difflib import StringIO +import glob +import subprocess from mininet.topo import Topo from mininet.net import Mininet @@ -87,6 +89,8 @@ from time import sleep import pytest +fatal_error = "" + def int2dpid(dpid): "Converting Integer to DPID" @@ -123,6 +127,12 @@ class QuaggaRouter(Node): # Enable forwarding on the router self.cmd('sysctl net.ipv4.ip_forward=1') self.cmd('sysctl net.ipv6.conf.all.forwarding=1') + # Enable coredumps + self.cmd('sysctl kernel.core_uses_pid=1') + self.cmd('sysctl fs.suid_dumpable=2') + self.cmd("sysctl kernel.core_pattern=/tmp/%s_%%e_core-sig_%%s-pid_%%p.dmp" % self.name) + self.cmd('ulimit -c unlimited') + # Set ownership of config files self.cmd('chown quagga:quaggavty /etc/quagga') self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0, 'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0} @@ -162,6 +172,10 @@ class QuaggaRouter(Node): with open("/etc/quagga/vtysh.conf", "w") as vtyshfile: vtyshfile.write('no service integrated-vtysh-config') self.cmd('chown quagga:quaggavty /etc/quagga/vtysh.conf') + # Try to find relevant old logfiles in /tmp and delete them + map(os.remove, glob.glob("/tmp/*%s*.log" % self.name)) + # Remove old core files + map(os.remove, glob.glob("/tmp/%s*.dmp" % self.name)) # Remove IP addresses from OS first - we have them in zebra.conf self.removeIPs() # Start Zebra first @@ -180,10 +194,27 @@ class QuaggaRouter(Node): self.waitOutput() print('%s: %s started' % (self, daemon)) def checkQuaggaRunning(self): + global fatal_error + daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"') for daemon in self.daemons: - if (self.daemons[daemon] == 1): - assert daemon in daemonsRunning, "Daemon %s not running" % daemon + if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning): + sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon)) + # Look for core file + corefiles = glob.glob("/tmp/%s_%s_core*.dmp" % (self.name, daemon)) + if (len(corefiles) > 0): + backtrace = subprocess.check_output(["gdb /usr/lib/quagga/%s %s --batch -ex bt 2> /dev/null" % (daemon, corefiles[0])], shell=True) + sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon)) + sys.stderr.write("%s\n" % backtrace) + else: + # No core found - If we find matching logfile in /tmp, then print last 20 lines from it. + if os.path.isfile("/tmp/%s-%s.log" % (self.name, daemon)): + log_tail = subprocess.check_output(["tail -n20 /tmp/%s-%s.log 2> /dev/null" % (self.name, daemon)], shell=True) + sys.stderr.write("\nFrom quagga %s %s log file:\n" % (self.name, daemon)) + sys.stderr.write("%s\n" % log_tail) + + fatal_error = "%s: Daemon %s not running" % (self.name, daemon) + assert False, "%s: Daemon %s not running" % (self.name, daemon) class LegacySwitch(OVSSwitch): @@ -281,8 +312,13 @@ def teardown_module(module): def test_quagga_running(): + global fatal_error global net + # Skip if previous fatal error condition is raised + if (fatal_error != ""): + pytest.skip(fatal_error) + print("\n\n** Check if Quagga is running on each Router node") print("******************************************\n") sleep(5) @@ -293,8 +329,13 @@ def test_quagga_running(): def test_ospf6_converged(): + global fatal_error global net + # Skip if previous fatal error condition is raised + if (fatal_error != ""): + pytest.skip(fatal_error) + # Wait for OSPF6 to converge (All Neighbors in either Full or TwoWay State) print("\n\n** Verify for OSPF6 daemons to converge") print("******************************************\n") @@ -319,7 +360,7 @@ def test_ospf6_converged(): else: # Bail out with error if a router fails to converge ospfStatus = net['r%s' % i].cmd('vtysh -c "show ipv6 ospf neigh"') - + fatal_error = "OSPFv6 did not converge" assert False, "OSPFv6 did not converge:\n%s" % ospfStatus print("OSPFv3 converged.") @@ -330,8 +371,13 @@ def test_ospf6_converged(): sleep(15) def test_ospf6_routingTable(): + global fatal_error global net + # Skip if previous fatal error condition is raised + if (fatal_error != ""): + pytest.skip(fatal_error) + thisDir = os.path.dirname(os.path.realpath(__file__)) # Verify OSPFv3 Routing Table @@ -367,9 +413,14 @@ def test_ospf6_routingTable(): assert failures == 0, "Routing Table verification failed for router r%s:\n%s" % (i, diff) + # For debugging after starting Quagga daemons, uncomment the next line + # CLI(net) + if __name__ == '__main__': setLogLevel('info') + # To suppress tracebacks, either use the following pytest call or add "--tb=no" to cli + # retval = pytest.main(["-s", "--tb=no"]) retval = pytest.main(["-s"]) sys.exit(retval) -- 2.39.5