diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/topotests/Dockerfile | 6 | ||||
| -rwxr-xr-x | tests/topotests/bgp-ecmp-topo2/test_ebgp_ecmp_topo2.py | 12 | ||||
| -rwxr-xr-x | tests/topotests/bgp-ecmp-topo2/test_ibgp_ecmp_topo2.py | 12 | ||||
| -rw-r--r-- | tests/topotests/lib/bgp.py | 21 | ||||
| -rw-r--r-- | tests/topotests/lib/common_config.py | 6 | ||||
| -rw-r--r-- | tests/topotests/lib/topotest.py | 125 |
6 files changed, 111 insertions, 71 deletions
diff --git a/tests/topotests/Dockerfile b/tests/topotests/Dockerfile index cdd0ae2f6e..b7c6298228 100644 --- a/tests/topotests/Dockerfile +++ b/tests/topotests/Dockerfile @@ -19,6 +19,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ libjson-c-dev \ libpcre3-dev \ libpython-dev \ + libpython3-dev \ libreadline-dev \ libc-ares-dev \ libcap-dev \ @@ -26,7 +27,10 @@ RUN export DEBIAN_FRONTEND=noninteractive \ mininet \ pkg-config \ python-pip \ - python-sphinx \ + python3 \ + python3-dev \ + python3-sphinx \ + python3-pytest \ rsync \ strace \ tcpdump \ diff --git a/tests/topotests/bgp-ecmp-topo2/test_ebgp_ecmp_topo2.py b/tests/topotests/bgp-ecmp-topo2/test_ebgp_ecmp_topo2.py index 087ba21e5e..948f641afb 100755 --- a/tests/topotests/bgp-ecmp-topo2/test_ebgp_ecmp_topo2.py +++ b/tests/topotests/bgp-ecmp-topo2/test_ebgp_ecmp_topo2.py @@ -63,7 +63,7 @@ from lib.common_config import ( reset_config_on_routers, ) from lib.topolog import logger -from lib.bgp import verify_bgp_convergence, create_router_bgp, clear_bgp_and_verify +from lib.bgp import verify_bgp_convergence, create_router_bgp, clear_bgp from lib.topojson import build_topo_from_json, build_config_from_json # Reading the data from JSON File for topology and configuration creation @@ -295,7 +295,7 @@ def test_modify_ecmp_max_paths(request, ecmp_num, test_type): addr_type, dut, input_dict_1, - next_hop=NEXT_HOPS[addr_type][:int(ecmp_num)], + next_hop=NEXT_HOPS[addr_type][: int(ecmp_num)], protocol=protocol, ) assert result is True, "Testcase {} : Failed \n Error: {}".format( @@ -336,8 +336,12 @@ def test_ecmp_after_clear_bgp(request, test_type): tc_name, result ) - # Clear bgp - result = clear_bgp_and_verify(tgen, topo, dut) + # Clear BGP + for addr_type in ADDR_TYPES: + clear_bgp(tgen, addr_type, dut) + + # Verify BGP convergence + result = verify_bgp_convergence(tgen, topo) assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result) for addr_type in ADDR_TYPES: diff --git a/tests/topotests/bgp-ecmp-topo2/test_ibgp_ecmp_topo2.py b/tests/topotests/bgp-ecmp-topo2/test_ibgp_ecmp_topo2.py index 94409ff3e1..5b997fdd16 100755 --- a/tests/topotests/bgp-ecmp-topo2/test_ibgp_ecmp_topo2.py +++ b/tests/topotests/bgp-ecmp-topo2/test_ibgp_ecmp_topo2.py @@ -63,7 +63,7 @@ from lib.common_config import ( reset_config_on_routers, ) from lib.topolog import logger -from lib.bgp import verify_bgp_convergence, create_router_bgp, clear_bgp_and_verify +from lib.bgp import verify_bgp_convergence, create_router_bgp, clear_bgp from lib.topojson import build_topo_from_json, build_config_from_json # Reading the data from JSON File for topology and configuration creation @@ -296,7 +296,7 @@ def test_modify_ecmp_max_paths(request, ecmp_num, test_type): addr_type, dut, input_dict_1, - next_hop=NEXT_HOPS[addr_type][:int(ecmp_num)], + next_hop=NEXT_HOPS[addr_type][: int(ecmp_num)], protocol=protocol, ) assert result is True, "Testcase {} : Failed \n Error: {}".format( @@ -337,8 +337,12 @@ def test_ecmp_after_clear_bgp(request, test_type): tc_name, result ) - # Clear bgp - result = clear_bgp_and_verify(tgen, topo, dut) + # Clear BGP + for addr_type in ADDR_TYPES: + clear_bgp(tgen, addr_type, dut) + + # Verify BGP convergence + result = verify_bgp_convergence(tgen, topo) assert result is True, "Testcase {} : Failed \n Error: {}".format(tc_name, result) for addr_type in ADDR_TYPES: diff --git a/tests/topotests/lib/bgp.py b/tests/topotests/lib/bgp.py index 44b7335058..971bbd0f3b 100644 --- a/tests/topotests/lib/bgp.py +++ b/tests/topotests/lib/bgp.py @@ -44,7 +44,6 @@ from lib.common_config import ( LOGDIR = "/tmp/topotests/" TMPDIR = None - def create_router_bgp(tgen, topo, input_dict=None, build=False, load_config=True): """ API to configure bgp on router @@ -882,7 +881,7 @@ def verify_router_id(tgen, topo, input_dict): return True -@retry(attempts=44, wait=3, return_is_str=True) +@retry(attempts=50, wait=3, return_is_str=True) def verify_bgp_convergence(tgen, topo, dut=None): """ API will verify if BGP is converged with in the given time frame. @@ -1052,11 +1051,13 @@ def verify_bgp_convergence(tgen, topo, dut=None): if nh_state == "Established": no_of_peer += 1 - if no_of_peer == total_peer: - logger.info("[DUT: %s] VRF: %s, BGP is Converged", router, vrf) - else: - errormsg = "[DUT: %s] VRF: %s, BGP is not converged" % (router, vrf) - return errormsg + if no_of_peer == total_peer: + logger.info("[DUT: %s] VRF: %s, BGP is Converged for %s address-family", + router, vrf, addr_type) + else: + errormsg = ("[DUT: %s] VRF: %s, BGP is not converged for %s address-family" % + (router, vrf, addr_type)) + return errormsg logger.debug("Exiting API: verify_bgp_convergence()") return True @@ -1326,7 +1327,7 @@ def verify_as_numbers(tgen, topo, input_dict): return True -@retry(attempts=44, wait=3, return_is_str=True) +@retry(attempts=50, wait=3, return_is_str=True) def verify_bgp_convergence_from_running_config(tgen, dut=None): """ API to verify BGP convergence b/w loopback and physical interface. @@ -1470,7 +1471,7 @@ def clear_bgp_and_verify(tgen, topo, router): sleeptime = 3 # Verifying BGP convergence before bgp clear command - for retry in range(44): + for retry in range(50): # Waiting for BGP to converge logger.info( "Waiting for %s sec for BGP to converge on router" " %s...", @@ -1552,7 +1553,7 @@ def clear_bgp_and_verify(tgen, topo, router): peer_uptime_after_clear_bgp = {} # Verifying BGP convergence after bgp clear command - for retry in range(44): + for retry in range(50): # Waiting for BGP to converge logger.info( diff --git a/tests/topotests/lib/common_config.py b/tests/topotests/lib/common_config.py index fb82b50628..d72d0aa223 100644 --- a/tests/topotests/lib/common_config.py +++ b/tests/topotests/lib/common_config.py @@ -690,6 +690,12 @@ def start_topology(tgen): router_list = tgen.routers() for rname in ROUTER_LIST: router = router_list[rname] + + # It will help in debugging the failures, will give more details on which + # specific kernel version tests are failing + linux_ver = router.run("uname -a") + logger.info("Logging platform related details: \n %s \n", linux_ver) + try: os.chdir(TMPDIR) diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index bffb8208e7..b5fa2ea59b 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -35,6 +35,7 @@ import tempfile import platform import difflib import time +import signal from lib.topolog import logger from copy import deepcopy @@ -51,6 +52,35 @@ from mininet.log import setLogLevel, info from mininet.cli import CLI from mininet.link import Intf +def gdb_core(obj, daemon, corefiles): + gdbcmds = ''' + info threads + bt full + disassemble + up + disassemble + up + disassemble + up + disassemble + up + disassemble + up + disassemble + ''' + gdbcmds = [['-ex', i.strip()] for i in gdbcmds.strip().split('\n')] + gdbcmds = [item for sl in gdbcmds for item in sl] + + daemon_path = os.path.join(obj.daemondir, daemon) + backtrace = subprocess.check_output( + ['gdb', daemon_path, corefiles[0], '--batch'] + gdbcmds + ) + sys.stderr.write( + "\n%s: %s crashed. Core file found - Backtrace follows:\n" + % (obj.name, daemon) + ) + sys.stderr.write("%s" % backtrace) + return backtrace class json_cmp_result(object): "json_cmp result class for better assertion messages" @@ -422,6 +452,10 @@ def pid_exists(pid): if pid <= 0: return False try: + os.waitpid(pid, os.WNOHANG) + except: + pass + try: os.kill(pid, 0) except OSError as err: if err.errno == errno.ESRCH: @@ -992,8 +1026,8 @@ class Router(Node): os.system("chmod -R go+rw /tmp/topotests") # Return count of running daemons - def countDaemons(self): - numRunning = 0 + def listDaemons(self): + ret = [] rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype) errors = "" if re.search(r"No such file or directory", rundaemons): @@ -1002,12 +1036,11 @@ class Router(Node): for d in StringIO.StringIO(rundaemons): daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip() if daemonpid.isdigit() and pid_exists(int(daemonpid)): - numRunning += 1 - return numRunning + ret.append(os.path.basename(d.rstrip().rsplit(".", 1)[0])) + return ret def stopRouter(self, wait=True, assertOnError=True, minErrorVersion="5.1"): # Stop Running FRR Daemons - numRunning = 0 rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype) errors = "" if re.search(r"No such file or directory", rundaemons): @@ -1016,24 +1049,36 @@ class Router(Node): for d in StringIO.StringIO(rundaemons): daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip() if daemonpid.isdigit() and pid_exists(int(daemonpid)): + daemonname = os.path.basename(d.rstrip().rsplit(".", 1)[0]) logger.info( "{}: stopping {}".format( - self.name, os.path.basename(d.rstrip().rsplit(".", 1)[0]) + self.name, daemonname ) ) - self.cmd("kill -TERM %s" % daemonpid) - self.waitOutput() - if pid_exists(int(daemonpid)): - numRunning += 1 - - if wait and numRunning > 0: - counter = 5 - while counter > 0 and numRunning > 0: - sleep(2, "{}: waiting for daemons stopping".format(self.name)) - numRunning = self.countDaemons() + try: + os.kill(int(daemonpid), signal.SIGTERM) + except OSError as err: + if err.errno == errno.ESRCH: + logger.error("{}: {} left a dead pidfile (pid={})".format(self.name, daemonname, daemonpid)) + else: + logger.info("{}: {} could not kill pid {}: {}".format(self.name, daemonname, daemonpid, str(err))) + + if not wait: + return errors + + running = self.listDaemons() + + if running: + sleep(0.1, "{}: waiting for daemons stopping: {}".format(self.name, ', '.join(running))) + running = self.listDaemons() + + counter = 20 + while counter > 0 and running: + sleep(0.5, "{}: waiting for daemons stopping: {}".format(self.name, ', '.join(running))) + running = self.listDaemons() counter -= 1 - if wait and numRunning > 0: + if running: # 2nd round of kill if daemons didn't exit for d in StringIO.StringIO(rundaemons): daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip() @@ -1048,13 +1093,15 @@ class Router(Node): self.waitOutput() self.cmd("rm -- {}".format(d.rstrip())) - if wait: - errors = self.checkRouterCores(reportOnce=True) - if self.checkRouterVersion("<", minErrorVersion): - # ignore errors in old versions - errors = "" - if assertOnError and len(errors) > 0: - assert "Errors found - details follow:" == 0, errors + if not wait: + return errors + + errors = self.checkRouterCores(reportOnce=True) + if self.checkRouterVersion("<", minErrorVersion): + # ignore errors in old versions + errors = "" + if assertOnError and len(errors) > 0: + assert "Errors found - details follow:" == 0, errors return errors def removeIPs(self): @@ -1348,20 +1395,7 @@ class Router(Node): "{}/{}/{}_core*.dmp".format(self.logdir, self.name, daemon) ) if len(corefiles) > 0: - daemon_path = os.path.join(self.daemondir, daemon) - backtrace = subprocess.check_output( - [ - "gdb {} {} --batch -ex bt 2> /dev/null".format( - daemon_path, corefiles[0] - ) - ], - shell=True, - ) - sys.stderr.write( - "\n%s: %s crashed. Core file found - Backtrace follows:\n" - % (self.name, daemon) - ) - sys.stderr.write("%s" % backtrace) + backtrace = gdb_core(self, daemon, corefiles) traces = ( traces + "\n%s: %s crashed. Core file found - Backtrace follows:\n%s" @@ -1431,20 +1465,7 @@ class Router(Node): "{}/{}/{}_core*.dmp".format(self.logdir, self.name, daemon) ) if len(corefiles) > 0: - daemon_path = os.path.join(self.daemondir, daemon) - backtrace = subprocess.check_output( - [ - "gdb {} {} --batch -ex bt 2> /dev/null".format( - daemon_path, corefiles[0] - ) - ], - shell=True, - ) - sys.stderr.write( - "\n%s: %s crashed. Core file found - Backtrace follows:\n" - % (self.name, daemon) - ) - sys.stderr.write("%s\n" % backtrace) + gdb_core(self, daemon, corefiles) else: # No core found - If we find matching logfile in /tmp, then print last 20 lines from it. if os.path.isfile( |