From 93d664c26a4657f69d6e6f94f67254312af1fff5 Mon Sep 17 00:00:00 2001 From: Donald Sharp Date: Mon, 29 Nov 2021 08:37:21 -0500 Subject: [PATCH] test: Fix addKernelRoute looking for positive results Under heavy system load, we are sometimes seeing this output for addKernelRoute: 2021-11-28 16:17:27,604 INFO: topolog: [DUT: b1]: Running command: [ip route add 224.0.0.13 dev b1-f1-eth0] 2021-11-28 16:17:27,604 DEBUG: topolog.b1: LinuxNamespace(b1): cmd_status("['/bin/bash', '-c', 'ip route add 224.0.0.13 dev b1-f1-eth0']", kwargs: {'encoding': 'utf-8', 'stdout': -1, 'stderr': -2, 'shell': False, 'stdin': None}) 2021-11-28 16:17:27,967 DEBUG: topolog.b1: LinuxNamespace(b1): cmd_status("['/bin/bash', '-c', 'ip route']", kwargs: {'encoding': 'utf-8', 'stdout': -1, 'stderr': -2, 'shell': False, 'stdin': None}) 2021-11-28 16:17:28,243 DEBUG: topolog: ip route 70.0.0.0/24 dev b1-f1-eth0 proto kernel scope link src 70.0.0.1 Signed-off-by: Donald Sharp This tells us that the ip route add succeeded but when looking for it the system failed to immediately find it. Why is this happening? Probably we are under heavy system load and the two different commands, 'ip route add..' and 'ip route show' are being executed on different cpu's and the data has not been copied to the different cpu yet in the kernel. This is not necessarily something normally seen but entirely possible. Giving the system a few extra seconds for the kernel to execute/work the memory barrier system seems prudent for long term success of our programming. Signed-off-by: Donald Sharp --- tests/topotests/lib/common_config.py | 39 ++++++++++++++++++---------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/tests/topotests/lib/common_config.py b/tests/topotests/lib/common_config.py index cf8efdea16..c744e5bbbf 100644 --- a/tests/topotests/lib/common_config.py +++ b/tests/topotests/lib/common_config.py @@ -26,6 +26,7 @@ import socket import subprocess import sys import traceback +import functools from collections import OrderedDict from copy import deepcopy from datetime import datetime, timedelta @@ -2970,24 +2971,34 @@ def addKernelRoute( logger.info("[DUT: {}]: Running command: [{}]".format(router, cmd)) output = rnode.run(cmd) - # Verifying if ip route added to kernal - result = rnode.run(verify_cmd) - logger.debug("{}\n{}".format(verify_cmd, result)) - if "/" in grp_addr: - ip, mask = grp_addr.split("/") - if mask == "32" or mask == "128": - grp_addr = ip - else: - mask = "32" if addr_type == "ipv4" else "128" + def check_in_kernel(rnode, verify_cmd, grp_addr, router): + # Verifying if ip route added to kernal + errormsg = None + result = rnode.run(verify_cmd) + logger.debug("{}\n{}".format(verify_cmd, result)) + if "/" in grp_addr: + ip, mask = grp_addr.split("/") + if mask == "32" or mask == "128": + grp_addr = ip + else: + mask = "32" if addr_type == "ipv4" else "128" - if not re_search(r"{}".format(grp_addr), result) and mask != "0": - errormsg = ( - "[DUT: {}]: Kernal route is not added for group" - " address {} Config output: {}".format(router, grp_addr, output) - ) + if not re_search(r"{}".format(grp_addr), result) and mask != "0": + errormsg = ( + "[DUT: {}]: Kernal route is not added for group" + " address {} Config output: {}".format( + router, grp_addr, output + ) + ) return errormsg + test_func = functools.partial( + check_in_kernel, rnode, verify_cmd, grp_addr, router + ) + (result, out) = topotest.run_and_expect(test_func, None, count=20, wait=1) + assert result, out + logger.debug("Exiting lib API: addKernelRoute()") return True -- 2.39.5