Merge pull request #10301 from donaldsharp/pim_multicast_fix

tools: Give longer for interface traffic in pim to work

6 files changed, 88 insertions, 61 deletions

diff --git a/tests/topotests/lib/pim.py b/tests/topotests/lib/pim.py
index dd2f787014..1f723eab93 100644
--- a/tests/topotests/lib/pim.py
+++ b/tests/topotests/lib/pim.py
@@ -1443,16 +1443,16 @@ def verify_pim_state(
     return True
 
 
-def verify_pim_interface_traffic(tgen, input_dict):
+def get_pim_interface_traffic(tgen, input_dict):
     """
-    Verify ip pim interface traffice by running
+    get ip pim interface traffice by running
     "show ip pim interface traffic" cli
 
     Parameters
     ----------
     * `tgen`: topogen object
     * `input_dict(dict)`: defines DUT, what and from which interfaces
-                          traffic needs to be verified
+                          traffic needs to be retrieved
     Usage
     -----
     input_dict = {
@@ -1466,7 +1466,7 @@ def verify_pim_interface_traffic(tgen, input_dict):
         }
     }
 
-    result = verify_pim_interface_traffic(tgen, input_dict)
+    result = get_pim_interface_traffic(tgen, input_dict)
 
     Returns
     -------
diff --git a/tests/topotests/multicast_pim_bsm_topo1/test_mcast_pim_bsmp_01.py b/tests/topotests/multicast_pim_bsm_topo1/test_mcast_pim_bsmp_01.py
index 38560c06e4..a8418d400d 100644
--- a/tests/topotests/multicast_pim_bsm_topo1/test_mcast_pim_bsmp_01.py
+++ b/tests/topotests/multicast_pim_bsm_topo1/test_mcast_pim_bsmp_01.py
@@ -104,7 +104,7 @@ from lib.pim import (
     enable_disable_pim_bsm,
     clear_ip_mroute,
     clear_ip_pim_interface_traffic,
-    verify_pim_interface_traffic,
+    get_pim_interface_traffic,
     McastTesterHelper,
 )
 from lib.topolog import logger
@@ -648,7 +648,7 @@ def test_BSR_CRP_with_blackhole_address_p1(request):
     step("Verify bsm transit count is not increamented" "show ip pim interface traffic")
     state_dict = {"f1": {intf_f1_i1: ["bsmTx"]}}
 
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -673,7 +673,7 @@ def test_BSR_CRP_with_blackhole_address_p1(request):
         tc_name, result
     )
 
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
diff --git a/tests/topotests/multicast_pim_sm_topo1/test_multicast_pim_sm_topo1.py b/tests/topotests/multicast_pim_sm_topo1/test_multicast_pim_sm_topo1.py
index dc14bc6468..9929f4b3c7 100755
--- a/tests/topotests/multicast_pim_sm_topo1/test_multicast_pim_sm_topo1.py
+++ b/tests/topotests/multicast_pim_sm_topo1/test_multicast_pim_sm_topo1.py
@@ -84,7 +84,7 @@ from lib.pim import (
     create_igmp_config,
     verify_igmp_groups,
     verify_ip_mroutes,
-    verify_pim_interface_traffic,
+    get_pim_interface_traffic,
     verify_upstream_iif,
     verify_ip_pim_join,
     clear_ip_mroute,
@@ -280,7 +280,7 @@ def test_multicast_data_traffic_static_RP_send_join_then_traffic_p0(request):
     step("get joinRx value before join")
     intf_r2_l1 = topo["routers"]["r2"]["links"]["l1"]["interface"]
     state_dict = {"r2": {intf_r2_l1: ["joinRx"]}}
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -352,7 +352,7 @@ def test_multicast_data_traffic_static_RP_send_join_then_traffic_p0(request):
         assert result is True, "Testcase {} : Failed Error: {}".format(tc_name, result)
 
     step("joinRx value after join sent")
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -425,7 +425,7 @@ def test_multicast_data_traffic_static_RP_send_traffic_then_join_p0(request):
     step("Enable IGMP on FRR1 interface and send IGMP join (225.1.1.1)")
     step("joinRx value before join sent")
     state_dict = {"r2": {"r2-l1-eth2": ["joinRx"]}}
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -474,7 +474,7 @@ def test_multicast_data_traffic_static_RP_send_traffic_then_join_p0(request):
         assert result is True, "Testcase {} : Failed Error: {}".format(tc_name, result)
 
     step("joinRx value after join sent")
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \n Error: {}".format(
diff --git a/tests/topotests/multicast_pim_sm_topo2/test_multicast_pim_sm_topo2.py b/tests/topotests/multicast_pim_sm_topo2/test_multicast_pim_sm_topo2.py
index c7d453ad81..57561c78eb 100755
--- a/tests/topotests/multicast_pim_sm_topo2/test_multicast_pim_sm_topo2.py
+++ b/tests/topotests/multicast_pim_sm_topo2/test_multicast_pim_sm_topo2.py
@@ -82,7 +82,7 @@ from lib.pim import (
     create_igmp_config,
     verify_igmp_groups,
     verify_ip_mroutes,
-    verify_pim_interface_traffic,
+    get_pim_interface_traffic,
     verify_upstream_iif,
     verify_pim_neighbors,
     verify_pim_state,
@@ -731,7 +731,7 @@ def test_verify_SPT_switchover_when_RPT_and_SPT_path_is_different_p0(request):
 
     step("registerRx and registerStopTx value before traffic sent")
     state_dict = {"c2": {"c2-f1-eth1": ["registerRx", "registerStopTx"]}}
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \nError: {}".format(
@@ -815,7 +815,7 @@ def test_verify_SPT_switchover_when_RPT_and_SPT_path_is_different_p0(request):
         assert result is True, "Testcase {} : Failed Error: {}".format(tc_name, result)
 
     step("registerRx and registerStopTx value after traffic sent")
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \nError: {}".format(
diff --git a/tests/topotests/multicast_pim_sm_topo3/test_multicast_pim_sm_topo4.py b/tests/topotests/multicast_pim_sm_topo3/test_multicast_pim_sm_topo4.py
index 2a8e29b1d4..1fc6fddefe 100755
--- a/tests/topotests/multicast_pim_sm_topo3/test_multicast_pim_sm_topo4.py
+++ b/tests/topotests/multicast_pim_sm_topo3/test_multicast_pim_sm_topo4.py
@@ -78,11 +78,13 @@ from lib.pim import (
     verify_upstream_iif,
     clear_ip_mroute,
     verify_pim_rp_info,
-    verify_pim_interface_traffic,
+    get_pim_interface_traffic,
     McastTesterHelper,
 )
 from lib.topolog import logger
 from lib.topojson import build_config_from_json
+from time import sleep
+
 
 TOPOLOGY = """
 
@@ -930,7 +932,7 @@ def test_PIM_hello_tx_rx_p1(request):
         }
     }
 
-    c1_state_before = verify_pim_interface_traffic(tgen, state_dict)
+    c1_state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         c1_state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -946,16 +948,25 @@ def test_PIM_hello_tx_rx_p1(request):
     )
     shutdown_bringup_interface(tgen, "c1", intf_c1_l1, True)
 
-    step("verify stats after on c1")
-    c1_state_after = verify_pim_interface_traffic(tgen, state_dict)
-    assert isinstance(
-        c1_state_after, dict
-    ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
-        tc_name, result
-    )
+    step("verify stats after on c1 and that they are incremented")
+
+    count = 0
+    done = False
+    while not done and count <= 7:
+        c1_state_after = get_pim_interface_traffic(tgen, state_dict)
+        assert isinstance(
+            c1_state_after, dict
+        ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
+            tc_name, result
+        )
+
+        result = verify_state_incremented(c1_state_before, c1_state_after)
+        if result is not True:
+            sleep(5)
+            count += 1
+        else:
+            done = True
 
-    step("verify stats incremented on c1")
-    result = verify_state_incremented(c1_state_before, c1_state_after)
     assert (
         result is True
     ), "Testcase{} : Failed Error: {}" "stats is not incremented".format(
@@ -969,7 +980,7 @@ def test_PIM_hello_tx_rx_p1(request):
         }
     }
 
-    l1_state_before = verify_pim_interface_traffic(tgen, l1_state_dict)
+    l1_state_before = get_pim_interface_traffic(tgen, l1_state_dict)
     assert isinstance(
         l1_state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -985,16 +996,24 @@ def test_PIM_hello_tx_rx_p1(request):
     )
     shutdown_bringup_interface(tgen, "l1", intf_l1_c1, True)
 
-    step("verify stats after on l1")
-    l1_state_after = verify_pim_interface_traffic(tgen, l1_state_dict)
-    assert isinstance(
-        l1_state_after, dict
-    ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
-        tc_name, result
-    )
+    step("verify stats after on l1 are incremented")
+    count = 0
+    done = False
+    while not done and count <= 7:
+        l1_state_after = get_pim_interface_traffic(tgen, l1_state_dict)
+        assert isinstance(
+            l1_state_after, dict
+        ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
+            tc_name, result
+        )
+
+        result = verify_state_incremented(l1_state_before, l1_state_after)
+        if result is True:
+            sleep(5)
+            count += 1
+        else:
+            done = True
 
-    step("verify stats not incremented on l1")
-    result = verify_state_incremented(l1_state_before, l1_state_after)
     assert (
         result is not True
     ), "Testcase{} : Failed Error: {}" "stats incremented".format(tc_name, result)
@@ -1012,7 +1031,7 @@ def test_PIM_hello_tx_rx_p1(request):
         }
     }
 
-    c1_state_before = verify_pim_interface_traffic(tgen, state_dict)
+    c1_state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         c1_state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -1035,20 +1054,28 @@ def test_PIM_hello_tx_rx_p1(request):
     result = apply_raw_config(tgen, raw_config)
     assert result is True, "Testcase {} : Failed Error: {}".format(tc_name, result)
 
-    step("verify stats after on c1")
-    c1_state_after = verify_pim_interface_traffic(tgen, state_dict)
-    assert isinstance(
-        c1_state_after, dict
-    ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
+    step("verify stats after on c1 are incremented")
+    count = 0
+    done = False
+    while not done and count <= 7:
+        c1_state_after = get_pim_interface_traffic(tgen, state_dict)
+        assert isinstance(
+            c1_state_after, dict
+        ), "Testcase{} : Failed \n state_before is not dictionary \n Error: {}".format(
+            tc_name, result
+        )
+
+        result = verify_state_incremented(c1_state_before, c1_state_after)
+        if result is not True:
+            sleep(5)
+            count += 1
+        else:
+            done = True
+
+    assert result is True, "Testcase{} : Failed Error: {}" "stats incremented".format(
         tc_name, result
     )
 
-    step("verify stats incremented on c1")
-    result = verify_state_incremented(c1_state_before, c1_state_after)
-    assert (
-        result is True
-    ), "Testcase{} : Failed Error: {}" "stats incremented".format(tc_name, result)
-
     write_test_footer(tc_name)
 
 
diff --git a/tests/topotests/multicast_pim_static_rp_topo1/test_multicast_pim_static_rp.py b/tests/topotests/multicast_pim_static_rp_topo1/test_multicast_pim_static_rp.py
index 9bbe3ca028..a5cec93813 100755
--- a/tests/topotests/multicast_pim_static_rp_topo1/test_multicast_pim_static_rp.py
+++ b/tests/topotests/multicast_pim_static_rp_topo1/test_multicast_pim_static_rp.py
@@ -137,7 +137,7 @@ from lib.pim import (
     verify_join_state_and_timer,
     verify_ip_mroutes,
     verify_pim_neighbors,
-    verify_pim_interface_traffic,
+    get_pim_interface_traffic,
     verify_pim_rp_info,
     verify_pim_state,
     clear_ip_pim_interface_traffic,
@@ -386,7 +386,7 @@ def test_add_delete_static_RP_p0(request):
     step("r1: Verify show ip pim interface traffic without any IGMP join")
     state_dict = {"r1": {"r1-r2-eth1": ["pruneTx"]}}
 
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase {} : Failed \n state_before is not dictionary\n Error: {}".format(
@@ -488,7 +488,7 @@ def test_add_delete_static_RP_p0(request):
     )
 
     step("r1: Verify show ip pim interface traffic without any IGMP join")
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase {} : Failed \n state_before is not dictionary \n Error: {}".format(
@@ -699,7 +699,7 @@ def test_not_reachable_static_RP_p0(request):
         "show ip pim interface traffic"
     )
     state_dict = {"r1": {"r1-r2-eth1": ["pruneTx"]}}
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -800,7 +800,7 @@ def test_not_reachable_static_RP_p0(request):
         "r1: (*,G) prune is sent towards the RP interface, verify using"
         "show ip pim interface traffic"
     )
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -888,7 +888,7 @@ def test_add_RP_after_join_received_p1(request):
 
     step("joinTx value before join sent")
     state_dict = {"r1": {"r1-r2-eth1": ["joinTx"]}}
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -979,7 +979,7 @@ def test_add_RP_after_join_received_p1(request):
     assert result is True, "Testcase {} :Failed \n Error: {}".format(tc_name, result)
     logger.info("Expected behavior: %s", result)
 
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -1024,7 +1024,7 @@ def test_reachable_static_RP_after_join_p0(request):
 
     step("r1 : Verify pim interface traffic")
     state_dict = {"r1": {"r1-r2-eth1": ["joinTx"]}}
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -1123,7 +1123,7 @@ def test_reachable_static_RP_after_join_p0(request):
     logger.info("Expected behavior: %s", result)
 
     step("r1 : Verify pim interface traffic")
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -1211,7 +1211,7 @@ def test_send_join_on_higher_preffered_rp_p1(request):
     step("r1 : Verify joinTx count before sending join")
     state_dict = {"r1": {"r1-r4-eth3": ["joinTx"], "r1-r2-eth1": ["pruneTx"]}}
 
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -1261,7 +1261,7 @@ def test_send_join_on_higher_preffered_rp_p1(request):
 
     step("r1 : Verify join is sent to higher preferred RP")
     step("r1 : Verify prune is sent to lower preferred RP")
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -1293,7 +1293,7 @@ def test_send_join_on_higher_preffered_rp_p1(request):
     step("r1 : Verify joinTx, pruneTx count before RP gets deleted")
     state_dict = {"r1": {"r1-r2-eth1": ["joinTx"], "r1-r4-eth3": ["pruneTx"]}}
 
-    state_before = verify_pim_interface_traffic(tgen, state_dict)
+    state_before = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_before, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(
@@ -1375,7 +1375,7 @@ def test_send_join_on_higher_preffered_rp_p1(request):
         "r1 : Verify prune is sent to higher preferred RP when higher"
         " preferred RP gets deleted"
     )
-    state_after = verify_pim_interface_traffic(tgen, state_dict)
+    state_after = get_pim_interface_traffic(tgen, state_dict)
     assert isinstance(
         state_after, dict
     ), "Testcase{} : Failed \n state_before is not dictionary \n " "Error: {}".format(