Merge pull request #16827 from LabNConsulting/chopps/fix-on-error-with-xdist

improvements for xdist mode

1 files changed, 17 insertions, 9 deletions

diff --git a/tests/topotests/munet/testing/util.py b/tests/topotests/munet/testing/util.py
index a1a94bcd1b..99687c0a83 100644
--- a/tests/topotests/munet/testing/util.py
+++ b/tests/topotests/munet/testing/util.py
@@ -52,12 +52,13 @@ def pause_test(desc=""):
     asyncio.run(async_pause_test(desc))
 
 
-def retry(retry_timeout, initial_wait=0, expected=True):
+def retry(retry_timeout, initial_wait=0, retry_sleep=2, expected=True):
     """decorator: retry while functions return is not None or raises an exception.
 
     * `retry_timeout`: Retry for at least this many seconds; after waiting
                        initial_wait seconds
     * `initial_wait`: Sleeps for this many seconds before first executing function
+    * `retry_sleep`: The time to sleep between retries.
     * `expected`: if False then the return logic is inverted, except for exceptions,
                   (i.e., a non None ends the retry loop, and returns that value)
     """
@@ -65,9 +66,8 @@ def retry(retry_timeout, initial_wait=0, expected=True):
     def _retry(func):
         @functools.wraps(func)
         def func_retry(*args, **kwargs):
-            retry_sleep = 2
-
             # Allow the wrapped function's args to override the fixtures
+            _retry_sleep = float(kwargs.pop("retry_sleep", retry_sleep))
             _retry_timeout = kwargs.pop("retry_timeout", retry_timeout)
             _expected = kwargs.pop("expected", expected)
             _initial_wait = kwargs.pop("initial_wait", initial_wait)
@@ -82,13 +82,21 @@ def retry(retry_timeout, initial_wait=0, expected=True):
             while True:
                 seconds_left = (retry_until - datetime.datetime.now()).total_seconds()
                 try:
-                    ret = func(*args, **kwargs)
-                    if _expected and ret is None:
+                    try:
+                        ret = func(*args, seconds_left=seconds_left, **kwargs)
+                    except TypeError as error:
+                        if "seconds_left" not in str(error):
+                            raise
+                        ret = func(*args, **kwargs)
+
+                    logging.debug("Function returned %s", ret)
+
+                    positive_result = ret is None
+                    if _expected == positive_result:
                         logging.debug("Function succeeds")
                         return ret
-                    logging.debug("Function returned %s", ret)
                 except Exception as error:
-                    logging.info("Function raised exception: %s", str(error))
+                    logging.info('Function raised exception: "%s"', error)
                     ret = error
 
                 if seconds_left < 0:
@@ -99,10 +107,10 @@ def retry(retry_timeout, initial_wait=0, expected=True):
 
                 logging.info(
                     "Sleeping %ds until next retry with %.1f retry time left",
-                    retry_sleep,
+                    _retry_sleep,
                     seconds_left,
                 )
-                time.sleep(retry_sleep)
+                time.sleep(_retry_sleep)
 
         func_retry._original = func  # pylint: disable=W0212
         return func_retry