diff options
Diffstat (limited to 'tests/topotests/lib/topotest.py')
| -rw-r--r-- | tests/topotests/lib/topotest.py | 93 |
1 files changed, 75 insertions, 18 deletions
diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index 4b18862101..7047c52b18 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -910,31 +910,63 @@ def sleep(amount, reason=None): time.sleep(amount) -def checkAddressSanitizerError(output, router, component): +def checkAddressSanitizerError(output, router, component, logdir=""): "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise" - addressSantizerError = re.search( - "(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", output - ) - if addressSantizerError: + def processAddressSanitizerError(asanErrorRe, output, router, component): sys.stderr.write( "%s: %s triggered an exception by AddressSanitizer\n" % (router, component) ) # Sanitizer Error found in log - pidMark = addressSantizerError.group(1) + pidMark = asanErrorRe.group(1) addressSantizerLog = re.search( "%s(.*)%s" % (pidMark, pidMark), output, re.DOTALL ) if addressSantizerLog: - callingTest = os.path.basename( - sys._current_frames().values()[0].f_back.f_back.f_globals["__file__"] - ) - callingProc = sys._getframe(2).f_code.co_name + # Find Calling Test. Could be multiple steps back + testframe=sys._current_frames().values()[0] + level=0 + while level < 10: + test=os.path.splitext(os.path.basename(testframe.f_globals["__file__"]))[0] + if (test != "topotest") and (test != "topogen"): + # Found the calling test + callingTest=os.path.basename(testframe.f_globals["__file__"]) + break + level=level+1 + testframe=testframe.f_back + if (level >= 10): + # somehow couldn't find the test script. + callingTest="unknownTest" + # + # Now finding Calling Procedure + level=0 + while level < 20: + callingProc=sys._getframe(level).f_code.co_name + if ((callingProc != "processAddressSanitizerError") and + (callingProc != "checkAddressSanitizerError") and + (callingProc != "checkRouterCores") and + (callingProc != "stopRouter") and + (callingProc != "__stop_internal") and + (callingProc != "stop") and + (callingProc != "stop_topology") and + (callingProc != "checkRouterRunning") and + (callingProc != "check_router_running") and + (callingProc != "routers_have_failure")): + # Found the calling test + break + level=level+1 + if (level >= 20): + # something wrong - couldn't found the calling test function + callingProc="unknownProc" with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile: sys.stderr.write( + "AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" + % (callingTest, callingProc, router) + ) + sys.stderr.write( "\n".join(addressSantizerLog.group(1).splitlines()) + "\n" ) - addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2)) + addrSanFile.write("## Error: %s\n\n" % asanErrorRe.group(2)) addrSanFile.write( "### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, router) @@ -945,7 +977,29 @@ def checkAddressSanitizerError(output, router, component): + "\n" ) addrSanFile.write("\n---------------\n") + return + + + addressSantizerError = re.search( + "(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", output + ) + if addressSantizerError: + processAddressSanitizerError(addressSantizerError, output, router, component) return True + + # No Address Sanitizer Error in Output. Now check for AddressSanitizer daemon file + if logdir: + filepattern=logdir+"/"+router+"/"+component+".asan.*" + sys.stderr.write("Log check for %s on %s, pattern %s\n" % (component, router, filepattern)) + for file in glob.glob(filepattern): + with open(file, "r") as asanErrorFile: + asanError=asanErrorFile.read() + addressSantizerError = re.search( + "(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", asanError + ) + if addressSantizerError: + processAddressSanitizerError(addressSantizerError, asanError, router, component) + return True return False @@ -1011,7 +1065,7 @@ class Router(Node): if self.logdir is None: cur_test = os.environ["PYTEST_CURRENT_TEST"] self.logdir = "/tmp/topotests/" + cur_test[ - 0 : cur_test.find(".py") + cur_test.find("/")+1 : cur_test.find(".py") ].replace("/", ".") # If the logdir is not created, then create it and set the @@ -1019,6 +1073,8 @@ class Router(Node): if not os.path.isdir(self.logdir): os.system("mkdir -p " + self.logdir + "/" + name) os.system("chmod -R go+rw /tmp/topotests") + # Erase logs of previous run + os.system("rm -rf " + self.logdir + "/" + name) self.daemondir = None self.hasmpls = False @@ -1185,7 +1241,7 @@ class Router(Node): if self.checkRouterVersion("<", minErrorVersion): # ignore errors in old versions errors = "" - if assertOnError and len(errors) > 0: + if assertOnError and errors is not None and len(errors) > 0: assert "Errors found - details follow:" == 0, errors return errors @@ -1324,6 +1380,7 @@ class Router(Node): # Starts actual daemons without init (ie restart) # cd to per node directory + self.cmd("install -d {}/{}".format(self.logdir, self.name)) self.cmd("cd {}/{}".format(self.logdir, self.name)) self.cmd("umask 000") @@ -1353,7 +1410,7 @@ class Router(Node): zebra_path = os.path.join(self.daemondir, "zebra") zebra_option = self.daemons_options["zebra"] self.cmd( - "{0} {1} --log stdout --log-level debug -s 90000000 -d > zebra.out 2> zebra.err".format( + "ASAN_OPTIONS=log_path=zebra.asan {0} {1} --log file:zebra.log --log-level debug -s 90000000 -d > zebra.out 2> zebra.err".format( zebra_path, zebra_option, self.logdir, self.name ) ) @@ -1368,7 +1425,7 @@ class Router(Node): staticd_path = os.path.join(self.daemondir, "staticd") staticd_option = self.daemons_options["staticd"] self.cmd( - "{0} {1} --log stdout --log-level debug -d > staticd.out 2> staticd.err".format( + "ASAN_OPTIONS=log_path=staticd.asan {0} {1} --log file:staticd.log --log-level debug -d > staticd.out 2> staticd.err".format( staticd_path, staticd_option, self.logdir, self.name ) ) @@ -1392,7 +1449,7 @@ class Router(Node): daemon_path = os.path.join(self.daemondir, daemon) self.cmd( - "{0} {1} --log stdout --log-level debug -d > {2}.out 2> {2}.err".format( + "ASAN_OPTIONS=log_path={2}.asan {0} {1} --log file:{2}.log --log-level debug -d > {2}.out 2> {2}.err".format( daemon_path, self.daemons_options.get(daemon, ""), daemon ) ) @@ -1514,7 +1571,7 @@ class Router(Node): reportMade = True # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found if checkAddressSanitizerError( - self.getStdErr(daemon), self.name, daemon + self.getStdErr(daemon), self.name, daemon, self.logdir ): sys.stderr.write( "%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon) @@ -1578,7 +1635,7 @@ class Router(Node): # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found if checkAddressSanitizerError( - self.getStdErr(daemon), self.name, daemon + self.getStdErr(daemon), self.name, daemon, self.logdir ): return "%s: Daemon %s not running - killed by AddressSanitizer" % ( self.name, |