From: Martin Winter Date: Fri, 19 May 2017 01:51:07 +0000 (-0700) Subject: Add support for collecting AddressSanitizer output. See README.md for details X-Git-Tag: frr-7.1-dev~151^2~325 X-Git-Url: https://git.puffer.fish/?a=commitdiff_plain;h=3a1f8275dbdb871398910c3eb1160f6e97b9bdfa;p=matthieu%2Ffrr.git Add support for collecting AddressSanitizer output. See README.md for details Signed-off-by: Martin Winter --- diff --git a/tests/topotests/README.md b/tests/topotests/README.md index ee099ea34a..b25910d2e0 100644 --- a/tests/topotests/README.md +++ b/tests/topotests/README.md @@ -133,6 +133,45 @@ the information to files with the given prefix (followed by testname), ie `/home/mydir/memcheck_test_bgp_multiview_topo1.txt` in case of a memory leak. +#### (Optional) Run topotests with GCC AddressSanitizer enabled + +Topotests can be run with the GCC AddressSanitizer. It requires GCC 4.8 or +newer. (Ubuntu 16.04 as suggested here is fine with GCC 5 as default) +For more information on AddressSanitizer, see +https://github.com/google/sanitizers/wiki/AddressSanitizer + +The checks are done automatically in the library call of `checkRouterRunning` +(ie at beginning of tests when there is a check for all daemons running). +No changes or extra configuration for topotests is required beside compiling +the suite with AddressSanitizer enabled. + +If a daemon crashed, then the errorlog is checked for AddressSanitizer +output. If found, then this is added with context (calling test) to +`/tmp/AddressSanitizer.txt` in markdown compatible format. + +Compiling for GCC AddressSanitizer requires to use gcc as a linker as well +(instead of ld). Here is a suggest way to compile frr with AddressSanitizer +for `stable/3.0` branch: + + git clone https://github.com/FRRouting/frr.git + cd frr + git checkout stable/3.0 + ./bootstrap.sh + export CC=gcc + export CFLAGS="-O1 -g -fsanitize=address -fno-omit-frame-pointer" + export LD=gcc + export LDFLAGS="-g -fsanitize=address -ldl" + ./configure --enable-shared=no \ + --prefix=/usr/lib/frr --sysconfdir=/etc/frr \ + --localstatedir=/var/run/frr \ + --sbindir=/usr/lib/frr --bindir=/usr/lib/frr \ + --enable-exampledir=/usr/lib/frr/examples \ + --with-moduledir=/usr/lib/frr/modules \ + --enable-multipath=0 --enable-rtadv \ + --enable-tcp-zebra --enable-fpm --enable-pimd + make + sudo make install + ## License All the configs and scripts are licensed under a ISC-style license. See diff --git a/tests/topotests/topotest.py b/tests/topotests/topotest.py new file mode 100644 index 0000000000..ca0599771e --- /dev/null +++ b/tests/topotests/topotest.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python + +# +# topotest.py +# Library of helper functions for NetDEF Topology Tests +# +# Copyright (c) 2016 by +# Network Device Education Foundation, Inc. ("NetDEF") +# +# Permission to use, copy, modify, and/or distribute this software +# for any purpose with or without fee is hereby granted, provided +# that the above copyright notice and this permission notice appear +# in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# + +import os +import errno +import re +import sys +import glob +import StringIO +import subprocess +import platform + +from mininet.topo import Topo +from mininet.net import Mininet +from mininet.node import Node, OVSSwitch, Host +from mininet.log import setLogLevel, info +from mininet.cli import CLI +from mininet.link import Intf + +from time import sleep + +def int2dpid(dpid): + "Converting Integer to DPID" + + try: + dpid = hex(dpid)[2:] + dpid = '0'*(16-len(dpid))+dpid + return dpid + except IndexError: + raise Exception('Unable to derive default datapath ID - ' + 'please either specify a dpid or use a ' + 'canonical switch name such as s23.') + +def pid_exists(pid): + "Check whether pid exists in the current process table." + + if pid <= 0: + return False + try: + os.kill(pid, 0) + except OSError as err: + if err.errno == errno.ESRCH: + # ESRCH == No such process + return False + elif err.errno == errno.EPERM: + # EPERM clearly means there's a process to deny access to + return True + else: + # According to "man 2 kill" possible error values are + # (EINVAL, EPERM, ESRCH) + raise + else: + return True + +def addRouter(topo, name): + "Adding a FRRouter (or Quagga) to Topology" + + MyPrivateDirs = ['/etc/frr', + '/etc/quagga', + '/var/run/frr', + '/var/run/quagga', + '/var/log'] + return topo.addNode(name, cls=Router, privateDirs=MyPrivateDirs) + +class LinuxRouter(Node): + "A Node with IPv4/IPv6 forwarding enabled." + + def config(self, **params): + super(LinuxRouter, self).config(**params) + # Enable forwarding on the router + self.cmd('sysctl net.ipv4.ip_forward=1') + self.cmd('sysctl net.ipv6.conf.all.forwarding=1') + def terminate(self): + """ + Terminate generic LinuxRouter Mininet instance + """ + self.cmd('sysctl net.ipv4.ip_forward=0') + self.cmd('sysctl net.ipv6.conf.all.forwarding=0') + super(LinuxRouter, self).terminate() + +class Router(Node): + "A Node with IPv4/IPv6 forwarding enabled and Quagga as Routing Engine" + + def config(self, **params): + super(Router, self).config(**params) + + # Check if Quagga or FRR is installed + if os.path.isfile('/usr/lib/frr/zebra'): + self.routertype = 'frr' + elif os.path.isfile('/usr/lib/quagga/zebra'): + self.routertype = 'quagga' + else: + raise Exception('No FRR or Quagga found in ususal location') + # Enable forwarding on the router + self.cmd('sysctl net.ipv4.ip_forward=1') + self.cmd('sysctl net.ipv6.conf.all.forwarding=1') + # Enable coredumps + self.cmd('sysctl kernel.core_uses_pid=1') + self.cmd('sysctl fs.suid_dumpable=2') + self.cmd("sysctl kernel.core_pattern=/tmp/%s_%%e_core-sig_%%s-pid_%%p.dmp" % self.name) + self.cmd('ulimit -c unlimited') + # Set ownership of config files + self.cmd('chown %s:%svty /etc/%s' % (self.routertype, self.routertype, self.routertype)) + self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0, + 'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0, + 'ldpd': 0} + def terminate(self): + # Delete Running Quagga or FRR Daemons + self.stopRouter() + # rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype) + # for d in StringIO.StringIO(rundaemons): + # self.cmd('kill -7 `cat %s`' % d.rstrip()) + # self.waitOutput() + # Disable forwarding + self.cmd('sysctl net.ipv4.ip_forward=0') + self.cmd('sysctl net.ipv6.conf.all.forwarding=0') + super(Router, self).terminate() + def stopRouter(self): + # Stop Running Quagga or FRR Daemons + rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype) + if rundaemons is not None: + for d in StringIO.StringIO(rundaemons): + daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip() + if pid_exists(int(daemonpid)): + self.cmd('kill -7 %s' % daemonpid) + self.waitOutput() + def removeIPs(self): + for interface in self.intfNames(): + self.cmd('ip address flush', interface) + def loadConf(self, daemon, source=None): + # print "Daemons before:", self.daemons + if daemon in self.daemons.keys(): + self.daemons[daemon] = 1 + if source is None: + self.cmd('touch /etc/%s/%s.conf' % (self.routertype, daemon)) + self.waitOutput() + else: + self.cmd('cp %s /etc/%s/%s.conf' % (source, self.routertype, daemon)) + self.waitOutput() + self.cmd('chmod 640 /etc/%s/%s.conf' % (self.routertype, daemon)) + self.waitOutput() + self.cmd('chown %s:%s /etc/%s/%s.conf' % (self.routertype, self.routertype, self.routertype, daemon)) + self.waitOutput() + else: + print("No daemon %s known" % daemon) + # print "Daemons after:", self.daemons + def startRouter(self): + # Disable integrated-vtysh-config + self.cmd('echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' % self.routertype) + self.cmd('chown %s:%svty /etc/%s/vtysh.conf' % (self.routertype, self.routertype, self.routertype)) + # Try to find relevant old logfiles in /tmp and delete them + map(os.remove, glob.glob("/tmp/*%s*.log" % self.name)) + # Remove old core files + map(os.remove, glob.glob("/tmp/%s*.dmp" % self.name)) + # Remove IP addresses from OS first - we have them in zebra.conf + self.removeIPs() + # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher + # No error - but return message and skip all the tests + if self.daemons['ldpd'] == 1: + if not os.path.isfile('/usr/lib/%s/ldpd' % self.routertype): + print("LDP Test, but no ldpd compiled or installed") + return "LDP Test, but no ldpd compiled or installed" + kernel_version = re.search(r'([0-9]+)\.([0-9]+).*', platform.release()) + + if kernel_version: + if (float(kernel_version.group(1)) < 4 or + (float(kernel_version.group(1)) == 4 and float(kernel_version.group(2)) < 5)): + print("LDP Test need Linux Kernel 4.5 minimum") + return "LDP Test need Linux Kernel 4.5 minimum" + # Add mpls modules to kernel if we use LDP + if self.daemons['ldpd'] == 1: + self.cmd('/sbin/modprobe mpls-router') + self.cmd('/sbin/modprobe mpls-iptunnel') + self.cmd('echo 100000 > /proc/sys/net/mpls/platform_labels') + # Init done - now restarting daemons + self.restartRouter() + return "" + def restartRouter(self): + # Starts actuall daemons without init (ie restart) + # Start Zebra first + if self.daemons['zebra'] == 1: +# self.cmd('/usr/lib/%s/zebra -d' % self.routertype) + self.cmd('/usr/lib/%s/zebra > /tmp/%s-zebra.out 2> /tmp/%s-zebra.err &' % (self.routertype, self.name, self.name)) + self.waitOutput() + print('%s: %s zebra started' % (self, self.routertype)) + sleep(1) + # Fix Link-Local Addresses + # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this + self.cmd('for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; IFS=\':\'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done') + # Now start all the other daemons + for daemon in self.daemons: + if (self.daemons[daemon] == 1) and (daemon != 'zebra'): +# self.cmd('/usr/lib/%s/%s -d' % (self.routertype, daemon)) + self.cmd('/usr/lib/%s/%s > /tmp/%s-%s.out 2> /tmp/%s-%s.err &' % (self.routertype, daemon, self.name, daemon, self.name, daemon)) + self.waitOutput() + print('%s: %s %s started' % (self, self.routertype, daemon)) + def getStdErr(self, daemon): + return self.getLog('err', daemon) + def getStdOut(self, daemon): + return self.getLog('out', daemon) + def getLog(self, log, daemon): + return self.cmd('cat /tmp/%s-%s.%s' % (self.name, daemon, log) ) + def checkRouterRunning(self): + "Check if router daemons are running and collect crashinfo they don't run" + + global fatal_error + + daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"') + for daemon in self.daemons: + if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning): + sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon)) + # Look for core file + corefiles = glob.glob("/tmp/%s_%s_core*.dmp" % (self.name, daemon)) + if (len(corefiles) > 0): + backtrace = subprocess.check_output(["gdb /usr/lib/%s/%s %s --batch -ex bt 2> /dev/null" % (self.routertype, daemon, corefiles[0])], shell=True) + sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon)) + sys.stderr.write("%s\n" % backtrace) + else: + # No core found - If we find matching logfile in /tmp, then print last 20 lines from it. + if os.path.isfile("/tmp/%s-%s.log" % (self.name, daemon)): + log_tail = subprocess.check_output(["tail -n20 /tmp/%s-%s.log 2> /dev/null" % (self.name, daemon)], shell=True) + sys.stderr.write("\nFrom %s %s %s log file:\n" % (self.routertype, self.name, daemon)) + sys.stderr.write("%s\n" % log_tail) + # + # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found + # only tested for GCC version 5.4 (as provided by Ubuntu 16.04) + # + errlog = self.getStdErr(daemon) + addressSantizerError = re.search('(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ', errlog) + if addressSantizerError: + # Sanitizer Error found in log + pidMark = addressSantizerError.group(1) + addressSantizerLog = re.search('%s(.*)%s' % (pidMark, pidMark), errlog, re.DOTALL) + if addressSantizerLog: + callingTest = os.path.basename(sys._current_frames().values()[0].f_back.f_globals['__file__']) + callingProc = sys._getframe(1).f_code.co_name + with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile: + addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2)) + addrSanFile.write("### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, self.name)) + addrSanFile.write(' '+ '\n '.join(addressSantizerLog.group(1).splitlines()) + '\n') + addrSanFile.write("\n---------------\n") + return "%s: Daemon %s not running" % (self.name, daemon) + return "" + def get_ipv6_linklocal(self): + "Get LinkLocal Addresses from interfaces" + + linklocal = [] + + ifaces = self.cmd('ip -6 address') + # Fix newlines (make them all the same) + ifaces = ('\n'.join(ifaces.splitlines()) + '\n').splitlines() + interface="" + ll_per_if_count=0 + for line in ifaces: + m = re.search('[0-9]+: ([^:@]+)[@if0-9:]+ <', line) + if m: + interface = m.group(1) + ll_per_if_count = 0 + m = re.search('inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link', line) + if m: + local = m.group(1) + ll_per_if_count += 1 + if (ll_per_if_count > 1): + linklocal += [["%s-%s" % (interface, ll_per_if_count), local]] + else: + linklocal += [[interface, local]] + return linklocal + def daemon_available(self, daemon): + "Check if specified daemon is installed (and for ldp if kernel supports MPLS)" + + if not os.path.isfile('/usr/lib/%s/%s' % (self.routertype, daemon)): + return False + if (daemon == 'ldpd'): + kernel_version = re.search(r'([0-9]+)\.([0-9]+).*', platform.release()) + if kernel_version: + if (float(kernel_version.group(1)) < 4 or + (float(kernel_version.group(1)) == 4 and float(kernel_version.group(2)) < 5)): + return False + else: + return False + return True + def get_routertype(self): + "Return the type of Router (frr or quagga)" + + return self.routertype + def report_memory_leaks(self, filename_prefix, testscript): + "Report Memory Leaks to file prefixed with given string" + + leakfound = False + filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt" + for daemon in self.daemons: + if (self.daemons[daemon] == 1): + log = self.getStdErr(daemon) + if "memstats" in log: + # Found memory leak + print("\nRouter %s %s StdErr Log:\n%s" % (self.name, daemon, log)) + if not leakfound: + leakfound = True + # Check if file already exists + fileexists = os.path.isfile(filename) + leakfile = open(filename, "a") + if not fileexists: + # New file - add header + leakfile.write("# Memory Leak Detection for topotest %s\n\n" % testscript) + leakfile.write("## Router %s\n" % self.name) + leakfile.write("### Process %s\n" % daemon) + log = re.sub("core_handler: ", "", log) + log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n#### \1\n", log) + log = re.sub("memstats: ", " ", log) + leakfile.write(log) + leakfile.write("\n") + if leakfound: + leakfile.close() + + +class LegacySwitch(OVSSwitch): + "A Legacy Switch without OpenFlow" + + def __init__(self, name, **params): + OVSSwitch.__init__(self, name, failMode='standalone', **params) + self.switchIP = None +