5 files changed, 450 insertions, 115 deletions

diff --git a/doc/developer/conf.py b/doc/developer/conf.py
index 9acfab739a..f4bb65ec79 100644
--- a/doc/developer/conf.py
+++ b/doc/developer/conf.py
@@ -21,48 +21,48 @@ from sphinx.highlighting import lexers
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-needs_sphinx = '1.0'
+needs_sphinx = "1.0"
 
 # prolog for various variable substitutions
-rst_prolog = ''
+rst_prolog = ""
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.todo', 'sphinx.ext.graphviz']
+extensions = ["sphinx.ext.todo", "sphinx.ext.graphviz"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 # source_suffix = ['.rst']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'FRR'
-copyright = u'2017, FRR'
-author = u'FRR authors'
+project = u"FRR"
+copyright = u"2017, FRR"
+author = u"FRR authors"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 
 # The short X.Y version.
-version = u'?.?'
+version = u"?.?"
 # The full version, including alpha/beta/rc tags.
-release = u'?.?-?'
+release = u"?.?-?"
 
 
 # -----------------------------------------------------------------------------
@@ -72,48 +72,49 @@ release = u'?.?-?'
 # Various installation prefixes. Values are extracted from config.status.
 # Reasonable defaults are set in case that file does not exist.
 replace_vars = {
-    'AUTHORS': author,
-    'COPYRIGHT_YEAR': '1999-2005',
-    'COPYRIGHT_STR': 'Copyright (c) 1999-2005',
-    'PACKAGE_NAME': project.lower(),
-    'PACKAGE_TARNAME': project.lower(),
-    'PACKAGE_STRING': project.lower() + ' latest',
-    'PACKAGE_URL': 'https://frrouting.org/',
-    'PACKAGE_VERSION': 'latest',
-    'INSTALL_PREFIX_ETC': '/etc/frr',
-    'INSTALL_PREFIX_SBIN': '/usr/lib/frr',
-    'INSTALL_PREFIX_STATE': '/var/run/frr',
-    'INSTALL_PREFIX_MODULES': '/usr/lib/frr/modules',
-    'INSTALL_USER': 'frr',
-    'INSTALL_GROUP': 'frr',
-    'INSTALL_VTY_GROUP': 'frrvty',
-    'GROUP': 'frr',
-    'USER': 'frr',
+    "AUTHORS": author,
+    "COPYRIGHT_YEAR": "1999-2005",
+    "COPYRIGHT_STR": "Copyright (c) 1999-2005",
+    "PACKAGE_NAME": project.lower(),
+    "PACKAGE_TARNAME": project.lower(),
+    "PACKAGE_STRING": project.lower() + " latest",
+    "PACKAGE_URL": "https://frrouting.org/",
+    "PACKAGE_VERSION": "latest",
+    "INSTALL_PREFIX_ETC": "/etc/frr",
+    "INSTALL_PREFIX_SBIN": "/usr/lib/frr",
+    "INSTALL_PREFIX_STATE": "/var/run/frr",
+    "INSTALL_PREFIX_MODULES": "/usr/lib/frr/modules",
+    "INSTALL_USER": "frr",
+    "INSTALL_GROUP": "frr",
+    "INSTALL_VTY_GROUP": "frrvty",
+    "GROUP": "frr",
+    "USER": "frr",
 }
 
 # extract version information, installation location, other stuff we need to
 # use when building final documents
 val = re.compile('^S\["([^"]+)"\]="(.*)"$')
 try:
-    with open('../../config.status', 'r') as cfgstatus:
+    with open("../../config.status", "r") as cfgstatus:
         for ln in cfgstatus.readlines():
             m = val.match(ln)
-            if not m or m.group(1) not in replace_vars.keys(): continue
+            if not m or m.group(1) not in replace_vars.keys():
+                continue
             replace_vars[m.group(1)] = m.group(2)
 except IOError:
     # if config.status doesn't exist, just ignore it
     pass
 
 # manually fill out some of these we can't get from config.status
-replace_vars['COPYRIGHT_STR'] = "Copyright (c)"
-replace_vars['COPYRIGHT_STR'] += ' {0}'.format(replace_vars['COPYRIGHT_YEAR'])
-replace_vars['COPYRIGHT_STR'] += ' {0}'.format(replace_vars['AUTHORS'])
-release = replace_vars['PACKAGE_VERSION']
-version = release.split('-')[0]
+replace_vars["COPYRIGHT_STR"] = "Copyright (c)"
+replace_vars["COPYRIGHT_STR"] += " {0}".format(replace_vars["COPYRIGHT_YEAR"])
+replace_vars["COPYRIGHT_STR"] += " {0}".format(replace_vars["AUTHORS"])
+release = replace_vars["PACKAGE_VERSION"]
+version = release.split("-")[0]
 
 # add substitutions to prolog
 for key, value in replace_vars.items():
-    rst_prolog += '.. |{0}| replace:: {1}\n'.format(key, value)
+    rst_prolog += ".. |{0}| replace:: {1}\n".format(key, value)
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -125,37 +126,42 @@ language = None
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ['_build', 'building-libyang.rst', 'topotests-snippets.rst', 'include-compile.rst']
+exclude_patterns = [
+    "_build",
+    "building-libyang.rst",
+    "topotests-snippets.rst",
+    "include-compile.rst",
+]
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
@@ -165,165 +171,158 @@ todo_include_todos = True
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = "default"
 
 try:
     import sphinx_rtd_theme
 
-    html_theme = 'sphinx_rtd_theme'
+    html_theme = "sphinx_rtd_theme"
 except ImportError:
     pass
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {
+# html_theme_options = {
 #    'sidebarbgcolor': '#374249'
-#}
+# }
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-html_logo = '../figures/frr-icon.svg'
+html_logo = "../figures/frr-icon.svg"
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-html_favicon = '../figures/frr-logo-icon.png'
+html_favicon = "../figures/frr-logo-icon.png"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Language to be used for generating the HTML full-text search index.
 # Sphinx supports the following languages:
 #   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
 #   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
-#html_search_language = 'en'
+# html_search_language = 'en'
 
 # A dictionary with options for the search language support, empty by default.
 # Now only 'ja' uses this config value
-#html_search_options = {'type': 'default'}
+# html_search_options = {'type': 'default'}
 
 # The name of a javascript file (relative to the configuration directory) that
 # implements a search results scorer. If empty, the default will be used.
-#html_search_scorer = 'scorer.js'
+# html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'FRRdoc'
+htmlhelp_basename = "FRRdoc"
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
-
-# Latex figure (float) alignment
-#'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+    # Latex figure (float) alignment
+    #'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'FRR.tex', u"FRR Developer's Manual",
-     u'FRR', 'manual'),
+    (master_doc, "FRR.tex", u"FRR Developer's Manual", u"FRR", "manual"),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-latex_logo = '../figures/frr-logo-medium.png'
+latex_logo = "../figures/frr-logo-medium.png"
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
 
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'frr', u"FRR Developer's Manual",
-     [author], 1)
-]
+man_pages = [(master_doc, "frr", u"FRR Developer's Manual", [author], 1)]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -332,38 +331,44 @@ man_pages = [
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'frr', u"FRR Developer's Manual",
-     author, 'FRR', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "frr",
+        u"FRR Developer's Manual",
+        author,
+        "FRR",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False
 
 # contents of ../extra/frrlexer.py.
 # This is read here to support VPATH build. Since this section is execfile()'d
 # with the file location, we can safely use a relative path here to save the
 # contents of the lexer file for later use even if our relative path changes
 # due to VPATH.
-with open('../extra/frrlexer.py', 'rb') as lex:
+with open("../extra/frrlexer.py", "rb") as lex:
     frrlexerpy = lex.read()
 
 # custom extensions here
 def setup(app):
     # object type for FRR CLI commands, can be extended to document parent CLI
     # node later on
-    app.add_object_type('clicmd', 'clicmd')
+    app.add_object_type("clicmd", "clicmd")
     # css overrides for HTML theme
-    app.add_stylesheet('overrides.css')
+    app.add_stylesheet("overrides.css")
     # load Pygments lexer for FRR config syntax
     #
     # NB: in Pygments 2.2+ this can be done with `load_lexer_from_file`, but we
@@ -373,4 +378,4 @@ def setup(app):
     # frrlexer = pygments.lexers.load_lexer_from_file('../extra/frrlexer.py', lexername="FRRLexer")
     custom_namespace = {}
     exec(frrlexerpy, custom_namespace)
-    lexers['frr'] = custom_namespace['FRRLexer']()
+    lexers["frr"] = custom_namespace["FRRLexer"]()
diff --git a/doc/developer/index.rst b/doc/developer/index.rst
index 1f803b3772..1ba0f31c8a 100644
--- a/doc/developer/index.rst
+++ b/doc/developer/index.rst
@@ -9,6 +9,7 @@ FRRouting Developer's Guide
    packaging
    process-architecture
    library
+   tracing
    testing
    bgpd
    fpm
diff --git a/doc/developer/logging.rst b/doc/developer/logging.rst
index 0430ad72a3..2f2444373c 100644
--- a/doc/developer/logging.rst
+++ b/doc/developer/logging.rst
@@ -83,6 +83,8 @@ Extensions
 +-----------+--------------------------+----------------------------------------------+
 | ``%pNHs`` | ``struct nexthop *``     | ``1.2.3.4 if 15``                            |
 +-----------+--------------------------+----------------------------------------------+
+| ``%pFX``  + ``struct bgp_dest *``    | ``fe80::1234/64`` available in BGP only      |
++-----------+--------------------------+----------------------------------------------+
 
 Printf features like field lengths can be used normally with these extensions,
 e.g. ``%-15pI4`` works correctly.
diff --git a/doc/developer/tracing.rst b/doc/developer/tracing.rst
new file mode 100644
index 0000000000..ee0a6be008
--- /dev/null
+++ b/doc/developer/tracing.rst
@@ -0,0 +1,314 @@
+.. _tracing:
+
+Tracing
+=======
+
+FRR has a small but growing number of static tracepoints available for use with
+various tracing systems. These tracepoints can assist with debugging,
+performance analysis and to help understand program flow. They can also be used
+for monitoring.
+
+Developers are encouraged to write new static tracepoints where sensible. They
+are not compiled in by default, and even when they are, they have no overhead
+unless enabled by a tracer, so it is okay to be liberal with them.
+
+
+Supported tracers
+-----------------
+
+Presently two types of tracepoints are supported:
+
+- `LTTng tracepoints <https://lttng.org/>`_
+- `USDT probes <http://dtrace.org/guide/chp-usdt.html>`_
+
+LTTng is a tracing framework for Linux only. It offers extremely low overhead
+and very rich tracing capabilities. FRR supports LTTng-UST, which is the
+userspace implementation. LTTng tracepoints are very rich in detail. No kernel
+modules are needed. Besides only being available for Linux, the primary
+downside of LTTng is the need to link to ``lttng-ust``.
+
+USDT probes originate from Solaris, where they were invented for use with
+dtrace. They are a kernel feature. At least Linux and FreeBSD support them. No
+library is needed; support is compiled in via a system header
+(``<sys/sdt.h>``). USDT probes are much slower than LTTng tracepoints and offer
+less flexibility in what information can be gleaned from them.
+
+LTTng is capable of tracing USDT probes but has limited support for them.
+SystemTap and dtrace both work only with USDT probes.
+
+
+Usage
+-----
+
+To compile with tracepoints, use one of the following configure flags:
+
+.. program:: configure.ac
+
+.. option:: --enable-lttng=yes
+
+   Generate LTTng tracepoints
+
+.. option:: --enable-usdt=yes
+
+   Generate USDT probes
+
+To trace with LTTng, compile with either one (prefer :option:`--enable-lttng`
+run the target in non-forking mode (no ``-d``) and use LTTng as usual (refer to
+LTTng user manual). When using USDT probes with LTTng, follow the example in
+`this article
+<https://lttng.org/blog/2019/10/15/new-dynamic-user-space-tracing-in-lttng/>`_.
+To trace with dtrace or SystemTap, compile with :option:`--enable-usdt=yes` and
+use your tracer as usual.
+
+To see available USDT probes::
+
+  readelf -n /usr/lib/frr/bgpd
+
+Example::
+
+  root@host ~> readelf -n /usr/lib/frr/bgpd
+
+  Displaying notes found in: .note.ABI-tag
+    Owner                 Data size	Description
+    GNU                  0x00000010	NT_GNU_ABI_TAG (ABI version tag)
+      OS: Linux, ABI: 3.2.0
+
+  Displaying notes found in: .note.gnu.build-id
+    Owner                 Data size	Description
+    GNU                  0x00000014	NT_GNU_BUILD_ID (unique build ID bitstring)
+      Build ID: 4f42933a69dcb42a519bc459b2105177c8adf55d
+
+  Displaying notes found in: .note.stapsdt
+    Owner                 Data size	Description
+    stapsdt              0x00000045	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: packet_read
+      Location: 0x000000000045ee48, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-96(%rbp) 8@-104(%rbp)
+    stapsdt              0x00000047	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: open_process
+      Location: 0x000000000047c43b, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-224(%rbp) 2@-226(%rbp)
+    stapsdt              0x00000049	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: update_process
+      Location: 0x000000000047c4bf, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-208(%rbp) 2@-210(%rbp)
+    stapsdt              0x0000004f	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: notification_process
+      Location: 0x000000000047c557, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-192(%rbp) 2@-194(%rbp)
+    stapsdt              0x0000004c	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: keepalive_process
+      Location: 0x000000000047c5db, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-176(%rbp) 2@-178(%rbp)
+    stapsdt              0x0000004a	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: refresh_process
+      Location: 0x000000000047c673, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-160(%rbp) 2@-162(%rbp)
+    stapsdt              0x0000004d	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: capability_process
+      Location: 0x000000000047c6f7, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-144(%rbp) 2@-146(%rbp)
+    stapsdt              0x0000006f	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: output_filter
+      Location: 0x000000000048e33a, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-144(%rbp) 8@-152(%rbp) 4@-156(%rbp) 4@-160(%rbp) 8@-168(%rbp)
+    stapsdt              0x0000007d	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: process_update
+      Location: 0x0000000000491f10, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-800(%rbp) 8@-808(%rbp) 4@-812(%rbp) 4@-816(%rbp) 4@-820(%rbp) 8@-832(%rbp)
+    stapsdt              0x0000006e	NT_STAPSDT (SystemTap probe descriptors)
+      Provider: frr_bgp
+      Name: input_filter
+      Location: 0x00000000004940ed, Base: 0x00000000005a09d2, Semaphore: 0x0000000000000000
+      Arguments: 8@-144(%rbp) 8@-152(%rbp) 4@-156(%rbp) 4@-160(%rbp) 8@-168(%rbp)
+
+
+To see available LTTng probes, run the target, create a session and then::
+
+  lttng list --userspace | grep frr
+
+Example::
+
+  root@host ~> lttng list --userspace | grep frr
+  PID: 11157 - Name: /usr/lib/frr/bgpd
+        frr_libfrr:route_node_get (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:list_sort (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:list_delete_node (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:list_remove (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:list_add (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:memfree (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:memalloc (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:frr_pthread_stop (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:frr_pthread_run (loglevel: TRACE_DEBUG_LINE (13)) (type: tracepoint)
+        frr_libfrr:thread_call (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:thread_cancel_async (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:thread_cancel (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:schedule_write (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:schedule_read (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:schedule_event (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:schedule_timer (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:hash_release (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:hash_insert (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_libfrr:hash_get (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:output_filter (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:input_filter (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:process_update (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:packet_read (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:refresh_process (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:capability_process (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:notification_process (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:update_process (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:keepalive_process (loglevel: TRACE_INFO (6)) (type: tracepoint)
+        frr_bgp:open_process (loglevel: TRACE_INFO (6)) (type: tracepoint)
+
+When using LTTng, you can also get zlogs as trace events by enabling
+the ``lttng_ust_tracelog:*`` event class.
+
+Concepts
+--------
+
+Tracepoints are statically defined points in code where a developer has
+determined that outside observers might gain something from knowing what is
+going on at that point. It's like logging but with the ability to dump large
+amounts of internal data with much higher performance. LTTng has a good summary
+`here <https://lttng.org/docs/#doc-what-is-tracing>`_.
+
+Each tracepoint has a "provider" and name. The provider is basically a
+namespace; for example, ``bgpd`` uses the provider name ``frr_bgp``. The name
+is arbitrary, but because providers share a global namespace on the user's
+system, all providers from FRR should be prefixed by ``frr_``. The tracepoint
+name is just the name of the event. Events are globally named by their provider
+and name. For example, the event when BGP reads a packet from a peer is
+``frr_bgp:packet_read``.
+
+To do tracing, the tracing tool of choice is told which events to listen to.
+For example, to listen to all events from FRR's BGP implementation, you would
+enable the events ``frr_bgp:*``. In the same tracing session you could also
+choose to record all memory allocations by enabling the ``malloc`` tracepoints
+in ``libc`` as well as all kernel skb operations using the various in-kernel
+tracepoints. This allows you to build as complete a view as desired of what the
+system is doing during the tracing window (subject to what tracepoints are
+available).
+
+Of particular use are the tracepoints for FRR's internal event scheduler;
+tracing these allows you to see all events executed by all event loops for the
+target(s) in question. Here's a couple events selected from a trace of BGP
+during startup::
+
+   ...
+
+   [18:41:35.750131763] (+0.000048901) host frr_libfrr:thread_call: { cpu_id =
+   1 }, { threadmaster_name = "default", function_name = "zclient_connect",
+   scheduled_from = "lib/zclient.c", scheduled_on_line = 3877, thread_addr =
+   0x0, file_descriptor = 0, event_value = 0, argument_ptr = 0xA37F70, timer =
+   0 }
+
+   [18:41:35.750175124] (+0.000020001) host frr_libfrr:thread_call: { cpu_id =
+   1 }, { threadmaster_name = "default", function_name = "frr_config_read_in",
+   scheduled_from = "lib/libfrr.c", scheduled_on_line = 934, thread_addr = 0x0,
+   file_descriptor = 0, event_value = 0, argument_ptr = 0x0, timer = 0 }
+
+   [18:41:35.753341264] (+0.000010532) host frr_libfrr:thread_call: { cpu_id =
+   1 }, { threadmaster_name = "default", function_name = "bgp_event",
+   scheduled_from = "bgpd/bgpd.c", scheduled_on_line = 142, thread_addr = 0x0,
+   file_descriptor = 2, event_value = 2, argument_ptr = 0xE4D780, timer = 2 }
+
+   [18:41:35.753404186] (+0.000004910) host frr_libfrr:thread_call: { cpu_id =
+   1 }, { threadmaster_name = "default", function_name = "zclient_read",
+   scheduled_from = "lib/zclient.c", scheduled_on_line = 3891, thread_addr =
+   0x0, file_descriptor = 40, event_value = 40, argument_ptr = 0xA37F70, timer
+   = 40 }
+
+   ...
+
+
+Very useful for getting a time-ordered look into what the process is doing.
+
+
+Adding Tracepoints
+------------------
+
+Adding new tracepoints is a two step process:
+
+1. Define the tracepoint
+2. Use the tracepoint
+
+Tracepoint definitions state the "provider" and name of the tracepoint, along
+with any values it will produce, and how to format them. This is done with
+macros provided by LTTng. USDT probes do not use definitions and are inserted
+at the trace site with a single macro. However, to maintain support for both
+platforms, you must define an LTTng tracepoint when adding a new one.
+``frrtrace()`` will expand to the appropriate ``DTRACE_PROBEn`` macro when USDT
+is in use.
+
+If you are adding new tracepoints to a daemon that has no tracepoints, that
+daemon's ``subdir.am`` must be updated to conditionally link ``lttng-ust``.
+Look at ``bgpd/subdir.am`` for an example of how to do this; grep for
+``UST_LIBS``. Create new files named ``<daemon>_trace.[ch]``. Use
+``bgpd/bgp_trace.[h]`` as boilerplate. If you are adding tracepoints to a
+daemon that already has them, look for the ``<daemon>_trace.h`` file;
+tracepoints are written here.
+
+Refer to the `LTTng developer docs
+<https://lttng.org/docs/#doc-c-application>`_ for details on how to define
+tracepoints.
+
+To use them, simply add a call to ``frrtrace()`` at the point you'd like the
+event to be emitted, like so:
+
+.. code-block:: c
+
+   ...
+
+   switch (type) {
+   case BGP_MSG_OPEN:
+           frrtrace(2, frr_bgp, open_process, peer, size); /* tracepoint */
+           atomic_fetch_add_explicit(&peer->open_in, 1,
+                                     memory_order_relaxed);
+           mprc = bgp_open_receive(peer, size);
+
+   ...
+
+After recompiling this tracepoint will now be available, either as a USDT probe
+or LTTng tracepoint, depending on your compilation choice.
+
+
+trace.h
+^^^^^^^
+
+Because FRR supports multiple types of tracepoints, the code for creating them
+abstracts away the underlying system being used. This abstraction code is in
+``lib/trace.h``. There are 2 function-like macros that are used for working
+with tracepoints.
+
+- ``frrtrace()`` defines tracepoints
+- ``frrtrace_enabled()`` checks whether a tracepoint is enabled
+
+There is also ``frrtracelog()``, which is used in zlog core code to make zlog
+messages available as trace events to LTTng. This should not be used elsewhere.
+
+There is additional documentation in the header. The key thing to note is that
+you should never include ``trace.h`` in source where you plan to put
+tracepoints; include the tracepoint definition header instead (e.g.
+:file:`bgp_trace.h`).
+
+
+Limitations
+-----------
+
+Tracers do not like ``fork()`` or ``dlopen()``. LTTng has some workarounds for
+this involving interceptor libraries using ``LD_PRELOAD``.
+
+USDT tracepoints are relatively high overhead and probably shouldn't be used
+for "flight recorder" functionality, i.e. enabling and passively recording all
+events for monitoring purposes. It's generally okay to use LTTng like this,
+though.
diff --git a/doc/developer/workflow.rst b/doc/developer/workflow.rst
index f345464a35..4183ac6480 100644
--- a/doc/developer/workflow.rst
+++ b/doc/developer/workflow.rst
@@ -779,13 +779,12 @@ necessary replacements.
 .. _style-exceptions:
 
 Exceptions
-^^^^^^^^^^
+""""""""""
 
 FRR project code comes from a variety of sources, so there are some
 stylistic exceptions in place. They are organized here by branch.
 
-For ``master``
-""""""""""""""
+For ``master``:
 
 BSD coding style applies to:
 
@@ -797,8 +796,7 @@ BSD coding style applies to:
 -  Indents are 4 spaces
 -  Function return types are on their own line
 
-For ``stable/3.0`` and ``stable/2.0``
-"""""""""""""""""""""""""""""""""""""
+For ``stable/3.0`` and ``stable/2.0``:
 
 GNU coding style apply to the following parts:
 
@@ -816,6 +814,21 @@ BSD coding style applies to:
 
 -  ``ldpd/``
 
+
+Python Code
+^^^^^^^^^^^
+
+Format all Python code with `black <https://github.com/psf/black>`_.
+
+In a line::
+
+   python3 -m black <file.py>
+
+Run this on any Python files you modify before committing.
+
+FRR's Python code has been formatted with black version 19.10b.
+
+
 YANG
 ^^^^