diff options
7 files changed, 192 insertions, 9 deletions
diff --git a/doc/developer/topotests-markers.rst b/doc/developer/topotests-markers.rst new file mode 100644 index 0000000000..02ffe3f777 --- /dev/null +++ b/doc/developer/topotests-markers.rst @@ -0,0 +1,115 @@ +.. _topotests-markers: + +Markers +-------- + +To allow for automated selective testing on large scale continuous integration +systems, all tests must be marked with at least one of the following markers: + +* babeld +* bfdd +* bgpd +* eigrpd +* isisd +* ldpd +* nhrpd +* ospf6d +* ospfd +* pathd +* pbrd +* pimd +* ripd +* ripngd +* sharpd +* staticd +* vrrpd + +The markers corespond to the daemon subdirectories in FRR's source code and have +to be added to tests on a module level depending on which daemons are used +during the test. + +The goal is to have continuous integration systems scan code submissions, detect +changes to files in a daemons subdirectory and select only tests using that +daemon to run to shorten developers waiting times for test results and save test +infrastructure resources. + +Newly written modules and code changes on tests, which do not contain any or +incorrect markers will be rejected by reviewers. + + +Registering markers +^^^^^^^^^^^^^^^^^^^ +The Registration of new markers takes place in the file +``tests/topotests/pytest.ini`` and should be discussed with members of the TSC +beforehand. + +.. code:: python3 + + # tests/topotests/pytest.ini + [pytest] + ... + markers = + babeld: Tests that run against BABELD + bfdd: Tests that run against BFDD + ... + vrrpd: Tests that run against VRRPD + + +Adding markers to tests +^^^^^^^^^^^^^^^^^^^^^^^ +Markers are added to a test by placing a global variable in the test module. + +Adding a single marker: + +.. code:: python3 + + import pytest + + ... + + pytestmark = pytest.mark.bfdd + + ... + + def test_using_bfdd(): + + +Adding multiple markers: + +.. code:: python3 + + import pytest + + ... + + pytestmark = [ + pytest.mark.bgpd, + pytest.mark.ospfd, + pytest.mark.ospf6d + ] + + ... + + def test_using_bgpd_ospfd_ospf6d(): + + +Selecting marked modules fort testing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Selecting by a single marker: + +.. code:: bash + + pytest -v -m isisd + +Selecting by multiple markers: + +.. code:: bash + + pytest -v -m "isisd or ldpd or nhrpd" + + +Further Information +^^^^^^^^^^^^^^^^^^^ +The `online pytest documentation <https://docs.pytest.org/en/stable/example/markers.html>`_ +provides further information and usage examples for pytest markers. + diff --git a/doc/developer/topotests.rst b/doc/developer/topotests.rst index 688f73c991..3e8987f126 100644 --- a/doc/developer/topotests.rst +++ b/doc/developer/topotests.rst @@ -792,6 +792,8 @@ Requirements: conforms with this, run it without the :option:`-s` parameter. - Use `black <https://github.com/psf/black>`_ code formatter before creating a pull request. This ensures we have a unified code style. +- Mark test modules with pytest markers depending on the daemons used during the + tests (s. Markers) Tips: @@ -950,6 +952,8 @@ does what you need. If nothing is similar, then you may create a new topology, preferably, using the newest template (:file:`tests/topotests/example-test/test_template.py`). +.. include:: topotests-markers.rst + .. include:: topotests-snippets.rst License diff --git a/tests/topotests/example-test/test_template.py b/tests/topotests/example-test/test_template.py index 4305e0199f..973303b830 100644 --- a/tests/topotests/example-test/test_template.py +++ b/tests/topotests/example-test/test_template.py @@ -44,6 +44,18 @@ from lib.topolog import logger from mininet.topo import Topo +#TODO: select markers based on daemons used during test +# pytest module level markers +""" +pytestmark = pytest.mark.bfdd # single marker +pytestmark = [ + pytest.mark.bgpd, + pytest.mark.ospfd, + pytest.mark.ospf6d +] # multiple markers +""" + + class TemplateTopo(Topo): "Test topology builder" diff --git a/tests/topotests/example-topojson-test/test_topo_json_multiple_links/test_example_topojson_multiple_links.py b/tests/topotests/example-topojson-test/test_topo_json_multiple_links/test_example_topojson_multiple_links.py index f24f463b8a..cd48716905 100755 --- a/tests/topotests/example-topojson-test/test_topo_json_multiple_links/test_example_topojson_multiple_links.py +++ b/tests/topotests/example-topojson-test/test_topo_json_multiple_links/test_example_topojson_multiple_links.py @@ -53,6 +53,19 @@ from lib.topolog import logger from lib.bgp import verify_bgp_convergence from lib.topojson import build_topo_from_json, build_config_from_json + +#TODO: select markers based on daemons used during test +# pytest module level markers +""" +pytestmark = pytest.mark.bfdd # single marker +pytestmark = [ + pytest.mark.bgpd, + pytest.mark.ospfd, + pytest.mark.ospf6d +] # multiple markers +""" + + # Reading the data from JSON File for topology and configuration creation jsonFile = "{}/example_topojson_multiple_links.json".format(CWD) try: diff --git a/tests/topotests/example-topojson-test/test_topo_json_single_link/test_example_topojson.py b/tests/topotests/example-topojson-test/test_topo_json_single_link/test_example_topojson.py index 3ae3c9f4fe..0c72e30044 100755 --- a/tests/topotests/example-topojson-test/test_topo_json_single_link/test_example_topojson.py +++ b/tests/topotests/example-topojson-test/test_topo_json_single_link/test_example_topojson.py @@ -52,6 +52,19 @@ from lib.topolog import logger from lib.bgp import verify_bgp_convergence from lib.topojson import build_topo_from_json, build_config_from_json + +#TODO: select markers based on daemons used during test +# pytest module level markers +""" +pytestmark = pytest.mark.bfdd # single marker +pytestmark = [ + pytest.mark.bgpd, + pytest.mark.ospfd, + pytest.mark.ospf6d +] # multiple markers +""" + + # Reading the data from JSON File for topology and configuration creation jsonFile = "{}/example_topojson.json".format(CWD) diff --git a/tests/topotests/example-topojson-test/test_topo_json_single_link_loopback/test_example_topojson.py b/tests/topotests/example-topojson-test/test_topo_json_single_link_loopback/test_example_topojson.py index 06fa2f4626..d05ad6db21 100755 --- a/tests/topotests/example-topojson-test/test_topo_json_single_link_loopback/test_example_topojson.py +++ b/tests/topotests/example-topojson-test/test_topo_json_single_link_loopback/test_example_topojson.py @@ -54,6 +54,19 @@ from lib.topolog import logger from lib.bgp import verify_bgp_convergence from lib.topojson import build_topo_from_json, build_config_from_json + +#TODO: select markers based on daemons used during test +# pytest module level markers +""" +pytestmark = pytest.mark.bfdd # single marker +pytestmark = [ + pytest.mark.bgpd, + pytest.mark.ospfd, + pytest.mark.ospf6d +] # multiple markers +""" + + # Reading the data from JSON File for topology and configuration creation jsonFile = "{}/example_topojson.json".format(CWD) diff --git a/tests/topotests/pytest.ini b/tests/topotests/pytest.ini index d1b18a57bb..0c45a09445 100644 --- a/tests/topotests/pytest.ini +++ b/tests/topotests/pytest.ini @@ -1,16 +1,29 @@ # Skip pytests example directory [pytest] norecursedirs = .git example-test example-topojson-test lib docker + +# Markers +# +# Please consult the documentation and discuss with TSC members before applying +# any changes to this list. markers = - babel: Tests that run against BABEL - bfd: Tests that run against BFDD - eigrp: Tests that run against EIGRPD - isis: Tests that run against ISISD - ldp: Tests that run against LDPD - ospf: Tests that run against OSPF( v2 and v3 ) - pbr: Tests that run against PBRD - pim: Tests that run against pim - rip: Tests that run against RIP, both v4 and v6 + babeld: Tests that run against BABELD + bfdd: Tests that run against BFDD + bgpd: Tests that run against BGPD + eigrpd: Tests that run against EIGRPD + isisd: Tests that run against ISISD + ldpd: Tests that run against LDPD + nhrpd: Tests that run against NHRPD + ospf6d: Tests that run against OSPF6D + ospfd: Tests that run against OSPFD + pathd: Tests that run against PATHD + pbrd: Tests that run against PBRD + pimd: Tests that run against PIMD + ripd: Tests that run against RIPD + ripngd: Tests that run against RIPNGD + sharpd: Tests that run against SHARPD + staticd: Tests that run against STATICD + vrrpd: Tests that run against VRRPD [topogen] # Default configuration values |