From 9bb02389d0423941e328c88b38e241aa722229d0 Mon Sep 17 00:00:00 2001 From: Mark Stapp Date: Sun, 19 Apr 2020 13:23:09 -0400 Subject: [PATCH] zebra: add zebra opaque module Add the zebra_opaque module, designed to offload some opaque zapi message processing to a new, dedicated pthread. Add to the build; also re-sort the lists of zebra files in subdir.am. Start, stop, and clean-up the opaque module, integrate with zebra start and shutdown. Signed-off-by: Mark Stapp --- zebra/main.c | 12 ++ zebra/subdir.am | 46 ++++---- zebra/zebra_dplane.c | 1 - zebra/zebra_opaque.c | 271 +++++++++++++++++++++++++++++++++++++++++++ zebra/zebra_opaque.h | 63 ++++++++++ 5 files changed, 370 insertions(+), 23 deletions(-) create mode 100644 zebra/zebra_opaque.c create mode 100644 zebra/zebra_opaque.h diff --git a/zebra/main.c b/zebra/main.c index f447e9aa07..05dd70ff7a 100644 --- a/zebra/main.c +++ b/zebra/main.c @@ -55,6 +55,7 @@ #include "zebra/zebra_vxlan.h" #include "zebra/zebra_routemap.h" #include "zebra/zebra_nb.h" +#include "zebra/zebra_opaque.h" #if defined(HANDLE_NETLINK_FUZZING) #include "zebra/kernel_netlink.h" @@ -151,18 +152,25 @@ static void sigint(void) frr_early_fini(); + /* Stop the opaque module pthread */ + zebra_opaque_stop(); + zebra_dplane_pre_finish(); /* Clean up GR related info. */ zebra_gr_stale_client_cleanup(zrouter.stale_client_list); list_delete_all_node(zrouter.stale_client_list); + /* Clean up zapi clients and server module */ for (ALL_LIST_ELEMENTS(zrouter.client_list, ln, nn, client)) zserv_close_client(client); zserv_close(); list_delete_all_node(zrouter.client_list); + /* Once all the zclients are cleaned up, clean up the opaque module */ + zebra_opaque_finish(); + zebra_ptm_finish(); if (retain_mode) @@ -427,6 +435,7 @@ int main(int argc, char **argv) zebra_mpls_vty_init(); zebra_pw_vty_init(); zebra_pbr_init(); + zebra_opaque_init(); /* For debug purpose. */ /* SET_FLAG (zebra_debug_event, ZEBRA_DEBUG_EVENT); */ @@ -458,6 +467,9 @@ int main(int argc, char **argv) /* Start dataplane system */ zebra_dplane_start(); + /* Start the ted module, before zserv */ + zebra_opaque_start(); + /* Start Zebra API server */ zserv_start(zserv_path); diff --git a/zebra/subdir.am b/zebra/subdir.am index 5601b4c379..d98ef52571 100644 --- a/zebra/subdir.am +++ b/zebra/subdir.am @@ -10,6 +10,7 @@ vtysh_scan += \ zebra/interface.c \ zebra/router-id.c \ zebra/rtadv.c \ + zebra/zebra_gr.c \ zebra/zebra_mlag_vty.c \ zebra/zebra_mpls_vty.c \ zebra/zebra_ptm.c \ @@ -17,7 +18,6 @@ vtysh_scan += \ zebra/zebra_routemap.c \ zebra/zebra_vty.c \ zebra/zserv.c \ - zebra/zebra_gr.c \ # end # can be loaded as DSO - always include for vtysh @@ -72,19 +72,30 @@ zebra_zebra_SOURCES = \ zebra/rtread_sysctl.c \ zebra/rule_netlink.c \ zebra/rule_socket.c \ + zebra/table_manager.c \ + zebra/zapi_msg.c \ + zebra/zebra_dplane.c \ + zebra/zebra_errors.c \ + zebra/zebra_gr.c \ + zebra/zebra_l2.c \ zebra/zebra_mlag.c \ zebra/zebra_mlag_vty.c \ - zebra/zebra_l2.c \ zebra/zebra_memory.c \ - zebra/zebra_dplane.c \ zebra/zebra_mpls.c \ zebra/zebra_mpls_netlink.c \ zebra/zebra_mpls_openbsd.c \ zebra/zebra_mpls_null.c \ zebra/zebra_mpls_vty.c \ zebra/zebra_mroute.c \ + zebra/zebra_nb.c \ + zebra/zebra_nb_config.c \ + zebra/zebra_nb_rpcs.c \ + zebra/zebra_nb_state.c \ + zebra/zebra_netns_id.c \ + zebra/zebra_netns_notify.c \ zebra/zebra_nhg.c \ zebra/zebra_ns.c \ + zebra/zebra_opaque.c \ zebra/zebra_pbr.c \ zebra/zebra_ptm.c \ zebra/zebra_ptm_redistribute.c \ @@ -97,16 +108,6 @@ zebra_zebra_SOURCES = \ zebra/zebra_vty.c \ zebra/zebra_vxlan.c \ zebra/zserv.c \ - zebra/zebra_netns_id.c \ - zebra/zebra_netns_notify.c \ - zebra/table_manager.c \ - zebra/zapi_msg.c \ - zebra/zebra_nb.c \ - zebra/zebra_nb_config.c \ - zebra/zebra_nb_rpcs.c \ - zebra/zebra_nb_state.c \ - zebra/zebra_errors.c \ - zebra/zebra_gr.c \ # end clippy_scan += \ @@ -137,17 +138,24 @@ noinst_HEADERS += \ zebra/rt_netlink.h \ zebra/rtadv.h \ zebra/rule_netlink.h \ - zebra/zebra_mlag.h \ - zebra/zebra_mlag_vty.h \ + zebra/table_manager.h \ + zebra/zapi_msg.h \ + zebra/zebra_dplane.h \ + zebra/zebra_errors.h \ zebra/zebra_fpm_private.h \ zebra/zebra_l2.h \ - zebra/zebra_dplane.h \ zebra/zebra_memory.h \ + zebra/zebra_mlag.h \ + zebra/zebra_mlag_vty.h \ zebra/zebra_mpls.h \ zebra/zebra_mroute.h \ + zebra/zebra_nb.h \ + zebra/zebra_netns_id.h \ + zebra/zebra_netns_notify.h \ zebra/zebra_nhg.h \ zebra/zebra_nhg_private.h \ zebra/zebra_ns.h \ + zebra/zebra_opaque.h \ zebra/zebra_pbr.h \ zebra/zebra_ptm.h \ zebra/zebra_ptm_redistribute.h \ @@ -159,12 +167,6 @@ noinst_HEADERS += \ zebra/zebra_vxlan.h \ zebra/zebra_vxlan_private.h \ zebra/zserv.h \ - zebra/zebra_netns_id.h \ - zebra/zebra_netns_notify.h \ - zebra/table_manager.h \ - zebra/zapi_msg.h \ - zebra/zebra_nb.h \ - zebra/zebra_errors.h \ # end zebra_zebra_irdp_la_SOURCES = \ diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index 2c8ef37cbe..568b398924 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -28,7 +28,6 @@ #include "lib/memory.h" #include "lib/queue.h" #include "lib/zebra.h" -#include "zebra/zebra_router.h" #include "zebra/zebra_memory.h" #include "zebra/zebra_router.h" #include "zebra/zebra_dplane.h" diff --git a/zebra/zebra_opaque.c b/zebra/zebra_opaque.c new file mode 100644 index 0000000000..570387e785 --- /dev/null +++ b/zebra/zebra_opaque.c @@ -0,0 +1,271 @@ +/* + * Zebra opaque message handler module + * Copyright (c) 2020 Volta Networks, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; see the file COPYING; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + + +#include +#include "lib/debug.h" +#include "lib/frr_pthread.h" +#include "lib/stream.h" +#include "zebra/debug.h" +#include "zebra/zserv.h" +#include "zebra/zebra_opaque.h" + +/* + * Globals + */ +static struct zebra_opaque_globals { + + /* Sentinel for run or start of shutdown */ + _Atomic uint32_t run; + + /* Limit number of pending, unprocessed updates */ + _Atomic uint32_t max_queued_updates; + + /* Limit number of new messages dequeued at once, to pace an + * incoming burst. + */ + uint32_t msgs_per_cycle; + + /* Stats: counters of incoming messages, errors, and yields (when + * the limit has been reached.) + */ + _Atomic uint32_t msgs_in; + _Atomic uint32_t msg_errors; + _Atomic uint32_t yields; + + /* pthread */ + struct frr_pthread *pthread; + + /* Event-delivery context 'master' for the module */ + struct thread_master *master; + + /* Event/'thread' pointer for queued zapi messages */ + struct thread *t_msgs; + + /* Input fifo queue to the module, and lock to protect it. */ + pthread_mutex_t mutex; + struct stream_fifo in_fifo; + +} zo_info; + +/* Name string for debugs/logs */ +static const char LOG_NAME[] = "Zebra Opaque"; + +/* Prototypes */ + +/* Main event loop, processing incoming message queue */ +static int process_messages(struct thread *event); + +/* + * Initialize the module at startup + */ +void zebra_opaque_init(void) +{ + memset(&zo_info, 0, sizeof(zo_info)); + + pthread_mutex_init(&zo_info.mutex, NULL); + stream_fifo_init(&zo_info.in_fifo); + + zo_info.msgs_per_cycle = ZEBRA_OPAQUE_MSG_LIMIT; +} + +/* + * Start the module pthread. This step is run later than the + * 'init' step, in case zebra has fork-ed. + */ +void zebra_opaque_start(void) +{ + struct frr_pthread_attr pattr = { + .start = frr_pthread_attr_default.start, + .stop = frr_pthread_attr_default.stop + }; + + if (IS_ZEBRA_DEBUG_EVENT) + zlog_debug("%s module starting", LOG_NAME); + + /* Start pthread */ + zo_info.pthread = frr_pthread_new(&pattr, "Zebra Opaque thread", + "zebra_opaque"); + + /* Associate event 'master' */ + zo_info.master = zo_info.pthread->master; + + atomic_store_explicit(&zo_info.run, 1, memory_order_relaxed); + + /* Enqueue an initial event for the pthread */ + thread_add_event(zo_info.master, process_messages, NULL, 0, + &zo_info.t_msgs); + + /* And start the pthread */ + frr_pthread_run(zo_info.pthread, NULL); +} + +/* + * Module stop, halting the dedicated pthread; called from the main pthread. + */ +void zebra_opaque_stop(void) +{ + if (IS_ZEBRA_DEBUG_EVENT) + zlog_debug("%s module stop", LOG_NAME); + + atomic_store_explicit(&zo_info.run, 0, memory_order_relaxed); + + frr_pthread_stop(zo_info.pthread, NULL); + + frr_pthread_destroy(zo_info.pthread); + + if (IS_ZEBRA_DEBUG_EVENT) + zlog_debug("%s module stop complete", LOG_NAME); +} + +/* + * Module final cleanup, called from the zebra main pthread. + */ +void zebra_opaque_finish(void) +{ + if (IS_ZEBRA_DEBUG_EVENT) + zlog_debug("%s module shutdown", LOG_NAME); + + pthread_mutex_destroy(&zo_info.mutex); + stream_fifo_deinit(&zo_info.in_fifo); +} + +/* + * Does this module handle (intercept) the specified zapi message type? + */ +bool zebra_opaque_handles_msgid(uint16_t id) +{ + bool ret = false; + + switch (id) { + case ZEBRA_OPAQUE_MESSAGE: + case ZEBRA_OPAQUE_REGISTER: + case ZEBRA_OPAQUE_UNREGISTER: + ret = true; + break; + default: + break; + } + + return ret; +} + +/* + * Enqueue a batch of messages for processing - this is the public api + * used from the zapi processing threads. + */ +uint32_t zebra_opaque_enqueue_batch(struct stream_fifo *batch) +{ + uint32_t counter = 0; + struct stream *msg; + + /* Dequeue messages from the incoming batch, and save them + * on the module fifo. + */ + frr_with_mutex(&zo_info.mutex) { + msg = stream_fifo_pop(batch); + while (msg) { + stream_fifo_push(&zo_info.in_fifo, msg); + counter++; + msg = stream_fifo_pop(batch); + } + } + + /* Schedule module pthread to process the batch */ + if (counter > 0) { + if (IS_ZEBRA_DEBUG_RECV) + zlog_debug("%s: received %u messages", + __func__, counter); + thread_add_event(zo_info.master, process_messages, NULL, 0, + &zo_info.t_msgs); + } + + return counter; +} + +/* + * Pthread event loop, process the incoming message queue. + */ +static int process_messages(struct thread *event) +{ + struct stream_fifo fifo; + struct stream *msg; + uint32_t i; + bool need_resched = false; + + stream_fifo_init(&fifo); + + /* Check for zebra shutdown */ + if (atomic_load_explicit(&zo_info.run, memory_order_relaxed) == 0) + goto done; + + /* Dequeue some messages from the incoming queue, temporarily + * save them on the local fifo + */ + frr_with_mutex(&zo_info.mutex) { + + for (i = 0; i < zo_info.msgs_per_cycle; i++) { + msg = stream_fifo_pop(&zo_info.in_fifo); + if (msg == NULL) + break; + + stream_fifo_push(&fifo, msg); + } + + /* We may need to reschedule, if there are still + * queued messages + */ + if (stream_fifo_head(&zo_info.in_fifo) != NULL) + need_resched = true; + } + + /* Update stats */ + atomic_fetch_add_explicit(&zo_info.msgs_in, i, memory_order_relaxed); + + /* Check for zebra shutdown */ + if (atomic_load_explicit(&zo_info.run, memory_order_relaxed) == 0) { + need_resched = false; + goto done; + } + + if (IS_ZEBRA_DEBUG_RECV) + zlog_debug("%s: processing %u messages", __func__, i); + + /* Process the messages on the local fifo */ + /* TODO -- just discarding the messages for now */ + msg = stream_fifo_pop(&fifo); + while (msg) { + stream_free(msg); + msg = stream_fifo_pop(&fifo); + } + +done: + + if (need_resched) { + atomic_fetch_add_explicit(&zo_info.yields, 1, + memory_order_relaxed); + thread_add_event(zo_info.master, process_messages, NULL, 0, + &zo_info.t_msgs); + } + + /* This will also free any leftover messages, in the shutdown case */ + stream_fifo_deinit(&fifo); + + return 0; +} diff --git a/zebra/zebra_opaque.h b/zebra/zebra_opaque.h new file mode 100644 index 0000000000..a9610bfef5 --- /dev/null +++ b/zebra/zebra_opaque.h @@ -0,0 +1,63 @@ +/* + * Zebra opaque message zapi message handler + * Copyright (c) 2020 Volta Networks, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; see the file COPYING; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#ifndef _ZEBRA_OPAQUE_H +#define _ZEBRA_OPAQUE_H 1 + +/* Default for number of messages to dequeue per lock cycle */ +#define ZEBRA_OPAQUE_MSG_LIMIT 1000 + +/* + * Initialize the module at startup + */ +void zebra_opaque_init(void); + +/* + * Start the module pthread. This step is run later than the + * 'init' step, in case zebra has fork-ed. + */ +void zebra_opaque_start(void); + +/* + * Does this module handle (intercept) the specified zapi message type? + */ +bool zebra_opaque_handles_msgid(uint16_t id); + +/* + * Module stop, called from the main pthread. This is synchronous: + * once it returns, the pthread has stopped and exited. + */ +void zebra_opaque_stop(void); + +/* + * Module cleanup, called from the zebra main pthread. When it returns, + * all module cleanup is complete. + */ +void zebra_opaque_finish(void); + +/* + * Enqueue a batch of messages for processing. Returns the number dequeued + * from the batch fifo. + */ +uint32_t zebra_opaque_enqueue_batch(struct stream_fifo *batch); + + +#endif /* _ZEBRA_OPAQUE_H */ -- 2.39.5