From 4c206c8f74f7c9309067994d149ead0c48e9c49a Mon Sep 17 00:00:00 2001 From: Mark Stapp Date: Fri, 21 Dec 2018 14:12:33 -0500 Subject: [PATCH] zebra: pass lists of results from dataplane to zebra Pass lists of results back to zebra from the dataplane subsystem (and pthread). This helps reduce the lock/unlock cycles when zebra is busy. Also remove a couple of typedefs that made their way into the dataplane header file - those violate the FRR style guidelines. Signed-off-by: Mark Stapp --- zebra/zebra_dplane.c | 44 +++++++++++++------------------------- zebra/zebra_dplane.h | 50 +++++++++++++++++++------------------------- zebra/zebra_rib.c | 32 +++++++++++++++++++--------- 3 files changed, 58 insertions(+), 68 deletions(-) diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index ba0f1b41aa..79e9e7edad 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -157,9 +157,9 @@ struct zebra_dplane_provider { /* Flags */ int dp_flags; - dplane_provider_process_fp dp_fp; + int (*dp_fp)(struct zebra_dplane_provider *prov); - dplane_provider_fini_fp dp_fini; + int (*dp_fini)(struct zebra_dplane_provider *prov, bool early_p); _Atomic uint32_t dp_in_counter; _Atomic uint32_t dp_in_queued; @@ -189,7 +189,7 @@ static struct zebra_dplane_globals { pthread_mutex_t dg_mutex; /* Results callback registered by zebra 'core' */ - dplane_results_fp dg_results_cb; + int (*dg_results_cb)(struct dplane_ctx_q *ctxlist); /* Sentinel for beginning of shutdown */ volatile bool dg_is_shutdown; @@ -988,8 +988,9 @@ int dplane_show_provs_helper(struct vty *vty, bool detailed) int dplane_provider_register(const char *name, enum dplane_provider_prio prio, int flags, - dplane_provider_process_fp fp, - dplane_provider_fini_fp fini_fp, + int (*fp)(struct zebra_dplane_provider *), + int (*fini_fp)(struct zebra_dplane_provider *, + bool early), void *data) { int ret = 0; @@ -1209,15 +1210,6 @@ int dplane_provider_work_ready(void) return AOK; } -/* - * Zebra registers a results callback with the dataplane system - */ -int dplane_results_register(dplane_results_fp fp) -{ - zdplane_info.dg_results_cb = fp; - return AOK; -} - /* * Kernel dataplane provider */ @@ -1677,27 +1669,20 @@ static int dplane_thread_loop(struct thread *event) counter, error_counter); /* - * TODO -- I'd rather hand lists through the api to zebra main, + * Hand lists through the api to zebra main, * to reduce the number of lock/unlock cycles */ - for (ctx = TAILQ_FIRST(&error_list); ctx; ) { - TAILQ_REMOVE(&error_list, ctx, zd_q_entries); - /* Call through to zebra main */ - (*zdplane_info.dg_results_cb)(ctx); - - ctx = TAILQ_FIRST(&error_list); - } + /* Call through to zebra main */ + (zdplane_info.dg_results_cb)(&error_list); + TAILQ_INIT(&error_list); - for (ctx = TAILQ_FIRST(&work_list); ctx; ) { - TAILQ_REMOVE(&work_list, ctx, zd_q_entries); - /* Call through to zebra main */ - (*zdplane_info.dg_results_cb)(ctx); + /* Call through to zebra main */ + (zdplane_info.dg_results_cb)(&work_list); - ctx = TAILQ_FIRST(&work_list); - } + TAILQ_INIT(&work_list); done: return 0; @@ -1782,7 +1767,8 @@ void zebra_dplane_start(void) /* * Initialize the dataplane module at startup; called by zebra rib_init() */ -void zebra_dplane_init(void) +void zebra_dplane_init(int (*results_fp)(struct dplane_ctx_q *)) { zebra_dplane_init_internal(&zebrad); + zdplane_info.dg_results_cb = results_fp; } diff --git a/zebra/zebra_dplane.h b/zebra/zebra_dplane.h index 1336850510..1d23ef0f82 100644 --- a/zebra/zebra_dplane.h +++ b/zebra/zebra_dplane.h @@ -250,21 +250,6 @@ enum dplane_provider_prio { DPLANE_PRIO_LAST }; -/* Provider's entry-point for incoming work, called in the context of the - * dataplane pthread. The dataplane pthread enqueues any new work to the - * provider's 'inbound' queue, then calls the callback. The dataplane - * then checks the provider's outbound queue. - */ -typedef int (*dplane_provider_process_fp)(struct zebra_dplane_provider *prov); - -/* Provider's entry-point for shutdown and cleanup. Called with 'early' - * during shutdown, to indicate that the dataplane subsystem is allowing - * work to move through the providers and finish. When called without 'early', - * the provider should release all resources (if it has any allocated). - */ -typedef int (*dplane_provider_fini_fp)(struct zebra_dplane_provider *prov, - bool early); - /* Flags values used during provider registration. */ #define DPLANE_PROV_FLAGS_DEFAULT 0x0 @@ -275,11 +260,25 @@ typedef int (*dplane_provider_fini_fp)(struct zebra_dplane_provider *prov, /* Provider registration: ordering or priority value, callbacks, and optional * opaque data value. */ + +/* Providers offer an entry-point for incoming work, called in the context of + * the dataplane pthread. The dataplane pthread enqueues any new work to the + * provider's 'inbound' queue, then calls the callback. The dataplane + * then checks the provider's outbound queue for completed work. + */ + +/* Providers offer an entry-point for shutdown and cleanup. This is called + * with 'early' during shutdown, to indicate that the dataplane subsystem + * is allowing work to move through the providers and finish. + * When called without 'early', the provider should release + * all resources (if it has any allocated). + */ int dplane_provider_register(const char *name, enum dplane_provider_prio prio, int flags, - dplane_provider_process_fp fp, - dplane_provider_fini_fp fini_fp, + int (*fp)(struct zebra_dplane_provider *), + int (*fini_fp)(struct zebra_dplane_provider *, + bool early), void *data); /* Accessors for provider attributes */ @@ -317,21 +316,14 @@ int dplane_provider_dequeue_in_list(struct zebra_dplane_provider *prov, void dplane_provider_enqueue_out_ctx(struct zebra_dplane_provider *prov, struct zebra_dplane_ctx *ctx); -/* - * Zebra registers a results callback with the dataplane. The callback is - * called in the dataplane pthread context, so the expectation is that the - * context is queued for the zebra main pthread or that processing - * is very limited. - */ -typedef int (*dplane_results_fp)(struct zebra_dplane_ctx *ctx); - -int dplane_results_register(dplane_results_fp fp); - /* * Initialize the dataplane modules at zebra startup. This is currently called - * by the rib module. + * by the rib module. Zebra registers a results callback with the dataplane. + * The callback is called in the dataplane pthread context, + * so the expectation is that the contexts are queued for the zebra + * main pthread. */ -void zebra_dplane_init(void); +void zebra_dplane_init(int (*) (struct dplane_ctx_q *)); /* * Start the dataplane pthread. This step needs to be run later than the diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index 5d6eac7533..56ded6e838 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -3194,21 +3194,34 @@ void rib_close_table(struct route_table *table) static int rib_process_dplane_results(struct thread *thread) { struct zebra_dplane_ctx *ctx; + struct dplane_ctx_q ctxlist; + + /* Dequeue a list of completed updates with one lock/unlock cycle */ do { + TAILQ_INIT(&ctxlist); + /* Take lock controlling queue of results */ pthread_mutex_lock(&dplane_mutex); { /* Dequeue context block */ - ctx = dplane_ctx_dequeue(&rib_dplane_q); + dplane_ctx_list_append(&ctxlist, &rib_dplane_q); } pthread_mutex_unlock(&dplane_mutex); - if (ctx) - rib_process_after(ctx); - else + /* Dequeue context block */ + ctx = dplane_ctx_dequeue(&ctxlist); + + /* If we've emptied the results queue, we're done */ + if (ctx == NULL) break; + while (ctx) { + rib_process_after(ctx); + + ctx = dplane_ctx_dequeue(&ctxlist); + } + } while (1); /* Check for nexthop tracking processing after finishing with results */ @@ -3222,17 +3235,17 @@ static int rib_process_dplane_results(struct thread *thread) * the dataplane pthread. We enqueue the results here for processing by * the main thread later. */ -static int rib_dplane_results(struct zebra_dplane_ctx *ctx) +static int rib_dplane_results(struct dplane_ctx_q *ctxlist) { /* Take lock controlling queue of results */ pthread_mutex_lock(&dplane_mutex); { - /* Enqueue context block */ - dplane_ctx_enqueue_tail(&rib_dplane_q, ctx); + /* Enqueue context blocks */ + dplane_ctx_list_append(&rib_dplane_q, ctxlist); } pthread_mutex_unlock(&dplane_mutex); - /* Ensure event is signalled to zebra main thread */ + /* Ensure event is signalled to zebra main pthread */ thread_add_event(zebrad.master, rib_process_dplane_results, NULL, 0, &t_dplane); @@ -3247,8 +3260,7 @@ void rib_init(void) /* Init dataplane, and register for results */ pthread_mutex_init(&dplane_mutex, NULL); TAILQ_INIT(&rib_dplane_q); - zebra_dplane_init(); - dplane_results_register(rib_dplane_results); + zebra_dplane_init(rib_dplane_results); } /* -- 2.39.5