From c9d17fe85ea55f011381dc98ba927bd107fd9d19 Mon Sep 17 00:00:00 2001 From: Mark Stapp Date: Tue, 11 Sep 2018 16:42:36 -0400 Subject: [PATCH] zebra: improve dataplane shutdown checks Update the dataplane shutdown checks to include the providers. Also revise the typedef for provider structs to make const work. Signed-off-by: Mark Stapp --- zebra/zebra_dplane.c | 71 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index 24975697ea..5fd27dc830 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -162,8 +162,10 @@ struct zebra_dplane_provider { dplane_provider_fini_fp dp_fini; _Atomic uint32_t dp_in_counter; + _Atomic uint32_t dp_in_queued; _Atomic uint32_t dp_in_max; _Atomic uint32_t dp_out_counter; + _Atomic uint32_t dp_out_queued; _Atomic uint32_t dp_out_max; _Atomic uint32_t dp_error_counter; @@ -959,8 +961,8 @@ int dplane_show_provs_helper(struct vty *vty, bool detailed) out_max = atomic_load_explicit(&prov->dp_out_max, memory_order_relaxed); - vty_out(vty, "%s (%u): in: %"PRIu64", max: %"PRIu64", " - "out: %"PRIu64", max: %"PRIu64"\n", + vty_out(vty, "%s (%u): in: %"PRIu64", q_max: %"PRIu64", " + "out: %"PRIu64", q_max: %"PRIu64"\n", prov->dp_name, prov->dp_id, in, in_max, out, out_max); DPLANE_LOCK(); @@ -1081,6 +1083,9 @@ struct zebra_dplane_ctx *dplane_provider_dequeue_in_ctx( ctx = TAILQ_FIRST(&(prov->dp_ctx_in_q)); if (ctx) { TAILQ_REMOVE(&(prov->dp_ctx_in_q), ctx, zd_q_entries); + + atomic_fetch_sub_explicit(&prov->dp_in_queued, 1, + memory_order_relaxed); } if (dplane_provider_is_threaded(prov)) @@ -1114,6 +1119,10 @@ int dplane_provider_dequeue_in_list(struct zebra_dplane_provider *prov, } } + if (ret > 0) + atomic_fetch_sub_explicit(&prov->dp_in_queued, ret, + memory_order_relaxed); + if (dplane_provider_is_threaded(prov)) DPLANE_PROV_UNLOCK(prov); @@ -1272,6 +1281,10 @@ static int test_dplane_process_func(struct zebra_dplane_provider *prov) dplane_provider_enqueue_out_ctx(prov, ctx); } + if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) + zlog_debug("dplane provider '%s': processed %d", + dplane_provider_get_name(prov), counter); + /* Ensure that we'll run the work loop again if there's still * more work to do. */ @@ -1342,7 +1355,7 @@ bool dplane_is_in_shutdown(void) * early during zebra shutdown, as a signal to stop new work and prepare * for updates generated by shutdown/cleanup activity, as zebra tries to * remove everything it's responsible for. - * NB: This runs in the main zebra thread context. + * NB: This runs in the main zebra pthread context. */ void zebra_dplane_pre_finish(void) { @@ -1351,7 +1364,7 @@ void zebra_dplane_pre_finish(void) zdplane_info.dg_is_shutdown = true; - /* Notify provider(s) of pending shutdown */ + /* TODO -- Notify provider(s) of pending shutdown */ } /* @@ -1360,7 +1373,9 @@ void zebra_dplane_pre_finish(void) */ static bool dplane_work_pending(void) { + bool ret = false; struct zebra_dplane_ctx *ctx; + struct zebra_dplane_provider *prov; /* TODO -- just checking incoming/pending work for now, must check * providers @@ -1368,10 +1383,40 @@ static bool dplane_work_pending(void) DPLANE_LOCK(); { ctx = TAILQ_FIRST(&zdplane_info.dg_route_ctx_q); + prov = TAILQ_FIRST(&zdplane_info.dg_providers_q); } DPLANE_UNLOCK(); - return (ctx != NULL); + if (ctx != NULL) { + ret = true; + goto done; + } + + while (prov) { + + if (dplane_provider_is_threaded(prov)) + DPLANE_PROV_LOCK(prov); + + ctx = TAILQ_FIRST(&(prov->dp_ctx_in_q)); + if (ctx == NULL) + ctx = TAILQ_FIRST(&(prov->dp_ctx_out_q)); + + if (dplane_provider_is_threaded(prov)) + DPLANE_PROV_UNLOCK(prov); + + if (ctx != NULL) + break; + + DPLANE_LOCK(); + prov = TAILQ_NEXT(prov, dp_prov_link); + DPLANE_UNLOCK(); + } + + if (ctx != NULL) + ret = true; + +done: + return ret; } /* @@ -1443,7 +1488,7 @@ static int dplane_thread_loop(struct thread *event) struct zebra_dplane_provider *prov; struct zebra_dplane_ctx *ctx, *tctx; int limit, counter, error_counter; - uint64_t lval; + uint64_t curr, high; /* Capture work limit per cycle */ limit = zdplane_info.dg_updates_per_cycle; @@ -1534,10 +1579,16 @@ static int dplane_thread_loop(struct thread *event) TAILQ_CONCAT(&(prov->dp_ctx_in_q), &work_list, zd_q_entries); - lval = atomic_add_fetch_explicit(&prov->dp_in_counter, counter, - memory_order_relaxed); - if (lval > prov->dp_in_max) - atomic_store_explicit(&prov->dp_in_max, lval, + atomic_fetch_add_explicit(&prov->dp_in_counter, counter, + memory_order_relaxed); + atomic_fetch_add_explicit(&prov->dp_in_queued, counter, + memory_order_relaxed); + curr = atomic_load_explicit(&prov->dp_in_queued, + memory_order_relaxed); + high = atomic_load_explicit(&prov->dp_in_max, + memory_order_relaxed); + if (curr > high) + atomic_store_explicit(&prov->dp_in_max, curr, memory_order_relaxed); if (dplane_provider_is_threaded(prov)) -- 2.39.5