/* Nexthops */
struct nexthop_group zd_ng;
- /* "Previous" nexthops, used only in route update case without netlink */
+ /* "Previous" nexthops, used only in route updates without netlink */
struct nexthop_group zd_old_ng;
/* TODO -- use fixed array of nexthops, to avoid mallocs? */
/* Results callback registered by zebra 'core' */
dplane_results_fp dg_results_cb;
- /* Sentinel for shutdown */
+ /* Sentinel for beginning of shutdown */
+ volatile bool dg_is_shutdown;
+
+ /* Sentinel for end of shutdown */
volatile bool dg_run;
/* Route-update context queue inbound to the dataplane */
_Atomic uint64_t dg_routes_in;
_Atomic uint32_t dg_routes_queued;
+ _Atomic uint32_t dg_routes_queued_max;
_Atomic uint64_t dg_route_errors;
/* Event-delivery context 'master' for the dplane */
/* Event/'thread' pointer for queued updates */
struct thread *dg_t_update;
+ /* Event pointer for pending shutdown check loop */
+ struct thread *dg_t_shutdown_check;
+
} zdplane_g;
/*
}
if ((*pctx)->zd_old_ng.nexthop) {
+ /* This deals with recursive nexthops too */
nexthops_free((*pctx)->zd_old_ng.nexthop);
}
/* TODO -- maybe use array of nexthops to avoid allocs? */
/* Ensure that the dplane's nexthop flag is clear. */
- for (ALL_NEXTHOPS(ctx->zd_ng, nexthop)) {
+ for (ALL_NEXTHOPS(ctx->zd_ng, nexthop))
UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
- }
/* Trying out the sequence number idea, so we can try to detect
* when a result is stale.
memory_order_relaxed);
if (ret == AOK) {
- atomic_fetch_add_explicit(&zdplane_g.dg_routes_queued, 1,
- memory_order_relaxed);
+ uint32_t high, curr;
+
+ curr = atomic_fetch_add_explicit(&zdplane_g.dg_routes_queued, 1,
+ memory_order_seq_cst);
+
+ /* We don't have add_and_fetch - sigh */
+ curr++;
+
+ /* Maybe update high-water counter also */
+ high = atomic_load_explicit(&zdplane_g.dg_routes_queued_max,
+ memory_order_seq_cst);
+ while (high < curr) {
+ if (atomic_compare_exchange_weak_explicit(
+ &zdplane_g.dg_routes_queued_max,
+ &high, curr,
+ memory_order_seq_cst,
+ memory_order_seq_cst))
+ break;
+ }
+
result = ZEBRA_DPLANE_REQUEST_QUEUED;
} else if (ctx) {
atomic_fetch_add_explicit(&zdplane_g.dg_route_errors, 1,
*/
int dplane_show_helper(struct vty *vty, bool detailed)
{
- uint64_t queued, errs, incoming;
+ uint64_t queued, queue_max, errs, incoming;
+ /* Using atomics because counters are being changed in different
+ * contexts.
+ */
incoming = atomic_load_explicit(&zdplane_g.dg_routes_in,
memory_order_relaxed);
queued = atomic_load_explicit(&zdplane_g.dg_routes_queued,
memory_order_relaxed);
+ queue_max = atomic_load_explicit(&zdplane_g.dg_routes_queued_max,
+ memory_order_relaxed);
errs = atomic_load_explicit(&zdplane_g.dg_route_errors,
memory_order_relaxed);
vty_out(vty, "Route updates: %"PRIu64"\n", incoming);
vty_out(vty, "Route update errors: %"PRIu64"\n", errs);
vty_out(vty, "Route update queue depth: %"PRIu64"\n", queued);
+ vty_out(vty, "Route update queue max: %"PRIu64"\n", queue_max);
return CMD_SUCCESS;
}
zdplane_g.dg_master = zebra->master;
}
+/* Indicates zebra shutdown/exit is in progress. Some operations may be
+ * simplified or skipped during shutdown processing.
+ */
+bool dplane_is_in_shutdown(void)
+{
+ return zdplane_g.dg_is_shutdown;
+}
+
+/*
+ * Early or pre-shutdown, de-init notification api. This runs pretty
+ * early during zebra shutdown, as a signal to stop new work and prepare
+ * for updates generated by shutdown/cleanup activity, as zebra tries to
+ * remove everything it's responsible for.
+ * NB: This runs in the main zebra thread context.
+ */
+void zebra_dplane_pre_finish(void)
+{
+ if (IS_ZEBRA_DEBUG_DPLANE)
+ zlog_debug("Zebra dataplane pre-fini called");
+
+ zdplane_g.dg_is_shutdown = true;
+
+ /* Notify provider(s) of pending shutdown */
+}
+
+/*
+ * Utility to determine whether work remains enqueued within the dplane;
+ * used during system shutdown processing.
+ */
+static bool dplane_work_pending(void)
+{
+ dplane_ctx_h ctx;
+
+ /* TODO -- just checking incoming/pending work for now */
+ DPLANE_LOCK();
+ {
+ ctx = TAILQ_FIRST(&zdplane_g.dg_route_ctx_q);
+ }
+ DPLANE_UNLOCK();
+
+ return (ctx != NULL);
+}
+
+/*
+ * Shutdown-time intermediate callback, used to determine when all pending
+ * in-flight updates are done. If there's still work to do, reschedules itself.
+ * If all work is done, schedules an event to the main zebra thread for
+ * final zebra shutdown.
+ * This runs in the dplane pthread context.
+ */
+static int dplane_check_shutdown_status(struct thread *event)
+{
+ if (IS_ZEBRA_DEBUG_DPLANE)
+ zlog_debug("Zebra dataplane shutdown status check called");
+
+ if (dplane_work_pending()) {
+ /* Reschedule dplane check on a short timer */
+ thread_add_timer_msec(zdplane_g.dg_master,
+ dplane_check_shutdown_status,
+ NULL, 100,
+ &zdplane_g.dg_t_shutdown_check);
+
+ /* TODO - give up and stop waiting after a short time? */
+
+ } else {
+ /* We appear to be done - schedule a final callback event
+ * for the zebra main pthread.
+ */
+ thread_add_event(zebrad.master, zebra_finalize, NULL, 0, NULL);
+ }
+
+ return 0;
+}
+
/*
* Shutdown, de-init api. This runs pretty late during shutdown,
- * because zebra tries to free/remove/uninstall all routes during shutdown.
+ * after zebra has tried to free/remove/uninstall all routes during shutdown.
+ * At this point, dplane work may still remain to be done, so we can't just
+ * blindly terminate. If there's still work to do, we'll periodically check
+ * and when done, we'll enqueue a task to the zebra main thread for final
+ * termination processing.
+ *
* NB: This runs in the main zebra thread context.
*/
void zebra_dplane_finish(void)
{
- /* Wait until all pending updates are processed */
+ if (IS_ZEBRA_DEBUG_DPLANE)
+ zlog_debug("Zebra dataplane fini called");
+
+ thread_add_event(zdplane_g.dg_master,
+ dplane_check_shutdown_status, NULL, 0,
+ &zdplane_g.dg_t_shutdown_check);
+}
+
+/*
+ * Final phase of shutdown, after all work enqueued to dplane has been
+ * processed. This is called from the zebra main pthread context.
+ */
+void zebra_dplane_shutdown(void)
+{
+ if (IS_ZEBRA_DEBUG_DPLANE)
+ zlog_debug("Zebra dataplane shutdown called");
/* Stop dplane thread, if it's running */
THREAD_OFF(zdplane_g.dg_t_update);
- /* Notify provider(s) of shutdown */
+ /* TODO */
+ /* frr_pthread_stop(...) */
+
+ /* Notify provider(s) of final shutdown */
/* Clean-up provider objects */
/* Clean queue(s) */
-
}
/*
void zebra_dplane_init(void)
{
zebra_dplane_init_internal(&zebrad);
-
- /* Finalize/cleanup code is called quite late during zebra shutdown;
- * zebra expects to try to clean up all vrfs and all routes during
- * shutdown, so the dplane must be available until very late.
- */
}
projects / mirror / frr.git / commitdiff