From 188a00e014cba150f7e973394373f26709bc4054 Mon Sep 17 00:00:00 2001 From: Mark Stapp Date: Sun, 14 Apr 2019 17:16:11 -0400 Subject: [PATCH] zebra: generate updates from notifications If an async notification changes a route that's current, generate an update to keep the kernel in sync. Signed-off-by: Mark Stapp --- zebra/zebra_dplane.c | 97 +++++++++++++++++++++++++++++++++++ zebra/zebra_dplane.h | 12 +++++ zebra/zebra_mpls.c | 118 ++++++++++++++++++++++++++++++++++++------- zebra/zebra_rib.c | 60 ++++++++++++++++++---- 4 files changed, 261 insertions(+), 26 deletions(-) diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c index 2509d90812..c7f0b26b31 100644 --- a/zebra/zebra_dplane.c +++ b/zebra/zebra_dplane.c @@ -1654,6 +1654,59 @@ done: return ret; } +/* + * Update from an async notification, to bring other fibs up-to-date. + */ +enum zebra_dplane_result +dplane_route_notif_update(struct route_node *rn, + struct route_entry *re, + enum dplane_op_e op, + struct zebra_dplane_ctx *ctx) +{ + enum zebra_dplane_result ret = ZEBRA_DPLANE_REQUEST_FAILURE; + struct zebra_dplane_ctx *new_ctx = NULL; + struct nexthop *nexthop; + + if (rn == NULL || re == NULL) + goto done; + + new_ctx = dplane_ctx_alloc(); + if (new_ctx == NULL) + goto done; + + /* Init context with info from zebra data structs */ + dplane_ctx_route_init(new_ctx, op, rn, re); + + /* For add/update, need to adjust the nexthops so that we match + * the notification state, which may not be the route-entry/RIB + * state. + */ + if (op == DPLANE_OP_ROUTE_UPDATE || + op == DPLANE_OP_ROUTE_INSTALL) { + + nexthops_free(new_ctx->u.rinfo.zd_ng.nexthop); + new_ctx->u.rinfo.zd_ng.nexthop = NULL; + + copy_nexthops(&(new_ctx->u.rinfo.zd_ng.nexthop), + (rib_active_nhg(re))->nexthop, NULL); + + for (ALL_NEXTHOPS(new_ctx->u.rinfo.zd_ng, nexthop)) + UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB); + + } + + /* Capture info about the source of the notification, in 'ctx' */ + dplane_ctx_set_notif_provider(new_ctx, + dplane_ctx_get_notif_provider(ctx)); + + dplane_route_enqueue(new_ctx); + + ret = ZEBRA_DPLANE_REQUEST_QUEUED; + +done: + return ret; +} + /* * Enqueue LSP add for the dataplane. */ @@ -1687,6 +1740,50 @@ enum zebra_dplane_result dplane_lsp_delete(zebra_lsp_t *lsp) return ret; } +/* Update or un-install resulting from an async notification */ +enum zebra_dplane_result +dplane_lsp_notif_update(zebra_lsp_t *lsp, + enum dplane_op_e op, + struct zebra_dplane_ctx *notif_ctx) +{ + enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE; + int ret = EINVAL; + struct zebra_dplane_ctx *ctx = NULL; + + /* Obtain context block */ + ctx = dplane_ctx_alloc(); + if (ctx == NULL) { + ret = ENOMEM; + goto done; + } + + ret = dplane_ctx_lsp_init(ctx, op, lsp); + if (ret != AOK) + goto done; + + /* Capture info about the source of the notification */ + dplane_ctx_set_notif_provider( + ctx, + dplane_ctx_get_notif_provider(notif_ctx)); + + ret = dplane_route_enqueue(ctx); + +done: + /* Update counter */ + atomic_fetch_add_explicit(&zdplane_info.dg_lsps_in, 1, + memory_order_relaxed); + + if (ret == AOK) + result = ZEBRA_DPLANE_REQUEST_QUEUED; + else { + atomic_fetch_add_explicit(&zdplane_info.dg_lsp_errors, 1, + memory_order_relaxed); + if (ctx) + dplane_ctx_free(&ctx); + } + return result; +} + /* * Enqueue pseudowire install for the dataplane. */ diff --git a/zebra/zebra_dplane.h b/zebra/zebra_dplane.h index a4c93cefd6..b3fbb3672d 100644 --- a/zebra/zebra_dplane.h +++ b/zebra/zebra_dplane.h @@ -317,6 +317,13 @@ enum zebra_dplane_result dplane_sys_route_add(struct route_node *rn, enum zebra_dplane_result dplane_sys_route_del(struct route_node *rn, struct route_entry *re); +/* Update from an async notification, to bring other fibs up-to-date */ +enum zebra_dplane_result dplane_route_notif_update( + struct route_node *rn, + struct route_entry *re, + enum dplane_op_e op, + struct zebra_dplane_ctx *ctx); + /* * Enqueue LSP change operations for the dataplane. */ @@ -324,6 +331,11 @@ enum zebra_dplane_result dplane_lsp_add(zebra_lsp_t *lsp); enum zebra_dplane_result dplane_lsp_update(zebra_lsp_t *lsp); enum zebra_dplane_result dplane_lsp_delete(zebra_lsp_t *lsp); +/* Update or un-install resulting from an async notification */ +enum zebra_dplane_result dplane_lsp_notif_update(zebra_lsp_t *lsp, + enum dplane_op_e op, + struct zebra_dplane_ctx *ctx); + /* * Enqueue pseudowire operations for the dataplane. */ diff --git a/zebra/zebra_mpls.c b/zebra/zebra_mpls.c index 18b480bdf9..48366417dd 100644 --- a/zebra/zebra_mpls.c +++ b/zebra/zebra_mpls.c @@ -1823,6 +1823,7 @@ void zebra_mpls_process_dplane_notify(struct zebra_dplane_ctx *ctx) struct nexthop *nexthop; const struct nexthop *ctx_nexthop; int start_count = 0, end_count = 0; /* Installed counts */ + bool changed_p = false; bool is_debug = (IS_ZEBRA_DEBUG_DPLANE | IS_ZEBRA_DEBUG_MPLS); if (is_debug) @@ -1847,8 +1848,11 @@ void zebra_mpls_process_dplane_notify(struct zebra_dplane_ctx *ctx) /* * The dataplane/forwarding plane is notifying zebra about the state - * of the nexthops associated with this LSP. We bring the zebra - * nexthop state into sync with the forwarding-plane state. + * of the nexthops associated with this LSP. First, we take a + * pre-scan pass to determine whether the LSP has transitioned + * from installed -> uninstalled. In that case, we need to have + * the existing state of the LSP objects available before making + * any changes. */ for (nhlfe = lsp->nhlfe_list; nhlfe; nhlfe = nhlfe->next) { char buf[NEXTHOP_STRLEN]; @@ -1890,41 +1894,121 @@ void zebra_mpls_process_dplane_notify(struct zebra_dplane_ctx *ctx) buf, tstr); } - /* Bring zebra nhlfe install state into sync */ + /* Test zebra nhlfe install state */ if (CHECK_FLAG(ctx_nhlfe->flags, NHLFE_FLAG_INSTALLED)) { - SET_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED); + + if (!CHECK_FLAG(nhlfe->flags, + NHLFE_FLAG_INSTALLED)) + changed_p = true; /* Update counter */ end_count++; - } else + } else { + + if (CHECK_FLAG(nhlfe->flags, + NHLFE_FLAG_INSTALLED)) + changed_p = true; + } + + } else { + /* Not mentioned in lfib set -> uninstalled */ + if (CHECK_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED) || + CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE) || + CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB)) { + changed_p = true; + } + + if (is_debug) + zlog_debug("LSP dplane notif: no match, nh %s", + buf); + } + } + + if (is_debug) + zlog_debug("LSP dplane notif: lfib start_count %d, end_count %d%s", + start_count, end_count, + changed_p ? ", changed" : ""); + + /* + * Has the LSP become uninstalled? + */ + if (start_count > 0 && end_count == 0) { + /* Inform other lfibs */ + dplane_lsp_notif_update(lsp, DPLANE_OP_LSP_DELETE, ctx); + } + + /* + * Now we take a second pass and bring the zebra + * nexthop state into sync with the forwarding-plane state. + */ + for (nhlfe = lsp->nhlfe_list; nhlfe; nhlfe = nhlfe->next) { + char buf[NEXTHOP_STRLEN]; + + nexthop = nhlfe->nexthop; + if (!nexthop) + continue; + + ctx_nexthop = NULL; + for (ctx_nhlfe = dplane_ctx_get_nhlfe(ctx); + ctx_nhlfe; ctx_nhlfe = ctx_nhlfe->next) { + + ctx_nexthop = ctx_nhlfe->nexthop; + if (!ctx_nexthop) + continue; + + if ((ctx_nexthop->type == nexthop->type) && + nexthop_same(ctx_nexthop, nexthop)) { + /* Matched */ + break; + } + } + + if (is_debug) + nexthop2str(nexthop, buf, sizeof(buf)); + + if (ctx_nhlfe && ctx_nexthop) { + + /* Bring zebra nhlfe install state into sync */ + if (CHECK_FLAG(ctx_nhlfe->flags, + NHLFE_FLAG_INSTALLED)) { + + SET_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED); + + } else { + UNSET_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED); + } if (CHECK_FLAG(ctx_nhlfe->nexthop->flags, - NEXTHOP_FLAG_FIB)) + NEXTHOP_FLAG_FIB)) { + SET_FLAG(nhlfe->nexthop->flags, + NEXTHOP_FLAG_ACTIVE); SET_FLAG(nhlfe->nexthop->flags, NEXTHOP_FLAG_FIB); - else + } else { + UNSET_FLAG(nhlfe->nexthop->flags, + NEXTHOP_FLAG_ACTIVE); UNSET_FLAG(nhlfe->nexthop->flags, NEXTHOP_FLAG_FIB); + } + } else { /* Not mentioned in lfib set -> uninstalled */ + UNSET_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED); UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB); - - if (is_debug) - zlog_debug("LSP dplane notif: no match, nh %s", - buf); + UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE); } } - if (is_debug) - zlog_debug("LSP dplane notif: lfib start_count %d, end_count %d", - start_count, end_count); - - if (end_count > 0) + if (end_count > 0) { SET_FLAG(lsp->flags, LSP_FLAG_INSTALLED); - else { + + if (changed_p) + dplane_lsp_notif_update(lsp, DPLANE_OP_LSP_UPDATE, ctx); + + } else { UNSET_FLAG(lsp->flags, LSP_FLAG_INSTALLED); clear_nhlfe_installed(lsp); } diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c index d1311dfdd2..557f426502 100644 --- a/zebra/zebra_rib.c +++ b/zebra/zebra_rib.c @@ -2070,7 +2070,17 @@ static bool rib_update_re_from_ctx(struct route_entry *re, (changed_p ? "true" : "false")); ctxnhg = dplane_ctx_get_ng(ctx); - copy_nexthops(&(re->fib_ng.nexthop), ctxnhg->nexthop, NULL); + + if (ctxnhg->nexthop) + copy_nexthops(&(re->fib_ng.nexthop), ctxnhg->nexthop, NULL); + else { + /* Bit of a special case when the fib has _no_ installed + * nexthops. + */ + nexthop = nexthop_new(); + nexthop->type = NEXTHOP_TYPE_IPV4; + nexthop_add(&(re->fib_ng.nexthop), nexthop); + } done: return changed_p; @@ -2444,7 +2454,8 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx) goto done; } - /* Perform follow-up work if the actual status of the prefix + /* + * Perform follow-up work if the actual status of the prefix * changed. */ @@ -2454,10 +2465,16 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx) end_count++; } - /* Various fib transitions: from installed to - * not-installed, or not-installed to installed. + /* Various fib transitions: changed nexthops; from installed to + * not-installed; or not-installed to installed. */ - if (start_count == 0 && end_count > 0) { + if (start_count > 0 && end_count > 0) { + + /* Changed nexthops - update kernel/others */ + dplane_route_notif_update(rn, re, + DPLANE_OP_ROUTE_UPDATE, ctx); + + } else if (start_count == 0 && end_count > 0) { if (debug_p) zlog_debug("%u:%s installed transition from dplane notification", dplane_ctx_get_vrf(ctx), dest_str); @@ -2467,6 +2484,9 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx) */ SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED); + /* Changed nexthops - update kernel/others */ + dplane_route_notif_update(rn, re, DPLANE_OP_ROUTE_INSTALL, ctx); + /* Redistribute, lsp, and nht update */ redistribute_update(dest_pfx, src_pfx, re, NULL); @@ -2488,6 +2508,9 @@ static void rib_process_dplane_notify(struct zebra_dplane_ctx *ctx) */ UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED); + /* Changed nexthops - update kernel/others */ + dplane_route_notif_update(rn, re, DPLANE_OP_ROUTE_DELETE, ctx); + /* Redistribute, lsp, and nht update */ redistribute_delete(dest_pfx, src_pfx, re); @@ -3725,8 +3748,18 @@ static int rib_process_dplane_results(struct thread *thread) case DPLANE_OP_ROUTE_INSTALL: case DPLANE_OP_ROUTE_UPDATE: case DPLANE_OP_ROUTE_DELETE: - rib_process_result(ctx); - break; + { + /* Bit of special case for route updates + * that were generated by async notifications: + * we don't want to continue processing these + * in the rib. + */ + if (dplane_ctx_get_notif_provider(ctx) == 0) + rib_process_result(ctx); + else + dplane_ctx_fini(&ctx); + } + break; case DPLANE_OP_ROUTE_NOTIFY: rib_process_dplane_notify(ctx); @@ -3735,8 +3768,17 @@ static int rib_process_dplane_results(struct thread *thread) case DPLANE_OP_LSP_INSTALL: case DPLANE_OP_LSP_UPDATE: case DPLANE_OP_LSP_DELETE: - zebra_mpls_lsp_dplane_result(ctx); - break; + { + /* Bit of special case for LSP updates + * that were generated by async notifications: + * we don't want to continue processing these. + */ + if (dplane_ctx_get_notif_provider(ctx) == 0) + zebra_mpls_lsp_dplane_result(ctx); + else + dplane_ctx_fini(&ctx); + } + break; case DPLANE_OP_LSP_NOTIFY: zebra_mpls_process_dplane_notify(ctx); -- 2.39.5