zebra: move LSP updates into dataplane subsystem

Start performing LSP updates through the async dataplane
subsystem. This is plumbed through for linux/netlink.

Signed-off-by: Mark Stapp <mjs@voltanet.io>

5 files changed, 182 insertions, 21 deletions

diff --git a/zebra/zebra_mpls.c b/zebra/zebra_mpls.c
index db2fcae5b7..9806f6836c 100644
--- a/zebra/zebra_mpls.c
+++ b/zebra/zebra_mpls.c
@@ -887,6 +887,7 @@ static wq_item_status lsp_process(struct work_queue *wq, void *data)
 	zebra_nhlfe_t *oldbest, *newbest;
 	char buf[BUFSIZ], buf2[BUFSIZ];
 	struct zebra_vrf *zvrf = vrf_info_lookup(VRF_DEFAULT);
+	enum zebra_dplane_result res;
 
 	lsp = (zebra_lsp_t *)data;
 	if (!lsp) // unexpected
@@ -916,13 +917,20 @@ static wq_item_status lsp_process(struct work_queue *wq, void *data)
 		if (newbest) {
 
 			UNSET_FLAG(lsp->flags, LSP_FLAG_CHANGED);
-			switch (kernel_add_lsp(lsp)) {
+
+			switch (dplane_lsp_add(lsp)) {
 			case ZEBRA_DPLANE_REQUEST_QUEUED:
-				flog_err(
-					EC_ZEBRA_DP_INVALID_RC,
-					"No current DataPlane interfaces can return this, please fix");
+				/* Set 'installed' flag so we will know
+				 * that an install is in-flight.
+				 */
+				SET_FLAG(lsp->flags, LSP_FLAG_INSTALLED);
+
+				zvrf->lsp_installs_queued++;
 				break;
 			case ZEBRA_DPLANE_REQUEST_FAILURE:
+				flog_warn(EC_ZEBRA_LSP_INSTALL_FAILURE,
+					  "LSP Install Failure: %u",
+					  lsp->ile.in_label);
 				break;
 			case ZEBRA_DPLANE_REQUEST_SUCCESS:
 				zvrf->lsp_installs++;
@@ -932,14 +940,22 @@ static wq_item_status lsp_process(struct work_queue *wq, void *data)
 	} else {
 		/* Installed, may need an update and/or delete. */
 		if (!newbest) {
+			res = dplane_lsp_delete(lsp);
+
+			/* We do some of the lsp cleanup immediately for
+			 * deletes.
+			 */
+			UNSET_FLAG(lsp->flags, LSP_FLAG_INSTALLED);
+			clear_nhlfe_installed(lsp);
 
-			switch (kernel_del_lsp(lsp)) {
+			switch (res) {
 			case ZEBRA_DPLANE_REQUEST_QUEUED:
-				flog_err(
-					EC_ZEBRA_DP_INVALID_RC,
-					"No current DataPlane interfaces can return this, please fix");
+				zvrf->lsp_removals_queued++;
 				break;
 			case ZEBRA_DPLANE_REQUEST_FAILURE:
+				flog_warn(EC_ZEBRA_LSP_DELETE_FAILURE,
+					  "LSP Deletion Failure: %u",
+					  lsp->ile.in_label);
 				break;
 			case ZEBRA_DPLANE_REQUEST_SUCCESS:
 				zvrf->lsp_removals++;
@@ -950,7 +966,10 @@ static wq_item_status lsp_process(struct work_queue *wq, void *data)
 			struct nexthop *nexthop;
 
 			UNSET_FLAG(lsp->flags, LSP_FLAG_CHANGED);
-			UNSET_FLAG(lsp->flags, LSP_FLAG_INSTALLED);
+
+			/* We leave the INSTALLED flag set here
+			 * so we know an update in in-flight.
+			 */
 
 			/*
 			 * Any NHLFE that was installed but is not
@@ -973,13 +992,14 @@ static wq_item_status lsp_process(struct work_queue *wq, void *data)
 				}
 			}
 
-			switch (kernel_upd_lsp(lsp)) {
+			switch (dplane_lsp_update(lsp)) {
 			case ZEBRA_DPLANE_REQUEST_QUEUED:
-				flog_err(
-					EC_ZEBRA_DP_INVALID_RC,
-					"No current DataPlane interfaces can return this, please fix");
+				zvrf->lsp_installs_queued++;
 				break;
 			case ZEBRA_DPLANE_REQUEST_FAILURE:
+				flog_warn(EC_ZEBRA_LSP_INSTALL_FAILURE,
+					  "LSP Update Failure: %u",
+					  lsp->ile.in_label);
 				break;
 			case ZEBRA_DPLANE_REQUEST_SUCCESS:
 				zvrf->lsp_installs++;
@@ -1756,6 +1776,87 @@ void kernel_lsp_pass_fail(zebra_lsp_t *lsp, enum zebra_dplane_status res)
 }
 
 /*
+ * Process LSP update results from zebra dataplane.
+ */
+void zebra_mpls_lsp_dplane_result(struct zebra_dplane_ctx *ctx)
+{
+	struct zebra_vrf *zvrf;
+	zebra_ile_t tmp_ile;
+	struct hash *lsp_table;
+	zebra_lsp_t *lsp;
+	zebra_nhlfe_t *nhlfe;
+	struct nexthop *nexthop;
+	enum dplane_op_e op;
+	enum zebra_dplane_status status;
+
+	op = dplane_ctx_get_op(ctx);
+	status = dplane_ctx_get_status(ctx);
+
+	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+		zlog_debug("LSP dplane ctx %p, op %s, in-label %u, result %s",
+			   ctx, dplane_op2str(op),
+			   dplane_ctx_get_in_label(ctx),
+			   dplane_res2str(status));
+
+	switch (op) {
+	case DPLANE_OP_LSP_INSTALL:
+	case DPLANE_OP_LSP_UPDATE:
+		/* Look for zebra LSP object */
+		zvrf = vrf_info_lookup(VRF_DEFAULT);
+		if (zvrf == NULL)
+			break;
+
+		lsp_table = zvrf->lsp_table;
+
+		tmp_ile.in_label = dplane_ctx_get_in_label(ctx);
+		lsp = hash_lookup(lsp_table, &tmp_ile);
+		if (lsp == NULL) {
+			if (IS_ZEBRA_DEBUG_DPLANE)
+				zlog_debug("LSP ctx %p: in-label %u not found",
+					   ctx, dplane_ctx_get_in_label(ctx));
+			break;
+		}
+
+		/* TODO -- Confirm that this result is still 'current' */
+
+		if (dplane_ctx_get_status(ctx) ==
+		    ZEBRA_DPLANE_REQUEST_SUCCESS) {
+			/* Update zebra object */
+			SET_FLAG(lsp->flags, LSP_FLAG_INSTALLED);
+			for (nhlfe = lsp->nhlfe_list; nhlfe;
+			     nhlfe = nhlfe->next) {
+				nexthop = nhlfe->nexthop;
+				if (!nexthop)
+					continue;
+
+				SET_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED);
+				SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
+			}
+		} else {
+			UNSET_FLAG(lsp->flags, LSP_FLAG_INSTALLED);
+			clear_nhlfe_installed(lsp);
+			flog_warn(EC_ZEBRA_LSP_INSTALL_FAILURE,
+				  "LSP Install Failure: in-label %u",
+				  lsp->ile.in_label);
+		}
+
+		break;
+
+	case DPLANE_OP_LSP_DELETE:
+		flog_warn(EC_ZEBRA_LSP_DELETE_FAILURE,
+			  "LSP Deletion Failure: in-label %u",
+			  dplane_ctx_get_in_label(ctx));
+		break;
+
+	default:
+		break;
+
+	} /* Switch */
+
+	dplane_ctx_fini(&ctx);
+}
+
+/*
  * Install dynamic LSP entry.
  */
 int zebra_mpls_lsp_install(struct zebra_vrf *zvrf, struct route_node *rn,
diff --git a/zebra/zebra_mpls.h b/zebra/zebra_mpls.h
index 2c95d22c4e..fb32adde31 100644
--- a/zebra/zebra_mpls.h
+++ b/zebra/zebra_mpls.h
@@ -342,6 +342,14 @@ int zebra_mpls_static_lsp_del(struct zebra_vrf *zvrf, mpls_label_t in_label,
 			      ifindex_t ifindex);
 
 /*
+ * Process LSP update results from zebra dataplane.
+ */
+/* Forward ref of dplane update context type */
+struct zebra_dplane_ctx;
+
+void zebra_mpls_lsp_dplane_result(struct zebra_dplane_ctx *ctx);
+
+/*
  * Schedule all MPLS label forwarding entries for processing.
  * Called upon changes that may affect one or more of them such as
  * interface or nexthop state changes.
diff --git a/zebra/zebra_mpls_netlink.c b/zebra/zebra_mpls_netlink.c
index 2a21f8686b..63f518ffc3 100644
--- a/zebra/zebra_mpls_netlink.c
+++ b/zebra/zebra_mpls_netlink.c
@@ -22,6 +22,7 @@
 
 #ifdef HAVE_NETLINK
 
+#include "zebra/debug.h"
 #include "zebra/rt.h"
 #include "zebra/rt_netlink.h"
 #include "zebra/zebra_mpls.h"
@@ -98,9 +99,38 @@ enum zebra_dplane_result kernel_del_lsp(zebra_lsp_t *lsp)
 	return ZEBRA_DPLANE_REQUEST_SUCCESS;
 }
 
+/*
+ * LSP forwarding update using dataplane context information.
+ */
 enum zebra_dplane_result kernel_lsp_update(struct zebra_dplane_ctx *ctx)
 {
-	return ZEBRA_DPLANE_REQUEST_FAILURE;
+	int cmd, ret = -1;
+
+	/* Call to netlink layer based on type of update */
+	if (dplane_ctx_get_op(ctx) == DPLANE_OP_LSP_DELETE) {
+		cmd = RTM_DELROUTE;
+	} else if (dplane_ctx_get_op(ctx) == DPLANE_OP_LSP_INSTALL ||
+		   dplane_ctx_get_op(ctx) == DPLANE_OP_LSP_UPDATE) {
+
+		/* Validate */
+		if (dplane_ctx_get_best_nhlfe(ctx) == NULL) {
+			if (IS_ZEBRA_DEBUG_KERNEL || IS_ZEBRA_DEBUG_MPLS)
+				zlog_debug("LSP in-label %u: update fails, no best NHLFE",
+					   dplane_ctx_get_in_label(ctx));
+			goto done;
+		}
+
+		cmd = RTM_NEWROUTE;
+	} else
+		/* Invalid op? */
+		goto done;
+
+	ret = netlink_mpls_multipath_ctx(cmd, ctx);
+
+done:
+
+	return (ret == 0 ?
+		ZEBRA_DPLANE_REQUEST_SUCCESS : ZEBRA_DPLANE_REQUEST_FAILURE);
 }
 
 int mpls_kernel_init(void)
diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c
index 7e4ac1ddd2..34bf513bfb 100644
--- a/zebra/zebra_rib.c
+++ b/zebra/zebra_rib.c
@@ -1811,7 +1811,7 @@ done:
 /*
  * Route-update results processing after async dataplane update.
  */
-static void rib_process_after(struct zebra_dplane_ctx *ctx)
+static void rib_process_result(struct zebra_dplane_ctx *ctx)
 {
 	struct route_table *table = NULL;
 	struct zebra_vrf *zvrf = NULL;
@@ -1931,8 +1931,6 @@ static void rib_process_after(struct zebra_dplane_ctx *ctx)
 	}
 
 	switch (op) {
-	case DPLANE_OP_NONE:
-		break;
 	case DPLANE_OP_ROUTE_INSTALL:
 	case DPLANE_OP_ROUTE_UPDATE:
 		if (status == ZEBRA_DPLANE_REQUEST_SUCCESS) {
@@ -2015,6 +2013,8 @@ static void rib_process_after(struct zebra_dplane_ctx *ctx)
 					     dest_str, sizeof(dest_str)));
 		}
 		break;
+	default:
+		break;
 	}
 done:
 
@@ -3191,7 +3191,8 @@ void rib_close_table(struct route_table *table)
 }
 
 /*
- *
+ * Handle results from the dataplane system. Dequeue update context
+ * structs, dispatch to appropriate internal handlers.
  */
 static int rib_process_dplane_results(struct thread *thread)
 {
@@ -3200,13 +3201,15 @@ static int rib_process_dplane_results(struct thread *thread)
 
 	/* Dequeue a list of completed updates with one lock/unlock cycle */
 
+	/* TODO -- dequeue a list with one lock/unlock cycle? */
+
 	do {
 		TAILQ_INIT(&ctxlist);
 
 		/* Take lock controlling queue of results */
 		pthread_mutex_lock(&dplane_mutex);
 		{
-			/* Dequeue context block */
+			/* Dequeue list of context structs */
 			dplane_ctx_list_append(&ctxlist, &rib_dplane_q);
 		}
 		pthread_mutex_unlock(&dplane_mutex);
@@ -3219,7 +3222,24 @@ static int rib_process_dplane_results(struct thread *thread)
 			break;
 
 		while (ctx) {
-			rib_process_after(ctx);
+			switch (dplane_ctx_get_op(ctx)) {
+			case DPLANE_OP_ROUTE_INSTALL:
+			case DPLANE_OP_ROUTE_UPDATE:
+			case DPLANE_OP_ROUTE_DELETE:
+				rib_process_result(ctx);
+				break;
+
+			case DPLANE_OP_LSP_INSTALL:
+			case DPLANE_OP_LSP_UPDATE:
+			case DPLANE_OP_LSP_DELETE:
+				zebra_mpls_lsp_dplane_result(ctx);
+				break;
+
+			default:
+				/* Don't expect this: just return the struct? */
+				dplane_ctx_fini(&ctx);
+				break;
+			} /* Dispatch by op code */
 
 			ctx = dplane_ctx_dequeue(&ctxlist);
 		}
diff --git a/zebra/zebra_vrf.h b/zebra/zebra_vrf.h
index c28025403b..2473299d17 100644
--- a/zebra/zebra_vrf.h
+++ b/zebra/zebra_vrf.h
@@ -137,12 +137,14 @@ struct zebra_vrf {
 	 */
 	enum vxlan_flood_control vxlan_flood_ctrl;
 
-	/* Route Installs */
+	/* Install stats */
 	uint64_t installs;
 	uint64_t removals;
 	uint64_t installs_queued;
 	uint64_t removals_queued;
 	uint64_t neigh_updates;
+	uint64_t lsp_installs_queued;
+	uint64_t lsp_removals_queued;
 	uint64_t lsp_installs;
 	uint64_t lsp_removals;
 };