1 files changed, 327 insertions, 244 deletions

diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c
index e34b6f23ff..6e8d35aa77 100644
--- a/zebra/zebra_dplane.c
+++ b/zebra/zebra_dplane.c
@@ -180,16 +180,19 @@ struct dplane_mac_info {
 	struct ethaddr mac;
 	struct in_addr vtep_ip;
 	bool is_sticky;
+	uint32_t nhg_id;
+	uint32_t update_flags;
 };
 
 /*
- * EVPN neighbor info for the dataplane
+ * Neighbor info for the dataplane
  */
 struct dplane_neigh_info {
 	struct ipaddr ip_addr;
 	struct ethaddr mac;
 	uint32_t flags;
 	uint16_t state;
+	uint32_t update_flags;
 };
 
 /*
@@ -204,6 +207,7 @@ struct dplane_ctx_rule {
 	/* Filter criteria */
 	uint32_t filter_bm;
 	uint32_t fwmark;
+	uint8_t dsfield;
 	struct prefix src_ip;
 	struct prefix dst_ip;
 };
@@ -440,13 +444,14 @@ static enum zebra_dplane_result mac_update_common(
 	enum dplane_op_e op, const struct interface *ifp,
 	const struct interface *br_ifp,
 	vlanid_t vid, const struct ethaddr *mac,
-	struct in_addr vtep_ip,	bool sticky);
+	struct in_addr vtep_ip,	bool sticky, uint32_t nhg_id,
+	uint32_t update_flags);
 static enum zebra_dplane_result neigh_update_internal(
 	enum dplane_op_e op,
 	const struct interface *ifp,
 	const struct ethaddr *mac,
 	const struct ipaddr *ip,
-	uint32_t flags, uint16_t state);
+	uint32_t flags, uint16_t state, uint32_t update_flags);
 
 /*
  * Public APIs
@@ -602,6 +607,7 @@ static void dplane_ctx_free_internal(struct zebra_dplane_ctx *ctx)
 	case DPLANE_OP_RULE_ADD:
 	case DPLANE_OP_RULE_DELETE:
 	case DPLANE_OP_RULE_UPDATE:
+	case DPLANE_OP_NEIGH_DISCOVER:
 	case DPLANE_OP_NONE:
 		break;
 	}
@@ -834,6 +840,10 @@ const char *dplane_op2str(enum dplane_op_e op)
 	case DPLANE_OP_RULE_UPDATE:
 		ret = "RULE_UPDATE";
 		break;
+
+	case DPLANE_OP_NEIGH_DISCOVER:
+		ret = "NEIGH_DISCOVER";
+		break;
 	}
 
 	return ret;
@@ -853,8 +863,6 @@ const char *dplane_res2str(enum zebra_dplane_result res)
 	case ZEBRA_DPLANE_REQUEST_SUCCESS:
 		ret = "SUCCESS";
 		break;
-	case ZEBRA_DPLANE_REQUEST_PENDING:
-		ret = "PENDING";
 	}
 
 	return ret;
@@ -1551,6 +1559,18 @@ bool dplane_ctx_mac_is_sticky(const struct zebra_dplane_ctx *ctx)
 	return ctx->u.macinfo.is_sticky;
 }
 
+uint32_t dplane_ctx_mac_get_nhg_id(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+	return ctx->u.macinfo.nhg_id;
+}
+
+uint32_t dplane_ctx_mac_get_update_flags(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+	return ctx->u.macinfo.update_flags;
+}
+
 const struct ethaddr *dplane_ctx_mac_get_addr(
 	const struct zebra_dplane_ctx *ctx)
 {
@@ -1598,6 +1618,12 @@ uint16_t dplane_ctx_neigh_get_state(const struct zebra_dplane_ctx *ctx)
 	return ctx->u.neigh.state;
 }
 
+uint32_t dplane_ctx_neigh_get_update_flags(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+	return ctx->u.neigh.update_flags;
+}
+
 /* Accessors for PBR rule information */
 int dplane_ctx_rule_get_sock(const struct zebra_dplane_ctx *ctx)
 {
@@ -1676,6 +1702,20 @@ uint32_t dplane_ctx_rule_get_old_fwmark(const struct zebra_dplane_ctx *ctx)
 	return ctx->u.rule.old.fwmark;
 }
 
+uint8_t dplane_ctx_rule_get_dsfield(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.new.dsfield;
+}
+
+uint8_t dplane_ctx_rule_get_old_dsfield(const struct zebra_dplane_ctx *ctx)
+{
+	DPLANE_CTX_VALID(ctx);
+
+	return ctx->u.rule.old.dsfield;
+}
+
 const struct prefix *
 dplane_ctx_rule_get_src_ip(const struct zebra_dplane_ctx *ctx)
 {
@@ -1942,6 +1982,7 @@ int dplane_ctx_nexthop_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 	 * it probably won't require two messages
 	 */
 	dplane_ctx_ns_init(ctx, zns, (op == DPLANE_OP_NH_UPDATE));
+	ctx->zd_is_update = (op == DPLANE_OP_NH_UPDATE);
 
 	ret = AOK;
 
@@ -1964,6 +2005,7 @@ int dplane_ctx_lsp_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 	/* Capture namespace info */
 	dplane_ctx_ns_init(ctx, zebra_ns_lookup(NS_DEFAULT),
 			   (op == DPLANE_OP_LSP_UPDATE));
+	ctx->zd_is_update = (op == DPLANE_OP_LSP_UPDATE);
 
 	memset(&ctx->u.lsp, 0, sizeof(ctx->u.lsp));
 
@@ -2001,10 +2043,19 @@ int dplane_ctx_lsp_init(struct zebra_dplane_ctx *ctx, enum dplane_op_e op,
 			break;
 		}
 
-		/* Need to copy flags too */
+		/* Need to copy flags and backup info too */
 		new_nhlfe->flags = nhlfe->flags;
 		new_nhlfe->nexthop->flags = nhlfe->nexthop->flags;
 
+		if (CHECK_FLAG(new_nhlfe->nexthop->flags,
+			       NEXTHOP_FLAG_HAS_BACKUP)) {
+			new_nhlfe->nexthop->backup_num =
+				nhlfe->nexthop->backup_num;
+			memcpy(new_nhlfe->nexthop->backup_idx,
+			       nhlfe->nexthop->backup_idx,
+			       new_nhlfe->nexthop->backup_num);
+		}
+
 		if (nhlfe == lsp->best_nhlfe)
 			ctx->u.lsp.best_nhlfe = new_nhlfe;
 	}
@@ -2104,8 +2155,15 @@ static int dplane_ctx_pw_init(struct zebra_dplane_ctx *ctx,
 
 			if (re) {
 				nhg = rib_get_fib_nhg(re);
-				copy_nexthops(&(ctx->u.pw.nhg.nexthop),
-					      nhg->nexthop, NULL);
+				if (nhg && nhg->nexthop)
+					copy_nexthops(&(ctx->u.pw.nhg.nexthop),
+						      nhg->nexthop, NULL);
+
+				/* Include any installed backup nexthops */
+				nhg = rib_get_fib_backup_nhg(re);
+				if (nhg && nhg->nexthop)
+					copy_nexthops(&(ctx->u.pw.nhg.nexthop),
+						      nhg->nexthop, NULL);
 			}
 			route_unlock_node(rn);
 		}
@@ -2129,6 +2187,7 @@ static void dplane_ctx_rule_init_single(struct dplane_ctx_rule *dplane_rule,
 
 	dplane_rule->filter_bm = rule->rule.filter.filter_bm;
 	dplane_rule->fwmark = rule->rule.filter.fwmark;
+	dplane_rule->dsfield = rule->rule.filter.dsfield;
 	prefix_copy(&(dplane_rule->dst_ip), &rule->rule.filter.dst_ip);
 	prefix_copy(&(dplane_rule->src_ip), &rule->rule.filter.src_ip);
 }
@@ -2168,6 +2227,7 @@ static int dplane_ctx_rule_init(struct zebra_dplane_ctx *ctx,
 
 	dplane_ctx_ns_init(ctx, zebra_ns_lookup(NS_DEFAULT),
 			   op == DPLANE_OP_RULE_UPDATE);
+	ctx->zd_is_update = (op == DPLANE_OP_RULE_UPDATE);
 
 	ctx->zd_vrf_id = new_rule->vrf_id;
 	memcpy(ctx->zd_ifname, new_rule->ifname, sizeof(new_rule->ifname));
@@ -2458,9 +2518,11 @@ dplane_route_notif_update(struct route_node *rn,
 			  enum dplane_op_e op,
 			  struct zebra_dplane_ctx *ctx)
 {
-	enum zebra_dplane_result ret = ZEBRA_DPLANE_REQUEST_FAILURE;
+	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
+	int ret = EINVAL;
 	struct zebra_dplane_ctx *new_ctx = NULL;
 	struct nexthop *nexthop;
+	struct nexthop_group *nhg;
 
 	if (rn == NULL || re == NULL)
 		goto done;
@@ -2482,8 +2544,17 @@ dplane_route_notif_update(struct route_node *rn,
 		nexthops_free(new_ctx->u.rinfo.zd_ng.nexthop);
 		new_ctx->u.rinfo.zd_ng.nexthop = NULL;
 
-		copy_nexthops(&(new_ctx->u.rinfo.zd_ng.nexthop),
-			      (rib_get_fib_nhg(re))->nexthop, NULL);
+		nhg = rib_get_fib_nhg(re);
+		if (nhg && nhg->nexthop)
+			copy_nexthops(&(new_ctx->u.rinfo.zd_ng.nexthop),
+				      nhg->nexthop, NULL);
+
+		/* Check for installed backup nexthops also */
+		nhg = rib_get_fib_backup_nhg(re);
+		if (nhg && nhg->nexthop) {
+			copy_nexthops(&(new_ctx->u.rinfo.zd_ng.nexthop),
+				      nhg->nexthop, NULL);
+		}
 
 		for (ALL_NEXTHOPS(new_ctx->u.rinfo.zd_ng, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
@@ -2494,12 +2565,15 @@ dplane_route_notif_update(struct route_node *rn,
 	dplane_ctx_set_notif_provider(new_ctx,
 				      dplane_ctx_get_notif_provider(ctx));
 
-	dplane_update_enqueue(new_ctx);
-
-	ret = ZEBRA_DPLANE_REQUEST_QUEUED;
+	ret = dplane_update_enqueue(new_ctx);
 
 done:
-	return ret;
+	if (ret == AOK)
+		result = ZEBRA_DPLANE_REQUEST_QUEUED;
+	else if (new_ctx)
+		dplane_ctx_free(&new_ctx);
+
+	return result;
 }
 
 /*
@@ -2583,6 +2657,8 @@ dplane_lsp_notif_update(zebra_lsp_t *lsp,
 	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
 	int ret = EINVAL;
 	struct zebra_dplane_ctx *ctx = NULL;
+	struct nhlfe_list_head *head;
+	zebra_nhlfe_t *nhlfe, *new_nhlfe;
 
 	/* Obtain context block */
 	ctx = dplane_ctx_alloc();
@@ -2591,10 +2667,27 @@ dplane_lsp_notif_update(zebra_lsp_t *lsp,
 		goto done;
 	}
 
+	/* Copy info from zebra LSP */
 	ret = dplane_ctx_lsp_init(ctx, op, lsp);
 	if (ret != AOK)
 		goto done;
 
+	/* Add any installed backup nhlfes */
+	head = &(ctx->u.lsp.backup_nhlfe_list);
+	frr_each(nhlfe_list, head, nhlfe) {
+
+		if (CHECK_FLAG(nhlfe->flags, NHLFE_FLAG_INSTALLED) &&
+		    CHECK_FLAG(nhlfe->nexthop->flags, NEXTHOP_FLAG_FIB)) {
+			new_nhlfe = zebra_mpls_lsp_add_nh(&(ctx->u.lsp),
+							  nhlfe->type,
+							  nhlfe->nexthop);
+
+			/* Need to copy flags too */
+			new_nhlfe->flags = nhlfe->flags;
+			new_nhlfe->nexthop->flags = nhlfe->nexthop->flags;
+		}
+	}
+
 	/* Capture info about the source of the notification */
 	dplane_ctx_set_notif_provider(
 		ctx,
@@ -2830,35 +2923,75 @@ static enum zebra_dplane_result intf_addr_update_internal(
 /*
  * Enqueue vxlan/evpn mac add (or update).
  */
-enum zebra_dplane_result dplane_mac_add(const struct interface *ifp,
+enum zebra_dplane_result dplane_rem_mac_add(const struct interface *ifp,
 					const struct interface *bridge_ifp,
 					vlanid_t vid,
 					const struct ethaddr *mac,
 					struct in_addr vtep_ip,
-					bool sticky)
+					bool sticky,
+					uint32_t nhg_id,
+					bool was_static)
 {
 	enum zebra_dplane_result result;
+	uint32_t update_flags = 0;
+
+	update_flags |= DPLANE_MAC_REMOTE;
+	if (was_static)
+		update_flags |= DPLANE_MAC_WAS_STATIC;
 
 	/* Use common helper api */
 	result = mac_update_common(DPLANE_OP_MAC_INSTALL, ifp, bridge_ifp,
-				   vid, mac, vtep_ip, sticky);
+				   vid, mac, vtep_ip, sticky, nhg_id, update_flags);
 	return result;
 }
 
 /*
  * Enqueue vxlan/evpn mac delete.
  */
-enum zebra_dplane_result dplane_mac_del(const struct interface *ifp,
+enum zebra_dplane_result dplane_rem_mac_del(const struct interface *ifp,
 					const struct interface *bridge_ifp,
 					vlanid_t vid,
 					const struct ethaddr *mac,
 					struct in_addr vtep_ip)
 {
 	enum zebra_dplane_result result;
+	uint32_t update_flags = 0;
+
+	update_flags |= DPLANE_MAC_REMOTE;
 
 	/* Use common helper api */
 	result = mac_update_common(DPLANE_OP_MAC_DELETE, ifp, bridge_ifp,
-				   vid, mac, vtep_ip, false);
+				   vid, mac, vtep_ip, false, 0, update_flags);
+	return result;
+}
+
+/*
+ * Enqueue local mac add (or update).
+ */
+enum zebra_dplane_result dplane_local_mac_add(const struct interface *ifp,
+					const struct interface *bridge_ifp,
+					vlanid_t vid,
+					const struct ethaddr *mac,
+					bool sticky,
+					uint32_t set_static,
+					uint32_t set_inactive)
+{
+	enum zebra_dplane_result result;
+	uint32_t update_flags = 0;
+	struct in_addr vtep_ip;
+
+	if (set_static)
+		update_flags |= DPLANE_MAC_SET_STATIC;
+
+	if (set_inactive)
+		update_flags |= DPLANE_MAC_SET_INACTIVE;
+
+	vtep_ip.s_addr = 0;
+
+	/* Use common helper api */
+	result = mac_update_common(DPLANE_OP_MAC_INSTALL, ifp, bridge_ifp,
+				     vid, mac, vtep_ip, sticky, 0,
+				     update_flags);
 	return result;
 }
 
@@ -2872,7 +3005,9 @@ void dplane_mac_init(struct zebra_dplane_ctx *ctx,
 		     vlanid_t vid,
 		     const struct ethaddr *mac,
 		     struct in_addr vtep_ip,
-		     bool sticky)
+		     bool sticky,
+		     uint32_t nhg_id,
+		     uint32_t update_flags)
 {
 	struct zebra_ns *zns;
 
@@ -2893,6 +3028,8 @@ void dplane_mac_init(struct zebra_dplane_ctx *ctx,
 	ctx->u.macinfo.mac = *mac;
 	ctx->u.macinfo.vid = vid;
 	ctx->u.macinfo.is_sticky = sticky;
+	ctx->u.macinfo.nhg_id = nhg_id;
+	ctx->u.macinfo.update_flags = update_flags;
 }
 
 /*
@@ -2905,7 +3042,9 @@ mac_update_common(enum dplane_op_e op,
 		  vlanid_t vid,
 		  const struct ethaddr *mac,
 		  struct in_addr vtep_ip,
-		  bool sticky)
+		  bool sticky,
+		  uint32_t nhg_id,
+		  uint32_t update_flags)
 {
 	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
 	int ret;
@@ -2925,7 +3064,8 @@ mac_update_common(enum dplane_op_e op,
 	ctx->zd_op = op;
 
 	/* Common init for the ctx */
-	dplane_mac_init(ctx, ifp, br_ifp, vid, mac, vtep_ip, sticky);
+	dplane_mac_init(ctx, ifp, br_ifp, vid, mac, vtep_ip, sticky,
+			nhg_id, update_flags);
 
 	/* Enqueue for processing on the dplane pthread */
 	ret = dplane_update_enqueue(ctx);
@@ -2949,30 +3089,56 @@ mac_update_common(enum dplane_op_e op,
 /*
  * Enqueue evpn neighbor add for the dataplane.
  */
-enum zebra_dplane_result dplane_neigh_add(const struct interface *ifp,
+enum zebra_dplane_result dplane_rem_neigh_add(const struct interface *ifp,
 					  const struct ipaddr *ip,
 					  const struct ethaddr *mac,
-					  uint32_t flags)
+					  uint32_t flags, bool was_static)
 {
 	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
+	uint32_t update_flags = 0;
+
+	update_flags |= DPLANE_NEIGH_REMOTE;
+
+	if (was_static)
+		update_flags |= DPLANE_NEIGH_WAS_STATIC;
 
 	result = neigh_update_internal(DPLANE_OP_NEIGH_INSTALL,
-				       ifp, mac, ip, flags, DPLANE_NUD_NOARP);
+				       ifp, mac, ip, flags, DPLANE_NUD_NOARP,
+				       update_flags);
 
 	return result;
 }
 
 /*
- * Enqueue evpn neighbor update for the dataplane.
+ * Enqueue local neighbor add for the dataplane.
  */
-enum zebra_dplane_result dplane_neigh_update(const struct interface *ifp,
-					     const struct ipaddr *ip,
-					     const struct ethaddr *mac)
+enum zebra_dplane_result dplane_local_neigh_add(const struct interface *ifp,
+					  const struct ipaddr *ip,
+					  const struct ethaddr *mac,
+					  bool set_router, bool set_static,
+					  bool set_inactive)
 {
-	enum zebra_dplane_result result;
+	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
+	uint32_t update_flags = 0;
+	uint32_t ntf = 0;
+	uint16_t state;
+
+	if (set_static)
+		update_flags |= DPLANE_NEIGH_SET_STATIC;
+
+	if (set_inactive) {
+		update_flags |= DPLANE_NEIGH_SET_INACTIVE;
+		state = DPLANE_NUD_STALE;
+	} else {
+		state = DPLANE_NUD_REACHABLE;
+	}
 
-	result = neigh_update_internal(DPLANE_OP_NEIGH_UPDATE,
-				       ifp, mac, ip, 0, DPLANE_NUD_PROBE);
+	if (set_router)
+		ntf |= DPLANE_NTF_ROUTER;
+
+	result = neigh_update_internal(DPLANE_OP_NEIGH_INSTALL,
+				       ifp, mac, ip, ntf,
+				       state, update_flags);
 
 	return result;
 }
@@ -2980,13 +3146,16 @@ enum zebra_dplane_result dplane_neigh_update(const struct interface *ifp,
 /*
  * Enqueue evpn neighbor delete for the dataplane.
  */
-enum zebra_dplane_result dplane_neigh_delete(const struct interface *ifp,
+enum zebra_dplane_result dplane_rem_neigh_delete(const struct interface *ifp,
 					     const struct ipaddr *ip)
 {
 	enum zebra_dplane_result result;
+	uint32_t update_flags = 0;
+
+	update_flags |= DPLANE_NEIGH_REMOTE;
 
 	result = neigh_update_internal(DPLANE_OP_NEIGH_DELETE,
-				       ifp, NULL, ip, 0, 0);
+				       ifp, NULL, ip, 0, 0, update_flags);
 
 	return result;
 }
@@ -3010,7 +3179,7 @@ enum zebra_dplane_result dplane_vtep_add(const struct interface *ifp,
 	addr.ipaddr_v4 = *ip;
 
 	result = neigh_update_internal(DPLANE_OP_VTEP_ADD,
-				       ifp, &mac, &addr, 0, 0);
+				       ifp, &mac, &addr, 0, 0, 0);
 
 	return result;
 }
@@ -3035,20 +3204,32 @@ enum zebra_dplane_result dplane_vtep_delete(const struct interface *ifp,
 	addr.ipaddr_v4 = *ip;
 
 	result = neigh_update_internal(DPLANE_OP_VTEP_DELETE,
-				       ifp, &mac, &addr, 0, 0);
+				       ifp, &mac, &addr, 0, 0, 0);
+
+	return result;
+}
+
+enum zebra_dplane_result dplane_neigh_discover(const struct interface *ifp,
+					       const struct ipaddr *ip)
+{
+	enum zebra_dplane_result result;
+
+	result = neigh_update_internal(DPLANE_OP_NEIGH_DISCOVER, ifp, NULL, ip,
+				       DPLANE_NTF_USE, DPLANE_NUD_INCOMPLETE, 0);
 
 	return result;
 }
 
 /*
- * Common helper api for evpn neighbor updates
+ * Common helper api for neighbor updates
  */
 static enum zebra_dplane_result
 neigh_update_internal(enum dplane_op_e op,
 		      const struct interface *ifp,
 		      const struct ethaddr *mac,
 		      const struct ipaddr *ip,
-		      uint32_t flags, uint16_t state)
+		      uint32_t flags, uint16_t state,
+			  uint32_t update_flags)
 {
 	enum zebra_dplane_result result = ZEBRA_DPLANE_REQUEST_FAILURE;
 	int ret;
@@ -3059,9 +3240,8 @@ neigh_update_internal(enum dplane_op_e op,
 		char buf1[ETHER_ADDR_STRLEN], buf2[PREFIX_STRLEN];
 
 		zlog_debug("init neigh ctx %s: ifp %s, mac %s, ip %s",
-			   dplane_op2str(op),
+			   dplane_op2str(op), ifp->name,
 			   prefix_mac2str(mac, buf1, sizeof(buf1)),
-			   ifp->name,
 			   ipaddr2str(ip, buf2, sizeof(buf2)));
 	}
 
@@ -3085,6 +3265,7 @@ neigh_update_internal(enum dplane_op_e op,
 		ctx->u.neigh.mac = *mac;
 	ctx->u.neigh.flags = flags;
 	ctx->u.neigh.state = state;
+	ctx->u.neigh.update_flags = update_flags;
 
 	/* Enqueue for processing on the dplane pthread */
 	ret = dplane_update_enqueue(ctx);
@@ -3261,8 +3442,9 @@ int dplane_show_provs_helper(struct vty *vty, bool detailed)
 		out_max = atomic_load_explicit(&prov->dp_out_max,
 					       memory_order_relaxed);
 
-		vty_out(vty, "%s (%u): in: %"PRIu64", q_max: %"PRIu64", "
-			"out: %"PRIu64", q_max: %"PRIu64"\n",
+		vty_out(vty,
+			"%s (%u): in: %" PRIu64 ", q_max: %" PRIu64
+			", out: %" PRIu64 ", q_max: %" PRIu64 "\n",
 			prov->dp_name, prov->dp_id, in, in_max, out, out_max);
 
 		DPLANE_LOCK();
@@ -3534,149 +3716,100 @@ void dplane_provider_enqueue_to_zebra(struct zebra_dplane_ctx *ctx)
  * Kernel dataplane provider
  */
 
-/*
- * Handler for kernel LSP updates
- */
-static enum zebra_dplane_result
-kernel_dplane_lsp_update(struct zebra_dplane_ctx *ctx)
+static void kernel_dplane_log_detail(struct zebra_dplane_ctx *ctx)
 {
-	return kernel_lsp_update(ctx);
-}
+	char buf[PREFIX_STRLEN];
 
-/*
- * Handler for kernel pseudowire updates
- */
-static enum zebra_dplane_result
-kernel_dplane_pw_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
-		zlog_debug("Dplane pw %s: op %s af %d loc: %u rem: %u",
-			   dplane_ctx_get_ifname(ctx),
-			   dplane_op2str(ctx->zd_op),
-			   dplane_ctx_get_pw_af(ctx),
-			   dplane_ctx_get_pw_local_label(ctx),
-			   dplane_ctx_get_pw_remote_label(ctx));
-
-	return kernel_pw_update(ctx);
-}
-
-/*
- * Handler for kernel route updates
- */
-static enum zebra_dplane_result
-kernel_dplane_route_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) {
-		char dest_str[PREFIX_STRLEN];
+	switch (dplane_ctx_get_op(ctx)) {
 
-		prefix2str(dplane_ctx_get_dest(ctx),
-			   dest_str, sizeof(dest_str));
+	case DPLANE_OP_ROUTE_INSTALL:
+	case DPLANE_OP_ROUTE_UPDATE:
+	case DPLANE_OP_ROUTE_DELETE:
+		prefix2str(dplane_ctx_get_dest(ctx), buf, sizeof(buf));
 
 		zlog_debug("%u:%s Dplane route update ctx %p op %s",
-			   dplane_ctx_get_vrf(ctx), dest_str,
-			   ctx, dplane_op2str(dplane_ctx_get_op(ctx)));
-	}
-
-	return kernel_route_update(ctx);
-}
-
-/*
- * Handler for kernel-facing interface address updates
- */
-static enum zebra_dplane_result
-kernel_dplane_address_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) {
-		char dest_str[PREFIX_STRLEN];
-
-		prefix2str(dplane_ctx_get_intf_addr(ctx), dest_str,
-			   sizeof(dest_str));
-
-		zlog_debug("Dplane intf %s, idx %u, addr %s",
-			   dplane_op2str(dplane_ctx_get_op(ctx)),
-			   dplane_ctx_get_ifindex(ctx), dest_str);
-	}
-
-	return kernel_address_update_ctx(ctx);
-}
+			   dplane_ctx_get_vrf(ctx), buf, ctx,
+			   dplane_op2str(dplane_ctx_get_op(ctx)));
+		break;
 
-/**
- * kernel_dplane_nexthop_update() - Handler for kernel nexthop updates
- *
- * @ctx:	Dataplane context
- *
- * Return:	Dataplane result flag
- */
-static enum zebra_dplane_result
-kernel_dplane_nexthop_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) {
+	case DPLANE_OP_NH_INSTALL:
+	case DPLANE_OP_NH_UPDATE:
+	case DPLANE_OP_NH_DELETE:
 		zlog_debug("ID (%u) Dplane nexthop update ctx %p op %s",
 			   dplane_ctx_get_nhe_id(ctx), ctx,
 			   dplane_op2str(dplane_ctx_get_op(ctx)));
-	}
+		break;
 
-	return kernel_nexthop_update(ctx);
-}
+	case DPLANE_OP_LSP_INSTALL:
+	case DPLANE_OP_LSP_UPDATE:
+	case DPLANE_OP_LSP_DELETE:
+		break;
 
-/*
- * Handler for kernel-facing EVPN MAC address updates
- */
-static enum zebra_dplane_result
-kernel_dplane_mac_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) {
-		char buf[ETHER_ADDR_STRLEN];
+	case DPLANE_OP_PW_INSTALL:
+	case DPLANE_OP_PW_UNINSTALL:
+		zlog_debug("Dplane pw %s: op %s af %d loc: %u rem: %u",
+			   dplane_ctx_get_ifname(ctx),
+			   dplane_op2str(ctx->zd_op), dplane_ctx_get_pw_af(ctx),
+			   dplane_ctx_get_pw_local_label(ctx),
+			   dplane_ctx_get_pw_remote_label(ctx));
+		break;
 
+	case DPLANE_OP_ADDR_INSTALL:
+	case DPLANE_OP_ADDR_UNINSTALL:
+		prefix2str(dplane_ctx_get_intf_addr(ctx), buf, sizeof(buf));
+
+		zlog_debug("Dplane intf %s, idx %u, addr %s",
+			   dplane_op2str(dplane_ctx_get_op(ctx)),
+			   dplane_ctx_get_ifindex(ctx), buf);
+		break;
+
+	case DPLANE_OP_MAC_INSTALL:
+	case DPLANE_OP_MAC_DELETE:
 		prefix_mac2str(dplane_ctx_mac_get_addr(ctx), buf,
 			       sizeof(buf));
 
 		zlog_debug("Dplane %s, mac %s, ifindex %u",
 			   dplane_op2str(dplane_ctx_get_op(ctx)),
 			   buf, dplane_ctx_get_ifindex(ctx));
-	}
-
-	return kernel_mac_update_ctx(ctx);
-}
-
-/*
- * Handler for kernel-facing EVPN neighbor updates
- */
-static enum zebra_dplane_result
-kernel_dplane_neigh_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL) {
-		char buf[PREFIX_STRLEN];
+		break;
 
+	case DPLANE_OP_NEIGH_INSTALL:
+	case DPLANE_OP_NEIGH_UPDATE:
+	case DPLANE_OP_NEIGH_DELETE:
+	case DPLANE_OP_VTEP_ADD:
+	case DPLANE_OP_VTEP_DELETE:
+	case DPLANE_OP_NEIGH_DISCOVER:
 		ipaddr2str(dplane_ctx_neigh_get_ipaddr(ctx), buf,
 			   sizeof(buf));
 
 		zlog_debug("Dplane %s, ip %s, ifindex %u",
 			   dplane_op2str(dplane_ctx_get_op(ctx)),
 			   buf, dplane_ctx_get_ifindex(ctx));
-	}
-
-	return kernel_neigh_update_ctx(ctx);
-}
+		break;
 
-/*
- *  Handler for kernel PBR rule updates
- */
-static enum zebra_dplane_result
-kernel_dplane_rule_update(struct zebra_dplane_ctx *ctx)
-{
-	if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+	case DPLANE_OP_RULE_ADD:
+	case DPLANE_OP_RULE_DELETE:
+	case DPLANE_OP_RULE_UPDATE:
 		zlog_debug("Dplane rule update op %s, if %s(%u), ctx %p",
 			   dplane_op2str(dplane_ctx_get_op(ctx)),
 			   dplane_ctx_get_ifname(ctx),
 			   dplane_ctx_get_ifindex(ctx), ctx);
+		break;
+
+	case DPLANE_OP_SYS_ROUTE_ADD:
+	case DPLANE_OP_SYS_ROUTE_DELETE:
+	case DPLANE_OP_ROUTE_NOTIFY:
+	case DPLANE_OP_LSP_NOTIFY:
 
-	return kernel_pbr_rule_update(ctx);
+	case DPLANE_OP_NONE:
+		break;
+	}
 }
 
-static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx,
-					enum zebra_dplane_result res)
+static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx)
 {
+	enum zebra_dplane_result res = dplane_ctx_get_status(ctx);
+
 	switch (dplane_ctx_get_op(ctx)) {
 
 	case DPLANE_OP_ROUTE_INSTALL:
@@ -3685,6 +3818,27 @@ static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx,
 		if (res != ZEBRA_DPLANE_REQUEST_SUCCESS)
 			atomic_fetch_add_explicit(&zdplane_info.dg_route_errors,
 						  1, memory_order_relaxed);
+
+		if ((dplane_ctx_get_op(ctx) != DPLANE_OP_ROUTE_DELETE)
+		    && (res == ZEBRA_DPLANE_REQUEST_SUCCESS)) {
+			struct nexthop *nexthop;
+
+			/* Update installed nexthops to signal which have been
+			 * installed.
+			 */
+			for (ALL_NEXTHOPS_PTR(dplane_ctx_get_ng(ctx),
+					      nexthop)) {
+				if (CHECK_FLAG(nexthop->flags,
+					       NEXTHOP_FLAG_RECURSIVE))
+					continue;
+
+				if (CHECK_FLAG(nexthop->flags,
+					       NEXTHOP_FLAG_ACTIVE)) {
+					SET_FLAG(nexthop->flags,
+						 NEXTHOP_FLAG_FIB);
+				}
+			}
+		}
 		break;
 
 	case DPLANE_OP_NH_INSTALL:
@@ -3731,6 +3885,7 @@ static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx,
 	case DPLANE_OP_NEIGH_DELETE:
 	case DPLANE_OP_VTEP_ADD:
 	case DPLANE_OP_VTEP_DELETE:
+	case DPLANE_OP_NEIGH_DISCOVER:
 		if (res != ZEBRA_DPLANE_REQUEST_SUCCESS)
 			atomic_fetch_add_explicit(&zdplane_info.dg_neigh_errors,
 						  1, memory_order_relaxed);
@@ -3749,11 +3904,14 @@ static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx,
 	case DPLANE_OP_SYS_ROUTE_DELETE:
 	case DPLANE_OP_ROUTE_NOTIFY:
 	case DPLANE_OP_LSP_NOTIFY:
+		break;
+
 	case DPLANE_OP_NONE:
+		if (res != ZEBRA_DPLANE_REQUEST_SUCCESS)
+			atomic_fetch_add_explicit(&zdplane_info.dg_other_errors,
+						  1, memory_order_relaxed);
 		break;
 	}
-
-	dplane_ctx_set_status(ctx, res);
 }
 
 /*
@@ -3761,7 +3919,6 @@ static void kernel_dplane_handle_result(struct zebra_dplane_ctx *ctx,
  */
 static int kernel_dplane_process_func(struct zebra_dplane_provider *prov)
 {
-	enum zebra_dplane_result res;
 	struct zebra_dplane_ctx *ctx, *tctx;
 	struct dplane_ctx_q work_list;
 	int counter, limit;
@@ -3775,97 +3932,21 @@ static int kernel_dplane_process_func(struct zebra_dplane_provider *prov)
 			   dplane_provider_get_name(prov));
 
 	for (counter = 0; counter < limit; counter++) {
-
 		ctx = dplane_provider_dequeue_in_ctx(prov);
 		if (ctx == NULL)
 			break;
 
-		/* A previous provider plugin may have asked to skip the
-		 * kernel update.
-		 */
-		if (dplane_ctx_is_skip_kernel(ctx)) {
-			res = ZEBRA_DPLANE_REQUEST_SUCCESS;
-			goto skip_one;
-		}
-
-		/* Dispatch to appropriate kernel-facing apis */
-		switch (dplane_ctx_get_op(ctx)) {
-
-		case DPLANE_OP_ROUTE_INSTALL:
-		case DPLANE_OP_ROUTE_UPDATE:
-		case DPLANE_OP_ROUTE_DELETE:
-			res = kernel_dplane_route_update(ctx);
-			break;
-
-		case DPLANE_OP_NH_INSTALL:
-		case DPLANE_OP_NH_UPDATE:
-		case DPLANE_OP_NH_DELETE:
-			res = kernel_dplane_nexthop_update(ctx);
-			break;
-
-		case DPLANE_OP_LSP_INSTALL:
-		case DPLANE_OP_LSP_UPDATE:
-		case DPLANE_OP_LSP_DELETE:
-			res = kernel_dplane_lsp_update(ctx);
-			break;
-
-		case DPLANE_OP_PW_INSTALL:
-		case DPLANE_OP_PW_UNINSTALL:
-			res = kernel_dplane_pw_update(ctx);
-			break;
-
-		case DPLANE_OP_ADDR_INSTALL:
-		case DPLANE_OP_ADDR_UNINSTALL:
-			res = kernel_dplane_address_update(ctx);
-			break;
-
-		case DPLANE_OP_MAC_INSTALL:
-		case DPLANE_OP_MAC_DELETE:
-			res = kernel_dplane_mac_update(ctx);
-			break;
-
-		case DPLANE_OP_NEIGH_INSTALL:
-		case DPLANE_OP_NEIGH_UPDATE:
-		case DPLANE_OP_NEIGH_DELETE:
-		case DPLANE_OP_VTEP_ADD:
-		case DPLANE_OP_VTEP_DELETE:
-			res = kernel_dplane_neigh_update(ctx);
-			break;
-
-		case DPLANE_OP_RULE_ADD:
-		case DPLANE_OP_RULE_DELETE:
-		case DPLANE_OP_RULE_UPDATE:
-			res = kernel_dplane_rule_update(ctx);
-			break;
-
-		/* Ignore 'notifications' - no-op */
-		case DPLANE_OP_SYS_ROUTE_ADD:
-		case DPLANE_OP_SYS_ROUTE_DELETE:
-		case DPLANE_OP_ROUTE_NOTIFY:
-		case DPLANE_OP_LSP_NOTIFY:
-			res = ZEBRA_DPLANE_REQUEST_SUCCESS;
-			break;
-
-		default:
-			atomic_fetch_add_explicit(
-				&zdplane_info.dg_other_errors, 1,
-				memory_order_relaxed);
-
-			res = ZEBRA_DPLANE_REQUEST_FAILURE;
-			break;
-		}
-
-	skip_one:
-		/* If the request isn't pending, we can handle the result right
-		 * away.
-		 */
-		if (res != ZEBRA_DPLANE_REQUEST_PENDING)
-			kernel_dplane_handle_result(ctx, res);
+		if (IS_ZEBRA_DEBUG_DPLANE_DETAIL)
+			kernel_dplane_log_detail(ctx);
 
 		TAILQ_INSERT_TAIL(&work_list, ctx, zd_q_entries);
 	}
 
+	kernel_update_multi(&work_list);
+
 	TAILQ_FOREACH_SAFE (ctx, &work_list, zd_q_entries, tctx) {
+		kernel_dplane_handle_result(ctx);
+
 		TAILQ_REMOVE(&work_list, ctx, zd_q_entries);
 		dplane_provider_enqueue_out_ctx(prov, ctx);
 	}
@@ -3910,7 +3991,6 @@ static int test_dplane_process_func(struct zebra_dplane_provider *prov)
 	limit = dplane_provider_get_work_limit(prov);
 
 	for (counter = 0; counter < limit; counter++) {
-
 		ctx = dplane_provider_dequeue_in_ctx(prov);
 		if (ctx == NULL)
 			break;
@@ -4003,19 +4083,19 @@ bool dplane_is_in_shutdown(void)
  */
 void zebra_dplane_pre_finish(void)
 {
-	struct zebra_dplane_provider *dp;
+	struct zebra_dplane_provider *prov;
 
 	if (IS_ZEBRA_DEBUG_DPLANE)
-		zlog_debug("Zebra dataplane pre-fini called");
+		zlog_debug("Zebra dataplane pre-finish called");
 
 	zdplane_info.dg_is_shutdown = true;
 
 	/* Notify provider(s) of pending shutdown. */
-	TAILQ_FOREACH(dp, &zdplane_info.dg_providers_q, dp_prov_link) {
-		if (dp->dp_fini == NULL)
+	TAILQ_FOREACH(prov, &zdplane_info.dg_providers_q, dp_prov_link) {
+		if (prov->dp_fini == NULL)
 			continue;
 
-		dp->dp_fini(dp, true);
+		prov->dp_fini(prov, true /* early */);
 	}
 }
 
@@ -4337,7 +4417,10 @@ void zebra_dplane_shutdown(void)
 	zdplane_info.dg_pthread = NULL;
 	zdplane_info.dg_master = NULL;
 
-	/* Notify provider(s) of final shutdown. */
+	/* Notify provider(s) of final shutdown.
+	 * Note that this call is in the main pthread, so providers must
+	 * be prepared for that.
+	 */
 	TAILQ_FOREACH(dp, &zdplane_info.dg_providers_q, dp_prov_link) {
 		if (dp->dp_fini == NULL)
 			continue;