1 files changed, 76 insertions, 40 deletions

diff --git a/zebra/zebra_nhg.c b/zebra/zebra_nhg.c
index de79c59caa..ebefa020c8 100644
--- a/zebra/zebra_nhg.c
+++ b/zebra/zebra_nhg.c
@@ -56,12 +56,12 @@ static bool g_nexthops_enabled = true;
 static bool proto_nexthops_only;
 
 static struct nhg_hash_entry *depends_find(const struct nexthop *nh, afi_t afi,
-					   int type);
+					   int type, bool from_dplane);
 static void depends_add(struct nhg_connected_tree_head *head,
 			struct nhg_hash_entry *depend);
 static struct nhg_hash_entry *
 depends_find_add(struct nhg_connected_tree_head *head, struct nexthop *nh,
-		 afi_t afi, int type);
+		 afi_t afi, int type, bool from_dplane);
 static struct nhg_hash_entry *
 depends_find_id_add(struct nhg_connected_tree_head *head, uint32_t id);
 static void depends_decrement_free(struct nhg_connected_tree_head *head);
@@ -442,11 +442,8 @@ static void *zebra_nhg_hash_alloc(void *arg)
 	/* Mark duplicate nexthops in a group at creation time. */
 	nexthop_group_mark_duplicates(&(nhe->nhg));
 
-	zebra_nhg_connect_depends(nhe, &(copy->nhg_depends));
-
 	/* Add the ifp now if it's not a group or recursive and has ifindex */
-	if (zebra_nhg_depends_is_empty(nhe) && nhe->nhg.nexthop
-	    && nhe->nhg.nexthop->ifindex) {
+	if (nhe->nhg.nexthop && nhe->nhg.nexthop->ifindex) {
 		struct interface *ifp = NULL;
 
 		ifp = if_lookup_by_index(nhe->nhg.nexthop->ifindex,
@@ -461,7 +458,6 @@ static void *zebra_nhg_hash_alloc(void *arg)
 				nhe->nhg.nexthop->vrf_id, nhe->id);
 	}
 
-
 	return nhe;
 }
 
@@ -672,7 +668,7 @@ static void handle_recursive_depend(struct nhg_connected_tree_head *nhg_depends,
 static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 			   struct nhg_hash_entry *lookup,
 			   struct nhg_connected_tree_head *nhg_depends,
-			   afi_t afi)
+			   afi_t afi, bool from_dplane)
 {
 	bool created = false;
 	bool recursive = false;
@@ -680,10 +676,11 @@ static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 	struct nexthop *nh = NULL;
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
-		zlog_debug("%s: id %u, lookup %p, vrf %d, type %d, depends %p",
-			   __func__, lookup->id, lookup,
-			   lookup->vrf_id, lookup->type,
-			   nhg_depends);
+		zlog_debug(
+			"%s: id %u, lookup %p, vrf %d, type %d, depends %p%s",
+			__func__, lookup->id, lookup, lookup->vrf_id,
+			lookup->type, nhg_depends,
+			(from_dplane ? " (from dplane)" : ""));
 
 	if (lookup->id)
 		(*nhe) = zebra_nhg_lookup_id(lookup->id);
@@ -705,7 +702,7 @@ static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 	if (lookup->id == 0)
 		lookup->id = nhg_get_next_id();
 
-	if (lookup->id < ZEBRA_NHG_PROTO_LOWER) {
+	if (!from_dplane && lookup->id < ZEBRA_NHG_PROTO_LOWER) {
 		/*
 		 * This is a zebra hashed/owned NHG.
 		 *
@@ -715,7 +712,8 @@ static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 		zebra_nhg_insert_id(newnhe);
 	} else {
 		/*
-		 * This is upperproto owned NHG and should not be hashed to.
+		 * This is upperproto owned NHG or one we read in from dataplane
+		 * and should not be hashed to.
 		 *
 		 * It goes in ID table.
 		 */
@@ -752,7 +750,7 @@ static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 	 * resolving nexthop; or a group of nexthops, where we need
 	 * relationships with the corresponding singletons.
 	 */
-	zebra_nhg_depends_init(lookup);
+	zebra_nhg_depends_init(newnhe);
 
 	nh = newnhe->nhg.nexthop;
 
@@ -779,12 +777,19 @@ static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 					   "(R)" : "");
 
 			depends_find_add(&newnhe->nhg_depends, nh, afi,
-					 newnhe->type);
+					 newnhe->type, from_dplane);
 		}
 	}
 
 	if (recursive)
-		SET_FLAG((*nhe)->flags, NEXTHOP_GROUP_RECURSIVE);
+		SET_FLAG(newnhe->flags, NEXTHOP_GROUP_RECURSIVE);
+
+	/* Attach dependent backpointers to singletons */
+	zebra_nhg_connect_depends(newnhe, &newnhe->nhg_depends);
+
+	/**
+	 * Backup Nexthops
+	 */
 
 	if (zebra_nhg_get_backup_nhg(newnhe) == NULL ||
 	    zebra_nhg_get_backup_nhg(newnhe)->nexthop == NULL)
@@ -820,7 +825,7 @@ static bool zebra_nhe_find(struct nhg_hash_entry **nhe, /* return value */
 					   "(R)" : "");
 
 			depends_find_add(&backup_nhe->nhg_depends, nh, afi,
-					 backup_nhe->type);
+					 backup_nhe->type, from_dplane);
 		}
 	}
 
@@ -838,7 +843,8 @@ done:
 static bool zebra_nhg_find(struct nhg_hash_entry **nhe, uint32_t id,
 			   struct nexthop_group *nhg,
 			   struct nhg_connected_tree_head *nhg_depends,
-			   vrf_id_t vrf_id, afi_t afi, int type)
+			   vrf_id_t vrf_id, afi_t afi, int type,
+			   bool from_dplane)
 {
 	struct nhg_hash_entry lookup = {};
 	bool created = false;
@@ -854,7 +860,7 @@ static bool zebra_nhg_find(struct nhg_hash_entry **nhe, uint32_t id,
 	lookup.nhg = *nhg;
 
 	lookup.vrf_id = vrf_id;
-	if (lookup.nhg.nexthop->next) {
+	if (nhg_depends || lookup.nhg.nexthop->next) {
 		/* Groups can have all vrfs and AF's in them */
 		lookup.afi = AFI_UNSPEC;
 	} else {
@@ -882,14 +888,16 @@ static bool zebra_nhg_find(struct nhg_hash_entry **nhe, uint32_t id,
 		}
 	}
 
-	created = zebra_nhe_find(nhe, &lookup, nhg_depends, afi);
+	created = zebra_nhe_find(nhe, &lookup, nhg_depends, afi, from_dplane);
 
 	return created;
 }
 
 /* Find/create a single nexthop */
-static struct nhg_hash_entry *
-zebra_nhg_find_nexthop(uint32_t id, struct nexthop *nh, afi_t afi, int type)
+static struct nhg_hash_entry *zebra_nhg_find_nexthop(uint32_t id,
+						     struct nexthop *nh,
+						     afi_t afi, int type,
+						     bool from_dplane)
 {
 	struct nhg_hash_entry *nhe = NULL;
 	struct nexthop_group nhg = {};
@@ -897,7 +905,7 @@ zebra_nhg_find_nexthop(uint32_t id, struct nexthop *nh, afi_t afi, int type)
 
 	nexthop_group_add_sorted(&nhg, nh);
 
-	zebra_nhg_find(&nhe, id, &nhg, NULL, vrf_id, afi, type);
+	zebra_nhg_find(&nhe, id, &nhg, NULL, vrf_id, afi, type, from_dplane);
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 		zlog_debug("%s: nh %pNHv => %p (%u)",
@@ -1151,14 +1159,14 @@ static int nhg_ctx_process_new(struct nhg_ctx *ctx)
 		}
 
 		if (!zebra_nhg_find(&nhe, id, nhg, &nhg_depends, vrf_id, afi,
-				    type))
+				    type, true))
 			depends_decrement_free(&nhg_depends);
 
 		/* These got copied over in zebra_nhg_alloc() */
 		nexthop_group_delete(&nhg);
 	} else
-		nhe = zebra_nhg_find_nexthop(id, nhg_ctx_get_nh(ctx), afi,
-					     type);
+		nhe = zebra_nhg_find_nexthop(id, nhg_ctx_get_nh(ctx), afi, type,
+					     true);
 
 	if (!nhe) {
 		flog_err(
@@ -1325,7 +1333,7 @@ static struct nhg_hash_entry *depends_find_recursive(const struct nexthop *nh,
 
 	lookup = nexthop_dup(nh, NULL);
 
-	nhe = zebra_nhg_find_nexthop(0, lookup, afi, type);
+	nhe = zebra_nhg_find_nexthop(0, lookup, afi, type, false);
 
 	nexthops_free(lookup);
 
@@ -1333,7 +1341,8 @@ static struct nhg_hash_entry *depends_find_recursive(const struct nexthop *nh,
 }
 
 static struct nhg_hash_entry *depends_find_singleton(const struct nexthop *nh,
-						     afi_t afi, int type)
+						     afi_t afi, int type,
+						     bool from_dplane)
 {
 	struct nhg_hash_entry *nhe;
 	struct nexthop lookup = {};
@@ -1343,7 +1352,7 @@ static struct nhg_hash_entry *depends_find_singleton(const struct nexthop *nh,
 	 */
 	nexthop_copy_no_recurse(&lookup, nh, NULL);
 
-	nhe = zebra_nhg_find_nexthop(0, &lookup, afi, type);
+	nhe = zebra_nhg_find_nexthop(0, &lookup, afi, type, from_dplane);
 
 	/* The copy may have allocated labels; free them if necessary. */
 	nexthop_del_labels(&lookup);
@@ -1356,7 +1365,7 @@ static struct nhg_hash_entry *depends_find_singleton(const struct nexthop *nh,
 }
 
 static struct nhg_hash_entry *depends_find(const struct nexthop *nh, afi_t afi,
-					   int type)
+					   int type, bool from_dplane)
 {
 	struct nhg_hash_entry *nhe = NULL;
 
@@ -1369,7 +1378,7 @@ static struct nhg_hash_entry *depends_find(const struct nexthop *nh, afi_t afi,
 	if (CHECK_FLAG(nh->flags, NEXTHOP_FLAG_RECURSIVE))
 		nhe = depends_find_recursive(nh, afi, type);
 	else
-		nhe = depends_find_singleton(nh, afi, type);
+		nhe = depends_find_singleton(nh, afi, type, from_dplane);
 
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL) {
@@ -1402,11 +1411,11 @@ static void depends_add(struct nhg_connected_tree_head *head,
 
 static struct nhg_hash_entry *
 depends_find_add(struct nhg_connected_tree_head *head, struct nexthop *nh,
-		 afi_t afi, int type)
+		 afi_t afi, int type, bool from_dplane)
 {
 	struct nhg_hash_entry *depend = NULL;
 
-	depend = depends_find(nh, afi, type);
+	depend = depends_find(nh, afi, type, from_dplane);
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 		zlog_debug("%s: nh %pNHv => %p",
@@ -1452,7 +1461,7 @@ struct nhg_hash_entry *zebra_nhg_rib_find(uint32_t id,
 	assert(nhg->nexthop);
 	vrf_id = !vrf_is_backend_netns() ? VRF_DEFAULT : nhg->nexthop->vrf_id;
 
-	zebra_nhg_find(&nhe, id, nhg, NULL, vrf_id, rt_afi, type);
+	zebra_nhg_find(&nhe, id, nhg, NULL, vrf_id, rt_afi, type, false);
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 		zlog_debug("%s: => nhe %p (%u)",
@@ -1476,7 +1485,7 @@ zebra_nhg_rib_find_nhe(struct nhg_hash_entry *rt_nhe, afi_t rt_afi)
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 		zlog_debug("%s: rt_nhe %p (%u)", __func__, rt_nhe, rt_nhe->id);
 
-	zebra_nhe_find(&nhe, rt_nhe, NULL, rt_afi);
+	zebra_nhe_find(&nhe, rt_nhe, NULL, rt_afi, false);
 
 	if (IS_ZEBRA_DEBUG_NHG_DETAIL)
 		zlog_debug("%s: => nhe %p (%u)",
@@ -1582,6 +1591,7 @@ void zebra_nhg_free(struct nhg_hash_entry *nhe)
 
 void zebra_nhg_hash_free(void *p)
 {
+	zebra_nhg_release_all_deps((struct nhg_hash_entry *)p);
 	zebra_nhg_free((struct nhg_hash_entry *)p);
 }
 
@@ -2705,6 +2715,30 @@ void zebra_nhg_sweep_table(struct hash *hash)
 	hash_iterate(hash, zebra_nhg_sweep_entry, NULL);
 }
 
+static void zebra_nhg_mark_keep_entry(struct hash_bucket *bucket, void *arg)
+{
+	struct nhg_hash_entry *nhe = bucket->data;
+
+	UNSET_FLAG(nhe->flags, NEXTHOP_GROUP_INSTALLED);
+}
+
+/*
+ * When we are shutting down and we have retain mode enabled
+ * in zebra the process is to mark each vrf that it's
+ * routes should not be deleted.  The problem with that
+ * is that shutdown actually free's up memory which
+ * causes the nexthop group's ref counts to go to zero
+ * we need a way to subtly tell the system to not remove
+ * the nexthop groups from the kernel at the same time.
+ * The easiest just looks like that we should not mark
+ * the nhg's as installed any more and when the ref count
+ * goes to zero we'll attempt to delete and do nothing
+ */
+void zebra_nhg_mark_keep(void)
+{
+	hash_iterate(zrouter.nhgs_id, zebra_nhg_mark_keep_entry, NULL);
+}
+
 /* Global control to disable use of kernel nexthops, if available. We can't
  * force the kernel to support nexthop ids, of course, but we can disable
  * zebra's use of them, for testing e.g. By default, if the kernel supports
@@ -2808,10 +2842,15 @@ struct nhg_hash_entry *zebra_nhg_proto_add(uint32_t id, int type,
 	if (old) {
 		/*
 		 * This is a replace, just release NHE from ID for now, The
-		 * depends/dependents may still be used in the replacement.
+		 * depends/dependents may still be used in the replacement so
+		 * we don't touch them other than to remove their refs to their
+		 * old parent.
 		 */
 		replace = true;
 		hash_release(zrouter.nhgs_id, old);
+
+		/* Free all the things */
+		zebra_nhg_release_all_deps(old);
 	}
 
 	new = zebra_nhg_rib_find_nhe(&lookup, afi);
@@ -2848,9 +2887,6 @@ struct nhg_hash_entry *zebra_nhg_proto_add(uint32_t id, int type,
 				zebra_nhg_decrement_ref(rb_node_dep->nhe);
 		}
 
-		/* Free all the things */
-		zebra_nhg_release_all_deps(old);
-
 		/* Dont call the dec API, we dont want to uninstall the ID */
 		old->refcnt = 0;
 		zebra_nhg_free(old);