zebra: Replace nexthop_group with pointer in route entry

In the route_entry we are keeping a non pointer based
nexthop group, switch the code to use a pointer for all
operations here and ensure we create and delete the memory.

Signed-off-by: Donald Sharp <sharpd@cumulusnetworks.com>

13 files changed, 80 insertions, 64 deletions

diff --git a/zebra/redistribute.c b/zebra/redistribute.c
index 0dc9de0c59..65b62679e8 100644
--- a/zebra/redistribute.c
+++ b/zebra/redistribute.c
@@ -643,7 +643,7 @@ int zebra_add_import_table_entry(struct zebra_vrf *zvrf, struct route_node *rn,
 	afi = family2afi(rn->p.family);
 	if (rmap_name)
 		ret = zebra_import_table_route_map_check(
-			afi, re->type, re->instance, &rn->p, re->ng.nexthop,
+			afi, re->type, re->instance, &rn->p, re->ng->nexthop,
 			zvrf->vrf->vrf_id, re->tag, rmap_name);
 
 	if (ret != RMAP_PERMITMATCH) {
@@ -679,7 +679,8 @@ int zebra_add_import_table_entry(struct zebra_vrf *zvrf, struct route_node *rn,
 	newre->nexthop_num = 0;
 	newre->uptime = monotime(NULL);
 	newre->instance = re->table;
-	route_entry_copy_nexthops(newre, re->ng.nexthop);
+	newre->ng = nexthop_group_new();
+	route_entry_copy_nexthops(newre, re->ng->nexthop);
 
 	rib_add_multipath(afi, SAFI_UNICAST, &p, NULL, newre);
 
@@ -696,7 +697,7 @@ int zebra_del_import_table_entry(struct zebra_vrf *zvrf, struct route_node *rn,
 	prefix_copy(&p, &rn->p);
 
 	rib_delete(afi, SAFI_UNICAST, zvrf->vrf->vrf_id, ZEBRA_ROUTE_TABLE,
-		   re->table, re->flags, &p, NULL, re->ng.nexthop,
+		   re->table, re->flags, &p, NULL, re->ng->nexthop,
 		   zvrf->table_id, re->metric, re->distance, false);
 
 	return 0;
diff --git a/zebra/rib.h b/zebra/rib.h
index ee1df89c0e..455003c930 100644
--- a/zebra/rib.h
+++ b/zebra/rib.h
@@ -88,7 +88,7 @@ struct route_entry {
 	struct re_list_item next;
 
 	/* Nexthop structure (from RIB) */
-	struct nexthop_group ng;
+	struct nexthop_group *ng;
 
 	/* Nexthop group from FIB (optional) */
 	struct nexthop_group fib_ng;
@@ -527,7 +527,7 @@ static inline struct nexthop_group *rib_active_nhg(struct route_entry *re)
 	if (re->fib_ng.nexthop)
 		return &(re->fib_ng);
 	else
-		return &(re->ng);
+		return re->ng;
 }
 
 extern void zebra_vty_init(void);
diff --git a/zebra/rt_netlink.c b/zebra/rt_netlink.c
index 43e44cad16..9bc919463b 100644
--- a/zebra/rt_netlink.c
+++ b/zebra/rt_netlink.c
@@ -627,6 +627,7 @@ static int netlink_route_change_read_unicast(struct nlmsghdr *h, ns_id_t ns_id,
 			re->nexthop_num = 0;
 			re->uptime = monotime(NULL);
 			re->tag = tag;
+			re->ng = nexthop_group_new();
 
 			for (;;) {
 				struct nexthop *nh = NULL;
@@ -722,9 +723,10 @@ static int netlink_route_change_read_unicast(struct nlmsghdr *h, ns_id_t ns_id,
 			zserv_nexthop_num_warn(__func__,
 					       (const struct prefix *)&p,
 					       re->nexthop_num);
-			if (re->nexthop_num == 0)
+			if (re->nexthop_num == 0) {
+				nexthop_group_delete(&re->ng);
 				XFREE(MTYPE_RE, re);
-			else
+			} else
 				rib_add_multipath(afi, SAFI_UNICAST, &p,
 						  &src_p, re);
 		}
diff --git a/zebra/zapi_msg.c b/zebra/zapi_msg.c
index e61e68b7fe..d207ee4046 100644
--- a/zebra/zapi_msg.c
+++ b/zebra/zapi_msg.c
@@ -564,7 +564,7 @@ int zsend_redistribute_route(int cmd, struct zserv *client,
 		SET_FLAG(api.message, ZAPI_MESSAGE_NEXTHOP);
 		api.nexthop_num = re->nexthop_active_num;
 	}
-	for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next) {
+	for (nexthop = re->ng->nexthop; nexthop; nexthop = nexthop->next) {
 		if (!CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 			continue;
 
@@ -665,7 +665,8 @@ static int zsend_ipv4_nexthop_lookup_mrib(struct zserv *client,
 		 * nexthop we are looking up. Therefore, we will just iterate
 		 * over the top chain of nexthops.
 		 */
-		for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next)
+		for (nexthop = re->ng->nexthop; nexthop;
+		     nexthop = nexthop->next)
 			if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 				num += zserv_encode_nexthop(s, nexthop);
 
@@ -1422,6 +1423,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 	re->flags = api.flags;
 	re->uptime = monotime(NULL);
 	re->vrf_id = vrf_id;
+	re->ng = nexthop_group_new();
+
 	if (api.tableid)
 		re->table = api.tableid;
 	else
@@ -1433,6 +1436,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 			  "%s: received a route without nexthops for prefix %pFX from client %s",
 			  __func__, &api.prefix,
 			  zebra_route_string(client->proto));
+
+		nexthop_group_delete(&re->ng);
 		XFREE(MTYPE_RE, re);
 		return;
 	}
@@ -1531,7 +1536,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 				EC_ZEBRA_NEXTHOP_CREATION_FAILED,
 				"%s: Nexthops Specified: %d but we failed to properly create one",
 				__PRETTY_FUNCTION__, api.nexthop_num);
-			nexthops_free(re->ng.nexthop);
+			nexthops_free(re->ng->nexthop);
+			nexthop_group_delete(&re->ng);
 			XFREE(MTYPE_RE, re);
 			return;
 		}
@@ -1573,7 +1579,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 		flog_warn(EC_ZEBRA_RX_SRCDEST_WRONG_AFI,
 			  "%s: Received SRC Prefix but afi is not v6",
 			  __PRETTY_FUNCTION__);
-		nexthops_free(re->ng.nexthop);
+		nexthops_free(re->ng->nexthop);
+		nexthop_group_delete(&re->ng);
 		XFREE(MTYPE_RE, re);
 		return;
 	}
diff --git a/zebra/zebra_dplane.c b/zebra/zebra_dplane.c
index bf343e06e5..31fcba083f 100644
--- a/zebra/zebra_dplane.c
+++ b/zebra/zebra_dplane.c
@@ -1419,7 +1419,7 @@ static int dplane_ctx_route_init(struct zebra_dplane_ctx *ctx,
 	ctx->u.rinfo.zd_safi = info->safi;
 
 	/* Copy nexthops; recursive info is included too */
-	copy_nexthops(&(ctx->u.rinfo.zd_ng.nexthop), re->ng.nexthop, NULL);
+	copy_nexthops(&(ctx->u.rinfo.zd_ng.nexthop), re->ng->nexthop, NULL);
 
 	/* Ensure that the dplane's nexthops flags are clear. */
 	for (ALL_NEXTHOPS(ctx->u.rinfo.zd_ng, nexthop))
@@ -1577,7 +1577,7 @@ static int dplane_ctx_pw_init(struct zebra_dplane_ctx *ctx,
 
 			if (re)
 				copy_nexthops(&(ctx->u.pw.nhg.nexthop),
-					      re->ng.nexthop, NULL);
+					      re->ng->nexthop, NULL);
 
 			route_unlock_node(rn);
 		}
diff --git a/zebra/zebra_fpm_netlink.c b/zebra/zebra_fpm_netlink.c
index f347d3955c..b54d8fbc12 100644
--- a/zebra/zebra_fpm_netlink.c
+++ b/zebra/zebra_fpm_netlink.c
@@ -314,7 +314,7 @@ static int netlink_route_info_fill(netlink_route_info_t *ri, int cmd,
 	ri->rtm_type = RTN_UNICAST;
 	ri->metric = &re->metric;
 
-	for (ALL_NEXTHOPS(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 		if (ri->num_nhs >= zrouter.multipath_num)
 			break;
 
diff --git a/zebra/zebra_fpm_protobuf.c b/zebra/zebra_fpm_protobuf.c
index 3054b8a34d..a11517ab8b 100644
--- a/zebra/zebra_fpm_protobuf.c
+++ b/zebra/zebra_fpm_protobuf.c
@@ -173,7 +173,7 @@ static Fpm__AddRoute *create_add_route_message(qpb_allocator_t *allocator,
 	 * Figure out the set of nexthops to be added to the message.
 	 */
 	num_nhs = 0;
-	for (ALL_NEXTHOPS(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 		if (num_nhs >= zrouter.multipath_num)
 			break;
 
diff --git a/zebra/zebra_mpls.c b/zebra/zebra_mpls.c
index 8088ec1bfe..6e0c0b8d88 100644
--- a/zebra/zebra_mpls.c
+++ b/zebra/zebra_mpls.c
@@ -185,7 +185,7 @@ static int lsp_install(struct zebra_vrf *zvrf, mpls_label_t label,
 	 * the label advertised by the recursive nexthop (plus we don't have the
 	 * logic yet to push multiple labels).
 	 */
-	for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next) {
+	for (nexthop = re->ng->nexthop; nexthop; nexthop = nexthop->next) {
 		/* Skip inactive and recursive entries. */
 		if (!CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_ACTIVE))
 			continue;
@@ -635,7 +635,7 @@ static int nhlfe_nexthop_active_ipv4(zebra_nhlfe_t *nhlfe,
 		    || !CHECK_FLAG(match->flags, ZEBRA_FLAG_SELECTED))
 			continue;
 
-		for (match_nh = match->ng.nexthop; match_nh;
+		for (match_nh = match->ng->nexthop; match_nh;
 		     match_nh = match_nh->next) {
 			if (match->type == ZEBRA_ROUTE_CONNECT
 			    || nexthop->ifindex == match_nh->ifindex) {
@@ -686,10 +686,10 @@ static int nhlfe_nexthop_active_ipv6(zebra_nhlfe_t *nhlfe,
 			break;
 	}
 
-	if (!match || !match->ng.nexthop)
+	if (!match || !match->ng->nexthop)
 		return 0;
 
-	nexthop->ifindex = match->ng.nexthop->ifindex;
+	nexthop->ifindex = match->ng->nexthop->ifindex;
 	return 1;
 }
 
@@ -2611,7 +2611,7 @@ int mpls_ftn_update(int add, struct zebra_vrf *zvrf, enum lsp_types_t type,
 		return -1;
 
 	found = false;
-	for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next) {
+	for (nexthop = re->ng->nexthop; nexthop; nexthop = nexthop->next) {
 		switch (nexthop->type) {
 		case NEXTHOP_TYPE_IPV4:
 		case NEXTHOP_TYPE_IPV4_IFINDEX:
@@ -2889,7 +2889,7 @@ static void mpls_ftn_uninstall_all(struct zebra_vrf *zvrf,
 	for (rn = route_top(table); rn; rn = route_next(rn)) {
 		update = 0;
 		RNODE_FOREACH_RE (rn, re) {
-			for (nexthop = re->ng.nexthop; nexthop;
+			for (nexthop = re->ng->nexthop; nexthop;
 			     nexthop = nexthop->next) {
 				if (nexthop->nh_label_type != lsp_type)
 					continue;
diff --git a/zebra/zebra_nhg.c b/zebra/zebra_nhg.c
index 772227a46b..407e29f8c3 100644
--- a/zebra/zebra_nhg.c
+++ b/zebra/zebra_nhg.c
@@ -164,7 +164,7 @@ void zebra_nhg_release(afi_t afi, struct route_entry *re)
 
 	lookup.vrf_id = re->vrf_id;
 	lookup.afi = afi;
-	lookup.nhg = re->ng;
+	lookup.nhg = *re->ng;
 
 	nhe = hash_lookup(zrouter.nhgs, &lookup);
 	nhe->refcnt--;
@@ -446,7 +446,7 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 
 		if (match->type == ZEBRA_ROUTE_CONNECT) {
 			/* Directly point connected route. */
-			newhop = match->ng.nexthop;
+			newhop = match->ng->nexthop;
 			if (newhop) {
 				if (nexthop->type == NEXTHOP_TYPE_IPV4
 				    || nexthop->type == NEXTHOP_TYPE_IPV6)
@@ -455,7 +455,7 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 			return 1;
 		} else if (CHECK_FLAG(re->flags, ZEBRA_FLAG_ALLOW_RECURSION)) {
 			resolved = 0;
-			for (ALL_NEXTHOPS(match->ng, newhop)) {
+			for (ALL_NEXTHOPS_PTR(match->ng, newhop)) {
 				if (!CHECK_FLAG(match->status,
 						ROUTE_ENTRY_INSTALLED))
 					continue;
@@ -475,7 +475,7 @@ static int nexthop_active(afi_t afi, struct route_entry *re,
 			return resolved;
 		} else if (re->type == ZEBRA_ROUTE_STATIC) {
 			resolved = 0;
-			for (ALL_NEXTHOPS(match->ng, newhop)) {
+			for (ALL_NEXTHOPS_PTR(match->ng, newhop)) {
 				if (!CHECK_FLAG(match->status,
 						ROUTE_ENTRY_INSTALLED))
 					continue;
@@ -660,7 +660,7 @@ int nexthop_active_update(struct route_node *rn, struct route_entry *re)
 	re->nexthop_active_num = 0;
 	UNSET_FLAG(re->status, ROUTE_ENTRY_CHANGED);
 
-	for (nexthop = re->ng.nexthop; nexthop; nexthop = nexthop->next) {
+	for (nexthop = re->ng->nexthop; nexthop; nexthop = nexthop->next) {
 		/* No protocol daemon provides src and so we're skipping
 		 * tracking it */
 		prev_src = nexthop->rmap_src;
diff --git a/zebra/zebra_pw.c b/zebra/zebra_pw.c
index 09edbc9a68..3f1567a95b 100644
--- a/zebra/zebra_pw.c
+++ b/zebra/zebra_pw.c
@@ -259,7 +259,7 @@ static int zebra_pw_check_reachability(struct zebra_pw *pw)
 	 * Need to ensure that there's a label binding for all nexthops.
 	 * Otherwise, ECMP for this route could render the pseudowire unusable.
 	 */
-	for (ALL_NEXTHOPS(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 		if (!nexthop->nh_label) {
 			if (IS_ZEBRA_DEBUG_PW)
 				zlog_debug("%s: unlabeled route for %s",
diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c
index f385a2d752..c8ba3f9e9d 100644
--- a/zebra/zebra_rib.c
+++ b/zebra/zebra_rib.c
@@ -196,7 +196,7 @@ int zebra_check_addr(const struct prefix *p)
 /* Add nexthop to the end of a rib node's nexthop list */
 void route_entry_nexthop_add(struct route_entry *re, struct nexthop *nexthop)
 {
-	_nexthop_group_add_sorted(&re->ng, nexthop);
+	_nexthop_group_add_sorted(re->ng, nexthop);
 	re->nexthop_num++;
 }
 
@@ -206,8 +206,8 @@ void route_entry_nexthop_add(struct route_entry *re, struct nexthop *nexthop)
  */
 void route_entry_copy_nexthops(struct route_entry *re, struct nexthop *nh)
 {
-	assert(!re->ng.nexthop);
-	copy_nexthops(&re->ng.nexthop, nh, NULL);
+	assert(!re->ng->nexthop);
+	copy_nexthops(&re->ng->nexthop, nh, NULL);
 	for (struct nexthop *nexthop = nh; nexthop; nexthop = nexthop->next)
 		re->nexthop_num++;
 }
@@ -220,7 +220,7 @@ void route_entry_nexthop_delete(struct route_entry *re, struct nexthop *nexthop)
 	if (nexthop->prev)
 		nexthop->prev->next = nexthop->next;
 	else
-		re->ng.nexthop = nexthop->next;
+		re->ng->nexthop = nexthop->next;
 	re->nexthop_num--;
 }
 
@@ -505,7 +505,7 @@ int zebra_rib_labeled_unicast(struct route_entry *re)
 	if (re->type != ZEBRA_ROUTE_BGP)
 		return 0;
 
-	for (ALL_NEXTHOPS(re->ng, nexthop))
+	for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
 		if (!nexthop->nh_label || !nexthop->nh_label->num_labels)
 			return 0;
 
@@ -529,15 +529,15 @@ void rib_install_kernel(struct route_node *rn, struct route_entry *re,
 	srcdest_rnode_prefixes(rn, &p, &src_p);
 
 	if (info->safi != SAFI_UNICAST) {
-		for (ALL_NEXTHOPS(re->ng, nexthop))
+		for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
 			SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 		return;
 	} else {
 		struct nexthop *prev;
 
-		for (ALL_NEXTHOPS(re->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 			UNSET_FLAG (nexthop->flags, NEXTHOP_FLAG_DUPLICATE);
-			for (ALL_NEXTHOPS(re->ng, prev)) {
+			for (ALL_NEXTHOPS_PTR(re->ng, prev)) {
 				if (prev == nexthop)
 					break;
 				if (nexthop_same_firsthop(nexthop, prev)) {
@@ -586,7 +586,7 @@ void rib_install_kernel(struct route_node *rn, struct route_entry *re,
 
 			if (!RIB_SYSTEM_ROUTE(old)) {
 				/* Clear old route's FIB flags */
-				for (ALL_NEXTHOPS(old->ng, nexthop)) {
+				for (ALL_NEXTHOPS_PTR(old->ng, nexthop)) {
 					UNSET_FLAG(nexthop->flags,
 						   NEXTHOP_FLAG_FIB);
 				}
@@ -624,7 +624,7 @@ void rib_uninstall_kernel(struct route_node *rn, struct route_entry *re)
 
 	if (info->safi != SAFI_UNICAST) {
 		UNSET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
-		for (ALL_NEXTHOPS(re->ng, nexthop))
+		for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 		return;
 	}
@@ -684,7 +684,7 @@ static void rib_uninstall(struct route_node *rn, struct route_entry *re)
 			re->fib_ng.nexthop = NULL;
 		}
 
-		for (ALL_NEXTHOPS(re->ng, nexthop))
+		for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 	}
 
@@ -860,7 +860,7 @@ static void rib_process_add_fib(struct zebra_vrf *zvrf, struct route_node *rn,
 
 	/* Update real nexthop. This may actually determine if nexthop is active
 	 * or not. */
-	if (!nexthop_group_active_nexthop_num(&new->ng)) {
+	if (!nexthop_group_active_nexthop_num(new->ng)) {
 		UNSET_FLAG(new->status, ROUTE_ENTRY_CHANGED);
 		return;
 	}
@@ -929,7 +929,7 @@ static void rib_process_update_fib(struct zebra_vrf *zvrf,
 
 		/* Update the nexthop; we could determine here that nexthop is
 		 * inactive. */
-		if (nexthop_group_active_nexthop_num(&new->ng))
+		if (nexthop_group_active_nexthop_num(new->ng))
 			nh_active = 1;
 
 		/* If nexthop is active, install the selected route, if
@@ -1047,7 +1047,7 @@ static struct route_entry *rib_choose_best(struct route_entry *current,
 		/* both are connected.  are either loop or vrf? */
 		struct nexthop *nexthop = NULL;
 
-		for (ALL_NEXTHOPS(alternate->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(alternate->ng, nexthop)) {
 			struct interface *ifp = if_lookup_by_index(
 				nexthop->ifindex, alternate->vrf_id);
 
@@ -1055,7 +1055,7 @@ static struct route_entry *rib_choose_best(struct route_entry *current,
 				return alternate;
 		}
 
-		for (ALL_NEXTHOPS(current->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(current->ng, nexthop)) {
 			struct interface *ifp = if_lookup_by_index(
 				nexthop->ifindex, current->vrf_id);
 
@@ -1380,7 +1380,7 @@ static void zebra_rib_fixup_system(struct route_node *rn)
 		SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
 		UNSET_FLAG(re->status, ROUTE_ENTRY_QUEUED);
 
-		for (ALL_NEXTHOPS(re->ng, nhop)) {
+		for (ALL_NEXTHOPS_PTR(re->ng, nhop)) {
 			if (CHECK_FLAG(nhop->flags, NEXTHOP_FLAG_RECURSIVE))
 				continue;
 
@@ -1532,7 +1532,7 @@ static bool rib_update_re_from_ctx(struct route_entry *re,
 	 * that is actually installed.
 	 */
 	matched = true;
-	for (ALL_NEXTHOPS(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 
 		if (CHECK_FLAG(nexthop->flags, NEXTHOP_FLAG_RECURSIVE))
 			continue;
@@ -2418,7 +2418,8 @@ void rib_unlink(struct route_node *rn, struct route_entry *re)
 	info = srcdest_rnode_table_info(rn);
 	zebra_nhg_release(info->afi, re);
 
-	nexthops_free(re->ng.nexthop);
+	nexthops_free(re->ng->nexthop);
+	nexthop_group_delete(&re->ng);
 	nexthops_free(re->fib_ng.nexthop);
 
 	XFREE(MTYPE_RE, re);
@@ -2486,7 +2487,7 @@ void _route_entry_dump(const char *func, union prefixconstptr pp,
 	zlog_debug("%s: nexthop_num == %u, nexthop_active_num == %u", straddr,
 		   re->nexthop_num, re->nexthop_active_num);
 
-	for (ALL_NEXTHOPS(re->ng, nexthop)) {
+	for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 		struct interface *ifp;
 		struct vrf *vrf = vrf_lookup_by_id(nexthop->vrf_id);
 
@@ -2650,6 +2651,8 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 	/* Lookup table.  */
 	table = zebra_vrf_table_with_table_id(afi, safi, re->vrf_id, re->table);
 	if (!table) {
+		nexthops_free(re->ng->nexthop);
+		nexthop_group_delete(&re->ng);
 		XFREE(MTYPE_RE, re);
 		return 0;
 	}
@@ -2659,7 +2662,7 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 	if (src_p)
 		apply_mask_ipv6(src_p);
 
-	zebra_nhg_find(afi, &re->ng, re);
+	zebra_nhg_find(afi, re->ng, re);
 	/* Set default distance by route type. */
 	if (re->distance == 0)
 		re->distance = route_distance(re->type);
@@ -2795,7 +2798,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 
 		if (re->type == ZEBRA_ROUTE_KERNEL && re->metric != metric)
 			continue;
-		if (re->type == ZEBRA_ROUTE_CONNECT && (rtnh = re->ng.nexthop)
+		if (re->type == ZEBRA_ROUTE_CONNECT && (rtnh = re->ng->nexthop)
 		    && rtnh->type == NEXTHOP_TYPE_IFINDEX && nh) {
 			if (rtnh->ifindex != nh->ifindex)
 				continue;
@@ -2808,7 +2811,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 				same = re;
 				break;
 			}
-			for (ALL_NEXTHOPS(re->ng, rtnh))
+			for (ALL_NEXTHOPS_PTR(re->ng, rtnh))
 				/*
 				 * No guarantee all kernel send nh with labels
 				 * on delete.
@@ -2849,7 +2852,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 			if (allow_delete) {
 				UNSET_FLAG(fib->status, ROUTE_ENTRY_INSTALLED);
 				/* Unset flags. */
-				for (rtnh = fib->ng.nexthop; rtnh;
+				for (rtnh = fib->ng->nexthop; rtnh;
 				     rtnh = rtnh->next)
 					UNSET_FLAG(rtnh->flags,
 						   NEXTHOP_FLAG_FIB);
@@ -2905,7 +2908,7 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 		if (CHECK_FLAG(flags, ZEBRA_FLAG_EVPN_ROUTE)) {
 			struct nexthop *tmp_nh;
 
-			for (ALL_NEXTHOPS(re->ng, tmp_nh)) {
+			for (ALL_NEXTHOPS_PTR(re->ng, tmp_nh)) {
 				struct ipaddr vtep_ip;
 
 				memset(&vtep_ip, 0, sizeof(struct ipaddr));
@@ -2959,6 +2962,7 @@ int rib_add(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 	re->nexthop_num = 0;
 	re->uptime = monotime(NULL);
 	re->tag = tag;
+	re->ng = nexthop_group_new();
 
 	/* Add nexthop. */
 	nexthop = nexthop_new();
@@ -3223,7 +3227,7 @@ void rib_sweep_table(struct route_table *table)
 			 * this decision needs to be revisited
 			 */
 			SET_FLAG(re->status, ROUTE_ENTRY_INSTALLED);
-			for (ALL_NEXTHOPS(re->ng, nexthop))
+			for (ALL_NEXTHOPS_PTR(re->ng, nexthop))
 				SET_FLAG(nexthop->flags, NEXTHOP_FLAG_FIB);
 
 			rib_uninstall_kernel(rn, re);
diff --git a/zebra/zebra_rnh.c b/zebra/zebra_rnh.c
index 5df5d94f4b..1494e3bed3 100644
--- a/zebra/zebra_rnh.c
+++ b/zebra/zebra_rnh.c
@@ -384,7 +384,7 @@ static void zebra_rnh_clear_nexthop_rnh_filters(struct route_entry *re)
 	struct nexthop *nexthop;
 
 	if (re) {
-		for (nexthop = re->ng.nexthop; nexthop;
+		for (nexthop = re->ng->nexthop; nexthop;
 		     nexthop = nexthop->next) {
 			UNSET_FLAG(nexthop->flags, NEXTHOP_FLAG_RNH_FILTERED);
 		}
@@ -403,7 +403,7 @@ static int zebra_rnh_apply_nht_rmap(afi_t afi, struct zebra_vrf *zvrf,
 	route_map_result_t ret;
 
 	if (prn && re) {
-		for (nexthop = re->ng.nexthop; nexthop;
+		for (nexthop = re->ng->nexthop; nexthop;
 		     nexthop = nexthop->next) {
 			ret = zebra_nht_route_map_check(
 				afi, proto, &prn->p, zvrf, re, nexthop);
@@ -688,7 +688,7 @@ zebra_rnh_resolve_nexthop_entry(struct zebra_vrf *zvrf, afi_t afi,
 			/* Just being SELECTED isn't quite enough - must
 			 * have an installed nexthop to be useful.
 			 */
-			for (ALL_NEXTHOPS(re->ng, nexthop)) {
+			for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 				if (rnh_nexthop_valid(re, nexthop))
 					break;
 			}
@@ -707,7 +707,7 @@ zebra_rnh_resolve_nexthop_entry(struct zebra_vrf *zvrf, afi_t afi,
 					break;
 				if (re->type == ZEBRA_ROUTE_NHRP) {
 
-					for (nexthop = re->ng.nexthop; nexthop;
+					for (nexthop = re->ng->nexthop; nexthop;
 					     nexthop = nexthop->next)
 						if (nexthop->type
 						    == NEXTHOP_TYPE_IFINDEX)
@@ -940,7 +940,8 @@ static void free_state(vrf_id_t vrf_id, struct route_entry *re,
 		return;
 
 	/* free RE and nexthops */
-	nexthops_free(re->ng.nexthop);
+	nexthops_free(re->ng->nexthop);
+	nexthop_group_delete(&re->ng);
 	XFREE(MTYPE_RE, re);
 }
 
@@ -963,8 +964,9 @@ static void copy_state(struct rnh *rnh, struct route_entry *re,
 	state->metric = re->metric;
 	state->vrf_id = re->vrf_id;
 	state->status = re->status;
+	state->ng = nexthop_group_new();
 
-	route_entry_copy_nexthops(state, re->ng.nexthop);
+	route_entry_copy_nexthops(state, re->ng->nexthop);
 	rnh->state = state;
 }
 
@@ -985,7 +987,7 @@ static int compare_state(struct route_entry *r1, struct route_entry *r2)
 	if (r1->nexthop_num != r2->nexthop_num)
 		return 1;
 
-	if (nexthop_group_hash(&r1->ng) != nexthop_group_hash(&r2->ng))
+	if (nexthop_group_hash(r1->ng) != nexthop_group_hash(r2->ng))
 		return 1;
 
 	return 0;
@@ -1035,7 +1037,7 @@ static int send_client(struct rnh *rnh, struct zserv *client, rnh_type_t type,
 		num = 0;
 		nump = stream_get_endp(s);
 		stream_putc(s, 0);
-		for (ALL_NEXTHOPS(re->ng, nh))
+		for (ALL_NEXTHOPS_PTR(re->ng, nh))
 			if (rnh_nexthop_valid(re, nh)) {
 				stream_putl(s, nh->vrf_id);
 				stream_putc(s, nh->type);
@@ -1135,7 +1137,7 @@ static void print_rnh(struct route_node *rn, struct vty *vty)
 	if (rnh->state) {
 		vty_out(vty, " resolved via %s\n",
 			zebra_route_string(rnh->state->type));
-		for (nexthop = rnh->state->ng.nexthop; nexthop;
+		for (nexthop = rnh->state->ng->nexthop; nexthop;
 		     nexthop = nexthop->next)
 			print_nh(nexthop, vty);
 	} else
diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index d9e8c6f968..32375e0a00 100644
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -259,7 +259,7 @@ static void vty_show_ip_route_detail(struct vty *vty, struct route_node *rn,
 				tm->tm_hour);
 		vty_out(vty, " ago\n");
 
-		for (ALL_NEXTHOPS(re->ng, nexthop)) {
+		for (ALL_NEXTHOPS_PTR(re->ng, nexthop)) {
 			char addrstr[32];
 
 			vty_out(vty, "  %c%s",
@@ -409,7 +409,7 @@ static void vty_show_ip_route(struct vty *vty, struct route_node *rn,
 	if (is_fib)
 		nhg = rib_active_nhg(re);
 	else
-		nhg = &(re->ng);
+		nhg = re->ng;
 
 	if (json) {
 		json_route = json_object_new_object();
@@ -1615,7 +1615,7 @@ static void vty_show_ip_route_summary_prefix(struct vty *vty,
 				fib_cnt[ZEBRA_ROUTE_TOTAL]++;
 				fib_cnt[re->type]++;
 			}
-			for (nexthop = re->ng.nexthop; (!cnt && nexthop);
+			for (nexthop = re->ng->nexthop; (!cnt && nexthop);
 			     nexthop = nexthop->next) {
 				cnt++;
 				rib_cnt[ZEBRA_ROUTE_TOTAL]++;