Merge pull request #7016 from donaldsharp/pbr_vrf_is_burf

Pbr vrf is burf

12 files changed, 362 insertions, 46 deletions

diff --git a/lib/if.c b/lib/if.c
index d8392708e1..10db574dc4 100644
--- a/lib/if.c
+++ b/lib/if.c
@@ -188,7 +188,9 @@ void if_destroy_via_zapi(struct interface *ifp)
 	if (ifp_master.destroy_hook)
 		(*ifp_master.destroy_hook)(ifp);
 
+	ifp->oldifindex = ifp->ifindex;
 	if_set_index(ifp, IFINDEX_INTERNAL);
+
 	if (!ifp->configured)
 		if_delete(&ifp);
 }
diff --git a/lib/if.h b/lib/if.h
index 1fb0757db2..a2a40d0957 100644
--- a/lib/if.h
+++ b/lib/if.h
@@ -224,6 +224,8 @@ struct interface {
 	   not work as expected.
 	 */
 	ifindex_t ifindex;
+	ifindex_t oldifindex;
+
 	/*
 	 * ifindex of parent interface, if any
 	 */
diff --git a/lib/nexthop_group.c b/lib/nexthop_group.c
index 8ae001e42a..687cac4062 100644
--- a/lib/nexthop_group.c
+++ b/lib/nexthop_group.c
@@ -940,6 +940,12 @@ DEFPY(ecmp_nexthops, ecmp_nexthops_cmd,
 			nhg_hooks.add_nexthop(nhgc, nh);
 	}
 
+	if (intf) {
+		struct interface *ifp = if_lookup_by_name_all_vrf(intf);
+
+		if (ifp)
+			ifp->configured = true;
+	}
 	return CMD_SUCCESS;
 }
 
@@ -952,24 +958,29 @@ static struct cmd_node nexthop_group_node = {
 	.config_write = nexthop_group_write,
 };
 
-void nexthop_group_write_nexthop(struct vty *vty, const struct nexthop *nh)
+void nexthop_group_write_nexthop_simple(struct vty *vty,
+					const struct nexthop *nh,
+					char *altifname)
 {
 	char buf[100];
-	struct vrf *vrf;
-	int i;
+	char *ifname;
 
 	vty_out(vty, "nexthop ");
 
+	if (altifname)
+		ifname = altifname;
+	else
+		ifname = (char *)ifindex2ifname(nh->ifindex, nh->vrf_id);
+
 	switch (nh->type) {
 	case NEXTHOP_TYPE_IFINDEX:
-		vty_out(vty, "%s", ifindex2ifname(nh->ifindex, nh->vrf_id));
+		vty_out(vty, "%s", ifname);
 		break;
 	case NEXTHOP_TYPE_IPV4:
 		vty_out(vty, "%s", inet_ntoa(nh->gate.ipv4));
 		break;
 	case NEXTHOP_TYPE_IPV4_IFINDEX:
-		vty_out(vty, "%s %s", inet_ntoa(nh->gate.ipv4),
-			ifindex2ifname(nh->ifindex, nh->vrf_id));
+		vty_out(vty, "%s %s", inet_ntoa(nh->gate.ipv4), ifname);
 		break;
 	case NEXTHOP_TYPE_IPV6:
 		vty_out(vty, "%s",
@@ -978,11 +989,19 @@ void nexthop_group_write_nexthop(struct vty *vty, const struct nexthop *nh)
 	case NEXTHOP_TYPE_IPV6_IFINDEX:
 		vty_out(vty, "%s %s",
 			inet_ntop(AF_INET6, &nh->gate.ipv6, buf, sizeof(buf)),
-			ifindex2ifname(nh->ifindex, nh->vrf_id));
+			ifname);
 		break;
 	case NEXTHOP_TYPE_BLACKHOLE:
 		break;
 	}
+}
+
+void nexthop_group_write_nexthop(struct vty *vty, const struct nexthop *nh)
+{
+	struct vrf *vrf;
+	int i;
+
+	nexthop_group_write_nexthop_simple(vty, nh, NULL);
 
 	if (nh->vrf_id != VRF_DEFAULT) {
 		vrf = vrf_lookup_by_id(nh->vrf_id);
@@ -1229,6 +1248,7 @@ void nexthop_group_interface_state_change(struct interface *ifp,
 				if (ifp->ifindex != nhop.ifindex)
 					continue;
 
+				ifp->configured = true;
 				nh = nexthop_new();
 
 				memcpy(nh, &nhop, sizeof(nhop));
diff --git a/lib/nexthop_group.h b/lib/nexthop_group.h
index 0b5ac91bb2..5f7bde0def 100644
--- a/lib/nexthop_group.h
+++ b/lib/nexthop_group.h
@@ -135,6 +135,9 @@ extern bool nexthop_group_equal(const struct nexthop_group *nhg1,
 
 extern struct nexthop_group_cmd *nhgc_find(const char *name);
 
+extern void nexthop_group_write_nexthop_simple(struct vty *vty,
+					       const struct nexthop *nh,
+					       char *altifname);
 extern void nexthop_group_write_nexthop(struct vty *vty,
 					const struct nexthop *nh);
 
diff --git a/lib/zclient.c b/lib/zclient.c
index 6b5f3e349a..b842e7c31b 100644
--- a/lib/zclient.c
+++ b/lib/zclient.c
@@ -2068,7 +2068,7 @@ static void zebra_interface_if_set_value(struct stream *s,
 	uint8_t link_params_status = 0;
 	ifindex_t old_ifindex, new_ifindex;
 
-	old_ifindex = ifp->ifindex;
+	old_ifindex = ifp->oldifindex;
 	/* Read interface's index. */
 	STREAM_GETL(s, new_ifindex);
 	if_set_index(ifp, new_ifindex);
diff --git a/pbrd/pbr_map.c b/pbrd/pbr_map.c
index 10a75a9f54..cce97b5dfb 100644
--- a/pbrd/pbr_map.c
+++ b/pbrd/pbr_map.c
@@ -771,6 +771,57 @@ void pbr_map_check_nh_group_change(const char *nh_group)
 	}
 }
 
+void pbr_map_check_vrf_nh_group_change(const char *nh_group,
+				       struct pbr_vrf *pbr_vrf,
+				       uint32_t old_vrf_id)
+{
+	struct pbr_map *pbrm;
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node;
+
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			if (pbrms->nhgrp_name)
+				continue;
+
+			if (pbrms->nhg
+			    && strcmp(nh_group, pbrms->internal_nhg_name))
+				continue;
+
+			if (pbrms->nhg->nexthop->vrf_id != old_vrf_id)
+				continue;
+
+			pbrms->nhg->nexthop->vrf_id = pbr_vrf_id(pbr_vrf);
+		}
+	}
+}
+
+void pbr_map_check_interface_nh_group_change(const char *nh_group,
+					     struct interface *ifp,
+					     ifindex_t oldifindex)
+{
+	struct pbr_map *pbrm;
+	struct pbr_map_sequence *pbrms;
+	struct listnode *node;
+
+	RB_FOREACH (pbrm, pbr_map_entry_head, &pbr_maps) {
+		for (ALL_LIST_ELEMENTS_RO(pbrm->seqnumbers, node, pbrms)) {
+			if (pbrms->nhgrp_name)
+				continue;
+
+			if (pbrms->nhg
+			    && strcmp(nh_group, pbrms->internal_nhg_name))
+				continue;
+
+			if (pbrms->nhg->nexthop->ifindex != oldifindex)
+				continue;
+
+			pbrms->nhg->nexthop->ifindex = ifp->ifindex;
+		}
+	}
+}
+
 void pbr_map_check(struct pbr_map_sequence *pbrms, bool changed)
 {
 	struct pbr_map *pbrm;
diff --git a/pbrd/pbr_map.h b/pbrd/pbr_map.h
index 64c090d2e8..43266f21e9 100644
--- a/pbrd/pbr_map.h
+++ b/pbrd/pbr_map.h
@@ -205,4 +205,11 @@ extern void pbr_map_install(struct pbr_map *pbrm);
 extern void pbr_map_policy_install(const char *name);
 extern void pbr_map_policy_delete(struct pbr_map *pbrm,
 				  struct pbr_map_interface *pmi);
+
+extern void pbr_map_check_vrf_nh_group_change(const char *nh_group,
+					      struct pbr_vrf *pbr_vrf,
+					      uint32_t old_vrf_id);
+extern void pbr_map_check_interface_nh_group_change(const char *nh_group,
+						    struct interface *ifp,
+						    ifindex_t oldifindex);
 #endif
diff --git a/pbrd/pbr_nht.c b/pbrd/pbr_nht.c
index 31da656793..e615818a10 100644
--- a/pbrd/pbr_nht.c
+++ b/pbrd/pbr_nht.c
@@ -37,7 +37,7 @@
 
 DEFINE_MTYPE_STATIC(PBRD, PBR_NHG, "PBR Nexthop Groups")
 
-static struct hash *pbr_nhg_hash;
+struct hash *pbr_nhg_hash;
 static struct hash *pbr_nhrc_hash;
 
 static uint32_t pbr_nhg_low_table;
@@ -91,15 +91,15 @@ static void *pbr_nh_alloc(void *p)
 	struct nhrc *nhrc;
 
 	new = XCALLOC(MTYPE_PBR_NHG, sizeof(*new));
-	nhrc = hash_get(pbr_nhrc_hash, pnhc->nexthop, pbr_nhrc_hash_alloc);
-	new->nexthop = &nhrc->nexthop;
+	nhrc = hash_get(pbr_nhrc_hash, &pnhc->nexthop, pbr_nhrc_hash_alloc);
+	new->nexthop = nhrc->nexthop;
 
 	/* Decremented again in pbr_nh_delete */
 	++nhrc->refcount;
 
 	DEBUGD(&pbr_dbg_nht, "%s: Sending nexthop to Zebra", __func__);
 
-	pbr_send_rnh(new->nexthop, true);
+	pbr_send_rnh(&new->nexthop, true);
 
 	new->valid = false;
 	return new;
@@ -109,14 +109,14 @@ static void pbr_nh_delete(struct pbr_nexthop_cache **pnhc)
 {
 	struct nhrc *nhrc;
 
-	nhrc = hash_lookup(pbr_nhrc_hash, (*pnhc)->nexthop);
+	nhrc = hash_lookup(pbr_nhrc_hash, &((*pnhc)->nexthop));
 
 	if (nhrc)
 		--nhrc->refcount;
 	if (!nhrc || nhrc->refcount == 0) {
 		DEBUGD(&pbr_dbg_nht, "%s: Removing nexthop from Zebra",
 		       __func__);
-		pbr_send_rnh((*pnhc)->nexthop, false);
+		pbr_send_rnh(&((*pnhc)->nexthop), false);
 	}
 	if (nhrc && nhrc->refcount == 0) {
 		hash_release(pbr_nhrc_hash, nhrc);
@@ -136,7 +136,7 @@ static uint32_t pbr_nh_hash_key(const void *arg)
 	uint32_t key;
 	const struct pbr_nexthop_cache *pbrnc = arg;
 
-	key = nexthop_hash(pbrnc->nexthop);
+	key = nexthop_hash(&pbrnc->nexthop);
 
 	return key;
 }
@@ -148,28 +148,28 @@ static bool pbr_nh_hash_equal(const void *arg1, const void *arg2)
 	const struct pbr_nexthop_cache *pbrnc2 =
 		(const struct pbr_nexthop_cache *)arg2;
 
-	if (pbrnc1->nexthop->vrf_id != pbrnc2->nexthop->vrf_id)
+	if (pbrnc1->nexthop.vrf_id != pbrnc2->nexthop.vrf_id)
 		return false;
 
-	if (pbrnc1->nexthop->ifindex != pbrnc2->nexthop->ifindex)
+	if (pbrnc1->nexthop.ifindex != pbrnc2->nexthop.ifindex)
 		return false;
 
-	if (pbrnc1->nexthop->type != pbrnc2->nexthop->type)
+	if (pbrnc1->nexthop.type != pbrnc2->nexthop.type)
 		return false;
 
-	switch (pbrnc1->nexthop->type) {
+	switch (pbrnc1->nexthop.type) {
 	case NEXTHOP_TYPE_IFINDEX:
-		return pbrnc1->nexthop->ifindex == pbrnc2->nexthop->ifindex;
+		return pbrnc1->nexthop.ifindex == pbrnc2->nexthop.ifindex;
 	case NEXTHOP_TYPE_IPV4_IFINDEX:
 	case NEXTHOP_TYPE_IPV4:
-		return pbrnc1->nexthop->gate.ipv4.s_addr
-		       == pbrnc2->nexthop->gate.ipv4.s_addr;
+		return pbrnc1->nexthop.gate.ipv4.s_addr
+		       == pbrnc2->nexthop.gate.ipv4.s_addr;
 	case NEXTHOP_TYPE_IPV6_IFINDEX:
 	case NEXTHOP_TYPE_IPV6:
-		return !memcmp(&pbrnc1->nexthop->gate.ipv6,
-			       &pbrnc2->nexthop->gate.ipv6, 16);
+		return !memcmp(&pbrnc1->nexthop.gate.ipv6,
+			       &pbrnc2->nexthop.gate.ipv6, 16);
 	case NEXTHOP_TYPE_BLACKHOLE:
-		return pbrnc1->nexthop->bh_type == pbrnc2->nexthop->bh_type;
+		return pbrnc1->nexthop.bh_type == pbrnc2->nexthop.bh_type;
 	}
 
 	/*
@@ -249,9 +249,8 @@ void pbr_nhgroup_add_nexthop_cb(const struct nexthop_group_cmd *nhgc,
 	pnhgc = hash_get(pbr_nhg_hash, &pnhgc_find, pbr_nhgc_alloc);
 
 	/* create & insert new pnhc into pnhgc->nhh */
-	pnhc_find.nexthop = (struct nexthop *)nhop;
+	pnhc_find.nexthop = *nhop;
 	pnhc = hash_get(pnhgc->nhh, &pnhc_find, pbr_nh_alloc);
-	pnhc_find.nexthop = NULL;
 
 	/* set parent pnhgc */
 	pnhc->parent = pnhgc;
@@ -291,7 +290,7 @@ void pbr_nhgroup_del_nexthop_cb(const struct nexthop_group_cmd *nhgc,
 	pnhgc = hash_lookup(pbr_nhg_hash, &pnhgc_find);
 
 	/* delete pnhc from pnhgc->nhh */
-	pnhc_find.nexthop = (struct nexthop *)nhop;
+	pnhc_find.nexthop = *nhop;
 	pnhc = hash_release(pnhgc->nhh, &pnhc_find);
 
 	/* delete pnhc */
@@ -499,7 +498,7 @@ void pbr_nht_change_group(const char *name)
 		struct pbr_nexthop_cache lookup;
 		struct pbr_nexthop_cache *pnhc;
 
-		lookup.nexthop = nhop;
+		lookup.nexthop = *nhop;
 		pnhc = hash_lookup(pnhgc->nhh, &lookup);
 		if (!pnhc) {
 			pnhc = hash_get(pnhgc->nhh, &lookup, pbr_nh_alloc);
@@ -553,9 +552,25 @@ void pbr_nht_add_individual_nexthop(struct pbr_map_sequence *pbrms,
 
 	pnhgc = hash_get(pbr_nhg_hash, &find, pbr_nhgc_alloc);
 
-	lookup.nexthop = pbrms->nhg->nexthop;
+	lookup.nexthop = *pbrms->nhg->nexthop;
 	pnhc = hash_get(pnhgc->nhh, &lookup, pbr_nh_alloc);
 	pnhc->parent = pnhgc;
+	if (nhop->vrf_id != VRF_DEFAULT) {
+		struct vrf *vrf = vrf_lookup_by_id(nhop->vrf_id);
+
+		if (vrf)
+			strlcpy(pnhc->vrf_name, vrf->name,
+				sizeof(pnhc->vrf_name));
+	}
+
+	if (nhop->ifindex != 0) {
+		struct interface *ifp =
+			if_lookup_by_index(nhop->ifindex, nhop->vrf_id);
+
+		if (ifp)
+			strlcpy(pnhc->intf_name, ifp->name,
+				sizeof(pnhc->intf_name));
+	}
 	pbr_nht_install_nexthop_group(pnhgc, *pbrms->nhg);
 }
 
@@ -574,7 +589,7 @@ static void pbr_nht_release_individual_nexthop(struct pbr_map_sequence *pbrms)
 
 	nh = pbrms->nhg->nexthop;
 	nh_type = nh->type;
-	lup.nexthop = nh;
+	lup.nexthop = *nh;
 	pnhc = hash_lookup(pnhgc->nhh, &lup);
 	pnhc->parent = NULL;
 	hash_release(pnhgc->nhh, pnhc);
@@ -625,7 +640,7 @@ struct pbr_nexthop_group_cache *pbr_nht_add_group(const char *name)
 		struct pbr_nexthop_cache lookupc;
 		struct pbr_nexthop_cache *pnhc;
 
-		lookupc.nexthop = nhop;
+		lookupc.nexthop = *nhop;
 		pnhc = hash_lookup(pnhgc->nhh, &lookupc);
 		if (!pnhc) {
 			pnhc = hash_get(pnhgc->nhh, &lookupc, pbr_nh_alloc);
@@ -689,8 +704,13 @@ bool pbr_nht_nexthop_group_valid(const char *name)
 struct pbr_nht_individual {
 	struct zapi_route *nhr;
 	struct interface *ifp;
+	struct pbr_vrf *pbr_vrf;
+	struct pbr_nexthop_cache *pnhc;
+	vrf_id_t old_vrf_id;
 
 	uint32_t valid;
+
+	bool nhr_matched;
 };
 
 static bool
@@ -704,12 +724,12 @@ pbr_nht_individual_nexthop_gw_update(struct pbr_nexthop_cache *pnhc,
 
 	switch (pnhi->nhr->prefix.family) {
 	case AF_INET:
-		if (pnhc->nexthop->gate.ipv4.s_addr
+		if (pnhc->nexthop.gate.ipv4.s_addr
 		    != pnhi->nhr->prefix.u.prefix4.s_addr)
 			goto done; /* Unrelated change */
 		break;
 	case AF_INET6:
-		if (memcmp(&pnhc->nexthop->gate.ipv6,
+		if (memcmp(&pnhc->nexthop.gate.ipv6,
 			   &pnhi->nhr->prefix.u.prefix6, 16)
 		    != 0)
 			goto done; /* Unrelated change */
@@ -721,8 +741,8 @@ pbr_nht_individual_nexthop_gw_update(struct pbr_nexthop_cache *pnhc,
 		goto done;
 	}
 
-	if (pnhc->nexthop->type == NEXTHOP_TYPE_IPV4_IFINDEX
-	    || pnhc->nexthop->type == NEXTHOP_TYPE_IPV6_IFINDEX) {
+	if (pnhc->nexthop.type == NEXTHOP_TYPE_IPV4_IFINDEX
+	    || pnhc->nexthop.type == NEXTHOP_TYPE_IPV6_IFINDEX) {
 
 		/* GATEWAY_IFINDEX type shouldn't resolve to group */
 		if (pnhi->nhr->nexthop_num > 1) {
@@ -733,7 +753,7 @@ pbr_nht_individual_nexthop_gw_update(struct pbr_nexthop_cache *pnhc,
 		/* If whatever we resolved to wasn't on the interface we
 		 * specified. (i.e. not a connected route), its invalid.
 		 */
-		if (pnhi->nhr->nexthops[0].ifindex != pnhc->nexthop->ifindex) {
+		if (pnhi->nhr->nexthops[0].ifindex != pnhc->nexthop.ifindex) {
 			is_valid = false;
 			goto done;
 		}
@@ -755,7 +775,7 @@ static bool pbr_nht_individual_nexthop_interface_update(
 	if (!pnhi->ifp) /* It doesn't care about non-interface updates */
 		goto done;
 
-	if (pnhc->nexthop->ifindex
+	if (pnhc->nexthop.ifindex
 	    != pnhi->ifp->ifindex) /* Un-related interface */
 		goto done;
 
@@ -779,12 +799,12 @@ pbr_nht_individual_nexthop_update(struct pbr_nexthop_cache *pnhc,
 {
 	assert(pnhi->nhr || pnhi->ifp); /* Either nexthop or interface update */
 
-	switch (pnhc->nexthop->type) {
+	switch (pnhc->nexthop.type) {
 	case NEXTHOP_TYPE_IFINDEX:
 		pbr_nht_individual_nexthop_interface_update(pnhc, pnhi);
 		break;
 	case NEXTHOP_TYPE_IPV6_IFINDEX:
-		if (IN6_IS_ADDR_LINKLOCAL(&pnhc->nexthop->gate.ipv6)) {
+		if (IN6_IS_ADDR_LINKLOCAL(&pnhc->nexthop.gate.ipv6)) {
 			pbr_nht_individual_nexthop_interface_update(pnhc, pnhi);
 			break;
 		}
@@ -827,7 +847,7 @@ static void pbr_nexthop_group_cache_iterate_to_group(struct hash_bucket *b,
 	struct nexthop_group *nhg = data;
 	struct nexthop *nh = NULL;
 
-	copy_nexthops(&nh, pnhc->nexthop, NULL);
+	copy_nexthops(&nh, &pnhc->nexthop, NULL);
 
 	_nexthop_add(&nhg->nexthop, nh);
 }
@@ -877,6 +897,178 @@ void pbr_nht_nexthop_update(struct zapi_route *nhr)
 	hash_iterate(pbr_nhg_hash, pbr_nht_nexthop_update_lookup, nhr);
 }
 
+struct nhrc_vrf_info {
+	struct pbr_vrf *pbr_vrf;
+	uint32_t old_vrf_id;
+	struct nhrc *nhrc;
+};
+
+static int pbr_nht_nhrc_vrf_change(struct hash_bucket *b, void *data)
+{
+	struct nhrc *nhrc = b->data;
+	struct nhrc_vrf_info *nhrcvi = data;
+
+	if (nhrc->nexthop.vrf_id == nhrcvi->old_vrf_id) {
+		nhrcvi->nhrc = nhrc;
+		return HASHWALK_ABORT;
+	}
+
+	return HASHWALK_CONTINUE;
+}
+
+static int pbr_nht_individual_nexthop_vrf_handle(struct hash_bucket *b,
+						 void *data)
+{
+	struct pbr_nexthop_cache *pnhc = b->data;
+	struct pbr_nht_individual *pnhi = data;
+
+	if (pnhc->looked_at == true)
+		return HASHWALK_CONTINUE;
+
+	if (pnhc->nexthop.vrf_id == VRF_DEFAULT)
+		return HASHWALK_CONTINUE;
+
+	if (strncmp(pnhc->vrf_name, pbr_vrf_name(pnhi->pbr_vrf),
+		    sizeof(pnhc->vrf_name))
+	    == 0) {
+		pnhi->pnhc = pnhc;
+
+		if (pnhc->nexthop.vrf_id != pbr_vrf_id(pnhi->pbr_vrf)) {
+			struct nhrc_vrf_info nhrcvi;
+
+			memset(&nhrcvi, 0, sizeof(nhrcvi));
+			nhrcvi.pbr_vrf = pnhi->pbr_vrf;
+			nhrcvi.old_vrf_id = pnhc->nexthop.vrf_id;
+
+			pnhi->nhr_matched = true;
+			pnhi->old_vrf_id = pnhc->nexthop.vrf_id;
+
+			do {
+				nhrcvi.nhrc = NULL;
+				hash_walk(pbr_nhrc_hash,
+					  pbr_nht_nhrc_vrf_change, &nhrcvi);
+				if (nhrcvi.nhrc) {
+					hash_release(pbr_nhrc_hash,
+						     nhrcvi.nhrc);
+					nhrcvi.nhrc->nexthop.vrf_id =
+						pbr_vrf_id(pnhi->pbr_vrf);
+					hash_get(pbr_nhrc_hash, nhrcvi.nhrc,
+						 hash_alloc_intern);
+					pbr_send_rnh(&nhrcvi.nhrc->nexthop, true);
+				}
+			} while (nhrcvi.nhrc);
+		}
+
+		pnhc->looked_at = true;
+		return HASHWALK_ABORT;
+	}
+
+	return HASHWALK_CONTINUE;
+}
+
+static void pbr_nht_clear_looked_at(struct hash_bucket *b, void *data)
+{
+	struct pbr_nexthop_cache *pnhc = b->data;
+
+	pnhc->looked_at = false;
+}
+
+static void pbr_nht_nexthop_vrf_handle(struct hash_bucket *b, void *data)
+{
+	struct pbr_nexthop_group_cache *pnhgc = b->data;
+	struct pbr_vrf *pbr_vrf = data;
+	struct pbr_nht_individual pnhi = {};
+
+	zlog_debug("pnhgc iterating");
+	hash_iterate(pnhgc->nhh, pbr_nht_clear_looked_at, NULL);
+	memset(&pnhi, 0, sizeof(pnhi));
+	pnhi.pbr_vrf = pbr_vrf;
+	do {
+		struct pbr_nexthop_cache *pnhc;
+
+		pnhi.pnhc = NULL;
+		hash_walk(pnhgc->nhh, pbr_nht_individual_nexthop_vrf_handle,
+			  &pnhi);
+
+		if (!pnhi.pnhc)
+			continue;
+
+		pnhc = pnhi.pnhc;
+		pnhc->nexthop.vrf_id = pnhi.old_vrf_id;
+		pnhi.pnhc = hash_release(pnhgc->nhh, pnhi.pnhc);
+		if (pnhi.pnhc) {
+			pnhi.pnhc->nexthop.vrf_id = pbr_vrf_id(pbr_vrf);
+
+			hash_get(pnhgc->nhh, pnhi.pnhc, hash_alloc_intern);
+		} else
+			pnhc->nexthop.vrf_id = pbr_vrf_id(pbr_vrf);
+
+		pbr_map_check_vrf_nh_group_change(pnhgc->name, pbr_vrf,
+						  pnhi.old_vrf_id);
+	} while (pnhi.pnhc);
+}
+
+void pbr_nht_vrf_update(struct pbr_vrf *pbr_vrf)
+{
+	hash_iterate(pbr_nhg_hash, pbr_nht_nexthop_vrf_handle, pbr_vrf);
+}
+
+static void pbr_nht_individual_nexthop_interface_handle(struct hash_bucket *b,
+							void *data)
+{
+	struct pbr_nexthop_cache *pnhc = b->data;
+	struct pbr_nht_individual *pnhi = data;
+
+	if (pnhc->nexthop.ifindex == 0)
+		return;
+
+	if ((strncmp(pnhc->intf_name, pnhi->ifp->name, sizeof(pnhc->intf_name))
+	     == 0)
+	    && pnhc->nexthop.ifindex != pnhi->ifp->ifindex)
+		pnhi->pnhc = pnhc;
+}
+
+static void pbr_nht_nexthop_interface_handle(struct hash_bucket *b, void *data)
+{
+	struct pbr_nexthop_group_cache *pnhgc = b->data;
+	struct interface *ifp = data;
+	struct pbr_nht_individual pnhi = {};
+	struct nhrc *nhrc;
+	uint32_t old_ifindex;
+
+	do {
+		memset(&pnhi, 0, sizeof(pnhi));
+		pnhi.ifp = ifp;
+		hash_iterate(pnhgc->nhh,
+			     pbr_nht_individual_nexthop_interface_handle,
+			     &pnhi);
+
+		if (!pnhi.pnhc)
+			continue;
+
+		pnhi.pnhc = hash_release(pnhgc->nhh, pnhi.pnhc);
+		old_ifindex = pnhi.pnhc->nexthop.ifindex;
+
+		nhrc = hash_lookup(pbr_nhrc_hash, &pnhi.pnhc->nexthop);
+		if (nhrc) {
+			hash_release(pbr_nhrc_hash, nhrc);
+			nhrc->nexthop.ifindex = ifp->ifindex;
+			hash_get(pbr_nhrc_hash, nhrc, hash_alloc_intern);
+		}
+		pnhi.pnhc->nexthop.ifindex = ifp->ifindex;
+
+		hash_get(pnhgc->nhh, pnhi.pnhc, hash_alloc_intern);
+
+		pbr_map_check_interface_nh_group_change(pnhgc->name, ifp,
+							old_ifindex);
+	} while (pnhi.pnhc);
+}
+
+void pbr_nht_interface_update(struct interface *ifp)
+{
+	hash_iterate(pbr_nhg_hash, pbr_nht_nexthop_interface_handle, ifp);
+}
+
 static void
 pbr_nht_individual_nexthop_interface_update_lookup(struct hash_bucket *b,
 						   void *data)
@@ -1036,7 +1228,7 @@ static void pbr_nht_show_nhg_nexthops(struct hash_bucket *b, void *data)
 	struct vty *vty = data;
 
 	vty_out(vty, "\tValid: %d ", pnhc->valid);
-	nexthop_group_write_nexthop(vty, pnhc->nexthop);
+	nexthop_group_write_nexthop(vty, &pnhc->nexthop);
 }
 
 static void pbr_nht_json_nhg_nexthops(struct hash_bucket *b, void *data)
@@ -1046,7 +1238,7 @@ static void pbr_nht_json_nhg_nexthops(struct hash_bucket *b, void *data)
 	json_object *this_hop;
 
 	this_hop = json_object_new_object();
-	nexthop_group_json_nexthop(this_hop, pnhc->nexthop);
+	nexthop_group_json_nexthop(this_hop, &pnhc->nexthop);
 	json_object_boolean_add(this_hop, "valid", pnhc->valid);
 
 	json_object_array_add(all_hops, this_hop);
diff --git a/pbrd/pbr_nht.h b/pbrd/pbr_nht.h
index cbcf71d2f5..bcd770c2cf 100644
--- a/pbrd/pbr_nht.h
+++ b/pbrd/pbr_nht.h
@@ -28,6 +28,8 @@
 
 #define PBR_NHC_NAMELEN PBR_MAP_NAMELEN + 10
 
+extern struct hash *pbr_nhg_hash;
+
 struct pbr_nexthop_group_cache {
 	char name[PBR_NHC_NAMELEN];
 
@@ -46,8 +48,12 @@ struct pbr_nexthop_group_cache {
 struct pbr_nexthop_cache {
 	struct pbr_nexthop_group_cache *parent;
 
-	struct nexthop *nexthop;
+	char vrf_name[VRF_NAMSIZ + 1];
+	char intf_name[INTERFACE_NAMSIZ + 1];
+
+	struct nexthop nexthop;
 
+	bool looked_at;
 	bool valid;
 };
 
@@ -126,4 +132,7 @@ extern void pbr_nht_nexthop_update(struct zapi_route *nhr);
 extern void pbr_nht_nexthop_interface_update(struct interface *ifp);
 
 extern void pbr_nht_init(void);
+
+extern void pbr_nht_vrf_update(struct pbr_vrf *pbr_vrf);
+extern void pbr_nht_interface_update(struct interface *ifp);
 #endif
diff --git a/pbrd/pbr_vrf.c b/pbrd/pbr_vrf.c
index d5a2bd0fef..389e5e8be0 100644
--- a/pbrd/pbr_vrf.c
+++ b/pbrd/pbr_vrf.c
@@ -25,6 +25,7 @@
 #include "pbr_memory.h"
 #include "pbr_map.h"
 #include "pbr_debug.h"
+#include "pbr_nht.h"
 
 DEFINE_MTYPE_STATIC(PBRD, PBR_MAP_VRF, "PBR Map VRF")
 
@@ -59,6 +60,7 @@ static int pbr_vrf_enable(struct vrf *vrf)
 {
 	DEBUGD(&pbr_dbg_event, "%s: %u (%s)", __func__, vrf->vrf_id, vrf->name);
 
+	pbr_nht_vrf_update(vrf->info);
 	pbr_map_vrf_update(vrf->info);
 
 	return 0;
diff --git a/pbrd/pbr_vty.c b/pbrd/pbr_vty.c
index a73d885ea6..9f966f617f 100644
--- a/pbrd/pbr_vty.c
+++ b/pbrd/pbr_vty.c
@@ -618,6 +618,33 @@ DEFPY (show_pbr,
 	return CMD_SUCCESS;
 }
 
+static void
+pbrms_nexthop_group_write_individual_nexthop(
+	struct vty *vty, const struct pbr_map_sequence *pbrms)
+{
+	struct pbr_nexthop_group_cache find;
+	struct pbr_nexthop_group_cache *pnhgc;
+	struct pbr_nexthop_cache lookup;
+	struct pbr_nexthop_cache *pnhc;
+
+	memset(&find, 0, sizeof(find));
+	strlcpy(find.name, pbrms->internal_nhg_name, sizeof(find.name));
+
+	pnhgc = hash_lookup(pbr_nhg_hash, &find);
+	assert(pnhgc);
+
+	lookup.nexthop = *pbrms->nhg->nexthop;
+	pnhc = hash_lookup(pnhgc->nhh, &lookup);
+
+	nexthop_group_write_nexthop_simple(
+		vty, pbrms->nhg->nexthop,
+		pnhc->nexthop.ifindex != 0 ? pnhc->intf_name : NULL);
+	if (pnhc->nexthop.vrf_id != VRF_DEFAULT)
+		vty_out(vty, " nexthop-vrf %s", pnhc->vrf_name);
+
+	vty_out(vty, "\n");
+}
+
 static void vty_show_pbrms(struct vty *vty,
 			   const struct pbr_map_sequence *pbrms, bool detail)
 {
@@ -670,7 +697,7 @@ static void vty_show_pbrms(struct vty *vty,
 
 	} else if (pbrms->nhg) {
 		vty_out(vty, "        ");
-		nexthop_group_write_nexthop(vty, pbrms->nhg->nexthop);
+		pbrms_nexthop_group_write_individual_nexthop(vty, pbrms);
 		if (detail)
 			vty_out(vty,
 				"          Installed: %u(%d) Tableid: %d\n",
@@ -1065,7 +1092,7 @@ static int pbr_vty_map_config_write_sequence(struct vty *vty,
 
 	if (pbrms->nhg) {
 		vty_out(vty, " set ");
-		nexthop_group_write_nexthop(vty, pbrms->nhg->nexthop);
+		pbrms_nexthop_group_write_individual_nexthop(vty, pbrms);
 	}
 
 	vty_out(vty, "!\n");
diff --git a/pbrd/pbr_zebra.c b/pbrd/pbr_zebra.c
index d0099a46e3..a7420974a9 100644
--- a/pbrd/pbr_zebra.c
+++ b/pbrd/pbr_zebra.c
@@ -67,6 +67,7 @@ int pbr_ifp_create(struct interface *ifp)
 	if (!ifp->info)
 		pbr_if_new(ifp);
 
+	pbr_nht_interface_update(ifp);
 	/* Update nexthops tracked from a `set nexthop` command */
 	pbr_nht_nexthop_interface_update(ifp);