bgpd: nexthop tracking with labels for vrf-vpn leaking

Routes that have labels must be sent via a nexthop that also has labels.
This change notes whether any path in a nexthop update from zebra contains
labels. If so, then the nexthop is valid for routes that have labels.

If a nexthop update has no labeled paths, then any labeled routes
referencing the nexthop are marked not valid.

Add a route flag BGP_INFO_ANNC_NH_SELF that means "advertise myself
as nexthop when announcing" so that we can track our notion of the
nexthop without revealing it to peers.

Signed-off-by: G. Paul Ziemba <paulz@labn.net>

13 files changed, 310 insertions, 103 deletions

diff --git a/bgpd/bgp_evpn.c b/bgpd/bgp_evpn.c
index ec526a3129..e5e5f72699 100644
--- a/bgpd/bgp_evpn.c
+++ b/bgpd/bgp_evpn.c
@@ -4041,7 +4041,7 @@ struct bgpevpn *bgp_evpn_new(struct bgp *bgp, vni_t vni,
 	derive_rd_rt_for_vni(bgp, vpn);
 
 	/* Initialize EVPN route table. */
-	vpn->route_table = bgp_table_init(AFI_L2VPN, SAFI_EVPN);
+	vpn->route_table = bgp_table_init(bgp, AFI_L2VPN, SAFI_EVPN);
 
 	/* Add to hash */
 	if (!hash_get(bgp->vnihash, vpn, hash_alloc_intern)) {
diff --git a/bgpd/bgp_fsm.c b/bgpd/bgp_fsm.c
index a625b7fd29..a11a4f78f7 100644
--- a/bgpd/bgp_fsm.c
+++ b/bgpd/bgp_fsm.c
@@ -1394,7 +1394,7 @@ int bgp_start(struct peer *peer)
 	else
 		connected = 0;
 
-	if (!bgp_find_or_add_nexthop(peer->bgp,
+	if (!bgp_find_or_add_nexthop(peer->bgp, peer->bgp,
 				     family2afi(peer->su.sa.sa_family), NULL,
 				     peer, connected)) {
 #if defined(HAVE_CUMULUS)
diff --git a/bgpd/bgp_mplsvpn.c b/bgpd/bgp_mplsvpn.c
index 7320f584e5..06d82f654d 100644
--- a/bgpd/bgp_mplsvpn.c
+++ b/bgpd/bgp_mplsvpn.c
@@ -44,6 +44,7 @@
 #include "bgpd/bgp_ecommunity.h"
 #include "bgpd/bgp_zebra.h"
 #include "bgpd/bgp_nexthop.h"
+#include "bgpd/bgp_nht.h"
 
 #if ENABLE_BGP_VNC
 #include "bgpd/rfapi/rfapi_backend.h"
@@ -256,19 +257,14 @@ int bgp_nlri_parse_vpn(struct peer *peer, struct attr *attr,
 void vpn_leak_zebra_vrf_label_update(struct bgp *bgp, afi_t afi)
 {
 	mpls_label_t label = MPLS_LABEL_NONE;
-	const char *name = "default";
 	int debug = BGP_DEBUG(vpn, VPN_LEAK_LABEL);
 
-	if (debug && (bgp->inst_type != BGP_INSTANCE_TYPE_DEFAULT)) {
-		name = bgp->name;
-	}
-
 	if (bgp->vrf_id == VRF_UNKNOWN) {
 		if (debug) {
 			zlog_debug(
 				"%s: vrf %s: afi %s: vrf_id not set, "
 				"can't set zebra vrf label",
-				__func__, name, afi2str(afi));
+				__func__, bgp->name_pretty, afi2str(afi));
 		}
 		return;
 	}
@@ -279,7 +275,8 @@ void vpn_leak_zebra_vrf_label_update(struct bgp *bgp, afi_t afi)
 
 	if (debug) {
 		zlog_debug("%s: vrf %s: afi %s: setting label %d for vrf id %d",
-			   __func__, name, afi2str(afi), label, bgp->vrf_id);
+			   __func__, bgp->name_pretty, afi2str(afi), label,
+			   bgp->vrf_id);
 	}
 
 	zclient_send_vrf_label(zclient, bgp->vrf_id, afi, label, ZEBRA_LSP_BGP);
@@ -306,7 +303,7 @@ void vpn_leak_zebra_vrf_label_withdraw(struct bgp *bgp, afi_t afi)
 
 	if (debug) {
 		zlog_debug("%s: deleting label for vrf %s (id=%d)", __func__,
-			   (bgp->name ? bgp->name : "default"), bgp->vrf_id);
+			   bgp->name_pretty, bgp->vrf_id);
 	}
 
 	zclient_send_vrf_label(zclient, bgp->vrf_id, afi, label, ZEBRA_LSP_BGP);
@@ -338,22 +335,31 @@ static int ecom_intersect(struct ecommunity *e1, struct ecommunity *e2)
  * returns pointer to new bgp_info upon success
  */
 static struct bgp_info *
-leak_update(struct bgp *bgp, /* destination bgp instance */
-	    struct bgp_node *bn, struct attr *new_attr, /* already interned */
-	    afi_t afi, safi_t safi, struct bgp_info *source_bi, uint8_t type,
-	    uint8_t sub_type, mpls_label_t *label, int num_labels, void *parent,
-	    struct bgp *bgp_orig, struct prefix *nexthop_orig, int debug)
+leak_update(
+	struct bgp	*bgp,		/* destination bgp instance */
+	struct bgp_node *bn,
+	struct attr	*new_attr,	/* already interned */
+	afi_t		afi,
+	safi_t		safi,
+	struct bgp_info	*source_bi,
+	mpls_label_t	*label,
+	int		num_labels,
+	void		*parent,
+	struct bgp	*bgp_orig,
+	struct prefix	*nexthop_orig,
+	int		nexthop_self_flag,
+	int		debug)
 {
 	struct prefix *p = &bn->p;
 	struct bgp_info *bi;
 	struct bgp_info *new;
 	char buf_prefix[PREFIX_STRLEN];
-	const char *pDestInstanceName = "default";
 
 	if (debug) {
 		prefix2str(&bn->p, buf_prefix, sizeof(buf_prefix));
-		if (bgp->name)
-			pDestInstanceName = bgp->name;
+		zlog_debug("%s: entry: leak-to=%s, p=%s, type=%d, sub_type=%d",
+			__func__, bgp->name_pretty, buf_prefix,
+			source_bi->type, source_bi->sub_type);
 	}
 
 	/*
@@ -372,7 +378,7 @@ leak_update(struct bgp *bgp, /* destination bgp instance */
 			if (debug)
 				zlog_debug(
 					"%s: ->%s: %s: Found route, no change",
-					__func__, pDestInstanceName,
+					__func__, bgp->name_pretty,
 					buf_prefix);
 			return NULL;
 		}
@@ -389,6 +395,35 @@ leak_update(struct bgp *bgp, /* destination bgp instance */
 		bi->attr = new_attr;
 		bi->uptime = bgp_clock();
 
+		if (nexthop_self_flag)
+			bgp_info_set_flag(bn, bi, BGP_INFO_ANNC_NH_SELF);
+
+		struct bgp *bgp_nexthop = bgp;
+		int nh_valid;
+
+		if (bi->extra && bi->extra->bgp_orig)
+			bgp_nexthop = bi->extra->bgp_orig;
+
+		/* No nexthop tracking for redistributed routes */
+		if (source_bi->sub_type == BGP_ROUTE_REDISTRIBUTE)
+			nh_valid = 1;
+		else
+			/*
+			 * TBD do we need to do anything about the
+			 * 'connected' parameter?
+			 */
+			nh_valid = bgp_find_or_add_nexthop(
+						bgp, bgp_nexthop,
+						afi, bi, NULL, 0);
+
+		if (debug)
+			zlog_debug("%s: nexthop is %svalid (in vrf %s)",
+				__func__, (nh_valid ? "" : "not "),
+				bgp_nexthop->name_pretty);
+
+		if (nh_valid)
+			bgp_info_set_flag(bn, bi, BGP_INFO_VALID);
+
 		/* Process change. */
 		bgp_aggregate_increment(bgp, p, bi, afi, safi);
 		bgp_process(bgp, bn, afi, safi);
@@ -396,13 +431,16 @@ leak_update(struct bgp *bgp, /* destination bgp instance */
 
 		if (debug)
 			zlog_debug("%s: ->%s: %s Found route, changed attr",
-				   __func__, pDestInstanceName, buf_prefix);
+				   __func__, bgp->name_pretty, buf_prefix);
 
 		return NULL;
 	}
 
-	new = info_make(type, sub_type, 0, bgp->peer_self, new_attr, bn);
-	SET_FLAG(new->flags, BGP_INFO_VALID);
+	new = info_make(ZEBRA_ROUTE_BGP, BGP_ROUTE_IMPORTED, 0,
+		bgp->peer_self, new_attr, bn);
+
+	if (nexthop_self_flag)
+		bgp_info_set_flag(bn, new, BGP_INFO_ANNC_NH_SELF);
 
 	bgp_info_extra_get(new);
 	if (label) {
@@ -428,6 +466,37 @@ leak_update(struct bgp *bgp, /* destination bgp instance */
 	if (nexthop_orig)
 		new->extra->nexthop_orig = *nexthop_orig;
 
+	/*
+	 * nexthop tracking for unicast routes
+	 */
+	struct bgp *bgp_nexthop = bgp;
+	int nh_valid;
+
+	if (new->extra && new->extra->bgp_orig)
+		bgp_nexthop = new->extra->bgp_orig;
+
+	/*
+	 * No nexthop tracking for redistributed routes because
+	 * their originating protocols will do the tracking and
+	 * withdraw those routes if the nexthops become unreachable
+	 */
+	if (source_bi->sub_type == BGP_ROUTE_REDISTRIBUTE)
+		nh_valid = 1;
+	else
+		/*
+		 * TBD do we need to do anything about the
+		 * 'connected' parameter?
+		 */
+		nh_valid = bgp_find_or_add_nexthop(bgp, bgp_nexthop,
+						afi, new, NULL, 0);
+
+	if (debug)
+		zlog_debug("%s: nexthop is %svalid (in vrf %s)",
+			__func__, (nh_valid ? "" : "not "),
+			bgp_nexthop->name_pretty);
+	if (nh_valid)
+		bgp_info_set_flag(bn, new, BGP_INFO_VALID);
+
 	bgp_aggregate_increment(bgp, p, new, afi, safi);
 	bgp_info_add(bn, new);
 
@@ -436,7 +505,7 @@ leak_update(struct bgp *bgp, /* destination bgp instance */
 
 	if (debug)
 		zlog_debug("%s: ->%s: %s: Added new route", __func__,
-			   pDestInstanceName, buf_prefix);
+			   bgp->name_pretty, buf_prefix);
 
 	return new;
 }
@@ -456,6 +525,10 @@ void vpn_leak_from_vrf_update(struct bgp *bgp_vpn,       /* to */
 	mpls_label_t label;
 	struct bgp_node *bn;
 	const char *debugmsg;
+	int nexthop_self_flag = 0;
+
+	if (debug)
+		zlog_debug("%s: from vrf %s", __func__, bgp_vrf->name_pretty);
 
 	if (debug && info_vrf->attr->ecommunity) {
 		char *s = ecommunity_ecom2str(info_vrf->attr->ecommunity,
@@ -506,7 +579,7 @@ void vpn_leak_from_vrf_update(struct bgp *bgp_vpn,       /* to */
 			if (debug)
 				zlog_debug(
 					"%s: vrf %s route map \"%s\" says DENY, returning",
-					__func__, bgp_vrf->name,
+					__func__, bgp_vrf->name_pretty,
 					bgp_vrf->vpn_policy[afi]
 						.rmap[BGP_VPN_POLICY_DIR_TOVPN]
 						->name);
@@ -580,23 +653,14 @@ void vpn_leak_from_vrf_update(struct bgp *bgp_vpn,       /* to */
 			assert(0);
 		}
 	} else {
-		switch (afi) {
-		case AFI_IP:
-		default:
-			/* Clear ipv4 */
-			static_attr.mp_nexthop_global_in.s_addr = 0;
+		if (afi == AFI_IP) {
+			/* For ipv4, copy to multiprotocol nexthop field */
+			static_attr.mp_nexthop_global_in = static_attr.nexthop;
 			static_attr.mp_nexthop_len = 4;
-			static_attr.nexthop.s_addr = 0; /* self */
-			static_attr.flag |= ATTR_FLAG_BIT(BGP_ATTR_NEXT_HOP);
-			break;
-
-		case AFI_IP6:
-			/* Clear ipv6 */
-			memset(&static_attr.mp_nexthop_global, 0,
-			       sizeof(static_attr.mp_nexthop_global));
-			static_attr.mp_nexthop_len = 16; /* bytes */
-			break;
+			/* XXX Leave static_attr.nexthop intact for NHT */
+			static_attr.flag &= ~ATTR_FLAG_BIT(BGP_ATTR_NEXT_HOP);
 		}
+		nexthop_self_flag = 1;
 	}
 
 	label_val = bgp_vrf->vpn_policy[afi].tovpn_label;
@@ -632,8 +696,8 @@ void vpn_leak_from_vrf_update(struct bgp *bgp_vpn,       /* to */
 	struct bgp_info *new_info;
 
 	new_info = leak_update(bgp_vpn, bn, new_attr, afi, safi, info_vrf,
-			       ZEBRA_ROUTE_BGP, BGP_ROUTE_IMPORTED, &label, 1,
-			       info_vrf, bgp_vrf, NULL, debug);
+			       &label, 1, info_vrf, bgp_vrf, NULL,
+			       nexthop_self_flag, debug);
 
 	/*
 	 * Routes actually installed in the vpn RIB must also be
@@ -659,6 +723,15 @@ void vpn_leak_from_vrf_withdraw(struct bgp *bgp_vpn,       /* to */
 	struct bgp_info *bi;
 	struct bgp_node *bn;
 	const char *debugmsg;
+	char buf_prefix[PREFIX_STRLEN];
+
+	if (debug) {
+		prefix2str(p, buf_prefix, sizeof(buf_prefix));
+		zlog_debug(
+			"%s: entry: leak-from=%s, p=%s, type=%d, sub_type=%d",
+			__func__, bgp_vrf->name_pretty, buf_prefix,
+			info_vrf->type, info_vrf->sub_type);
+	}
 
 	if (info_vrf->type != ZEBRA_ROUTE_BGP) {
 		if (debug)
@@ -785,7 +858,7 @@ void vpn_leak_from_vrf_update_all(struct bgp *bgp_vpn, /* to */
 
 	if (debug)
 		zlog_debug("%s: entry, afi=%d, vrf=%s", __func__, afi,
-			   bgp_vrf->name);
+			   bgp_vrf->name_pretty);
 
 	for (bn = bgp_table_top(bgp_vrf->rib[afi][SAFI_UNICAST]); bn;
 	     bn = bgp_route_next(bn)) {
@@ -818,6 +891,7 @@ static void vpn_leak_to_vrf_update_onevrf(struct bgp *bgp_vrf,       /* to */
 	struct prefix nexthop_orig;
 	mpls_label_t *pLabels = NULL;
 	int num_labels = 0;
+	int nexthop_self_flag = 1;
 
 	int debug = BGP_DEBUG(vpn, VPN_LEAK_TO_VRF);
 
@@ -836,7 +910,8 @@ static void vpn_leak_to_vrf_update_onevrf(struct bgp *bgp_vrf,       /* to */
 	}
 
 	if (debug)
-		zlog_debug("%s: updating to vrf %s", __func__, bgp_vrf->name);
+		zlog_debug("%s: updating to vrf %s", __func__,
+				bgp_vrf->name_pretty);
 
 	bgp_attr_dup(&static_attr, info_vpn->attr); /* shallow copy */
 
@@ -858,20 +933,13 @@ static void vpn_leak_to_vrf_update_onevrf(struct bgp *bgp_vrf,       /* to */
 		/* save */
 		nexthop_orig.u.prefix4 = info_vpn->attr->mp_nexthop_global_in;
 		nexthop_orig.prefixlen = 32;
-
-		static_attr.nexthop.s_addr = 0; /* self */
 		static_attr.flag |= ATTR_FLAG_BIT(BGP_ATTR_NEXT_HOP);
-
 		break;
 
 	case AF_INET6:
 		/* save */
 		nexthop_orig.u.prefix6 = info_vpn->attr->mp_nexthop_global;
 		nexthop_orig.prefixlen = 128;
-
-		memset(&static_attr.mp_nexthop_global, 0,
-		       sizeof(static_attr.mp_nexthop_global)); /* clear */
-		static_attr.mp_nexthop_len = 16;	       /* bytes */
 		break;
 	}
 
@@ -894,12 +962,18 @@ static void vpn_leak_to_vrf_update_onevrf(struct bgp *bgp_vrf,       /* to */
 			if (debug)
 				zlog_debug(
 					"%s: vrf %s vpn-policy route map \"%s\" says DENY, returning",
-					__func__, bgp_vrf->name,
+					__func__, bgp_vrf->name_pretty,
 					bgp_vrf->vpn_policy[afi]
 						.rmap[BGP_VPN_POLICY_DIR_FROMVPN]
 						->name);
 			return;
 		}
+		/*
+		 * if route-map changed nexthop, don't nexthop-self on output
+		 */
+		if (!CHECK_FLAG(static_attr.rmap_change_flags,
+						BATTR_RMAP_NEXTHOP_UNCHANGED))
+			nexthop_self_flag = 0;
 	}
 
 	new_attr = bgp_attr_intern(&static_attr);
@@ -923,10 +997,10 @@ static void vpn_leak_to_vrf_update_onevrf(struct bgp *bgp_vrf,       /* to */
 			   num_labels);
 	}
 
-	leak_update(bgp_vrf, bn, new_attr, afi, safi, info_vpn, ZEBRA_ROUTE_BGP,
-		    BGP_ROUTE_IMPORTED, pLabels, num_labels,
-		    info_vpn, /* parent */
-		    bgp_vpn, &nexthop_orig, debug);
+	leak_update(bgp_vrf, bn, new_attr, afi, safi, info_vpn,
+		pLabels, num_labels,
+		info_vpn, /* parent */
+		bgp_vpn, &nexthop_orig, nexthop_self_flag, debug);
 }
 
 void vpn_leak_to_vrf_update(struct bgp *bgp_vpn,       /* from */
@@ -961,9 +1035,17 @@ void vpn_leak_to_vrf_withdraw(struct bgp *bgp_vpn,       /* from */
 	struct bgp_node *bn;
 	struct bgp_info *bi;
 	const char *debugmsg;
+	char buf_prefix[PREFIX_STRLEN];
 
 	int debug = BGP_DEBUG(vpn, VPN_LEAK_TO_VRF);
 
+	if (debug) {
+		prefix2str(&info_vpn->net->p, buf_prefix, sizeof(buf_prefix));
+		zlog_debug("%s: entry: p=%s, type=%d, sub_type=%d",
+			__func__, buf_prefix,
+			info_vpn->type, info_vpn->sub_type);
+	}
+
 	if (debug)
 		zlog_debug("%s: start (info_vpn=%p)", __func__, info_vpn);
 
@@ -1004,7 +1086,7 @@ void vpn_leak_to_vrf_withdraw(struct bgp *bgp_vpn,       /* from */
 
 		if (debug)
 			zlog_debug("%s: withdrawing from vrf %s", __func__,
-				   bgp->name);
+				   bgp->name_pretty);
 
 		bn = bgp_afi_node_get(bgp->rib[afi][safi], afi, safi, p, NULL);
 		for (bi = (bn ? bn->info : NULL); bi; bi = bi->next) {
diff --git a/bgpd/bgp_nexthop.c b/bgpd/bgp_nexthop.c
index 79463ee142..3700778c73 100644
--- a/bgpd/bgp_nexthop.c
+++ b/bgpd/bgp_nexthop.c
@@ -632,10 +632,11 @@ void bgp_scan_init(struct bgp *bgp)
 
 	for (afi = AFI_IP; afi < AFI_MAX; afi++) {
 		bgp->nexthop_cache_table[afi] =
-			bgp_table_init(afi, SAFI_UNICAST);
-		bgp->connected_table[afi] = bgp_table_init(afi, SAFI_UNICAST);
+			bgp_table_init(bgp, afi, SAFI_UNICAST);
+		bgp->connected_table[afi] = bgp_table_init(bgp, afi,
+			SAFI_UNICAST);
 		bgp->import_check_table[afi] =
-			bgp_table_init(afi, SAFI_UNICAST);
+			bgp_table_init(bgp, afi, SAFI_UNICAST);
 	}
 }
 
diff --git a/bgpd/bgp_nexthop.h b/bgpd/bgp_nexthop.h
index ed772868eb..519f092762 100644
--- a/bgpd/bgp_nexthop.h
+++ b/bgpd/bgp_nexthop.h
@@ -52,6 +52,7 @@ struct bgp_nexthop_cache {
 #define BGP_NEXTHOP_PEER_NOTIFIED     (1 << 3)
 #define BGP_STATIC_ROUTE              (1 << 4)
 #define BGP_STATIC_ROUTE_EXACT_MATCH  (1 << 5)
+#define BGP_NEXTHOP_LABELED_VALID     (1 << 6)
 
 	uint16_t change_flags;
 
diff --git a/bgpd/bgp_nht.c b/bgpd/bgp_nht.c
index 54c0f85cb3..8b6ff3fa22 100644
--- a/bgpd/bgp_nht.c
+++ b/bgpd/bgp_nht.c
@@ -59,6 +59,12 @@ static int bgp_isvalid_nexthop(struct bgp_nexthop_cache *bnc)
 		|| (bnc && CHECK_FLAG(bnc->flags, BGP_NEXTHOP_VALID)));
 }
 
+static int bgp_isvalid_labeled_nexthop(struct bgp_nexthop_cache *bnc)
+{
+	return (bgp_zebra_num_connects() == 0
+		|| (bnc && CHECK_FLAG(bnc->flags, BGP_NEXTHOP_LABELED_VALID)));
+}
+
 int bgp_find_nexthop(struct bgp_info *path, int connected)
 {
 	struct bgp_nexthop_cache *bnc = path->nexthop;
@@ -132,7 +138,12 @@ void bgp_unlink_nexthop_by_peer(struct peer *peer)
 	bgp_unlink_nexthop_check(bnc);
 }
 
-int bgp_find_or_add_nexthop(struct bgp *bgp, afi_t afi, struct bgp_info *ri,
+/*
+ * A route and its nexthop might belong to different VRFs. Therefore,
+ * we need both the bgp_route and bgp_nexthop pointers.
+ */
+int bgp_find_or_add_nexthop(struct bgp *bgp_route, struct bgp *bgp_nexthop,
+			    afi_t afi, struct bgp_info *ri,
 			    struct peer *peer, int connected)
 {
 	struct bgp_node *rn;
@@ -175,15 +186,15 @@ int bgp_find_or_add_nexthop(struct bgp *bgp, afi_t afi, struct bgp_info *ri,
 		return 0;
 
 	if (is_bgp_static_route)
-		rn = bgp_node_get(bgp->import_check_table[afi], &p);
+		rn = bgp_node_get(bgp_nexthop->import_check_table[afi], &p);
 	else
-		rn = bgp_node_get(bgp->nexthop_cache_table[afi], &p);
+		rn = bgp_node_get(bgp_nexthop->nexthop_cache_table[afi], &p);
 
 	if (!rn->info) {
 		bnc = bnc_new();
 		rn->info = bnc;
 		bnc->node = rn;
-		bnc->bgp = bgp;
+		bnc->bgp = bgp_nexthop;
 		bgp_lock_node(rn);
 		if (BGP_DEBUG(nht, NHT)) {
 			char buf[PREFIX2STR_BUFFER];
@@ -199,12 +210,12 @@ int bgp_find_or_add_nexthop(struct bgp *bgp, afi_t afi, struct bgp_info *ri,
 		SET_FLAG(bnc->flags, BGP_STATIC_ROUTE);
 
 		/* If we're toggling the type, re-register */
-		if ((bgp_flag_check(bgp, BGP_FLAG_IMPORT_CHECK))
+		if ((bgp_flag_check(bgp_route, BGP_FLAG_IMPORT_CHECK))
 		    && !CHECK_FLAG(bnc->flags, BGP_STATIC_ROUTE_EXACT_MATCH)) {
 			SET_FLAG(bnc->flags, BGP_STATIC_ROUTE_EXACT_MATCH);
 			UNSET_FLAG(bnc->flags, BGP_NEXTHOP_REGISTERED);
 			UNSET_FLAG(bnc->flags, BGP_NEXTHOP_VALID);
-		} else if ((!bgp_flag_check(bgp, BGP_FLAG_IMPORT_CHECK))
+		} else if ((!bgp_flag_check(bgp_route, BGP_FLAG_IMPORT_CHECK))
 			   && CHECK_FLAG(bnc->flags,
 					 BGP_STATIC_ROUTE_EXACT_MATCH)) {
 			UNSET_FLAG(bnc->flags, BGP_STATIC_ROUTE_EXACT_MATCH);
@@ -235,7 +246,7 @@ int bgp_find_or_add_nexthop(struct bgp *bgp, afi_t afi, struct bgp_info *ri,
 		UNSET_FLAG(bnc->flags, BGP_NEXTHOP_REGISTERED);
 		UNSET_FLAG(bnc->flags, BGP_NEXTHOP_VALID);
 	}
-	if (bgp->inst_type == BGP_INSTANCE_TYPE_VIEW) {
+	if (bgp_route->inst_type == BGP_INSTANCE_TYPE_VIEW) {
 		bnc->flags |= BGP_NEXTHOP_REGISTERED;
 		bnc->flags |= BGP_NEXTHOP_VALID;
 	} else if (!CHECK_FLAG(bnc->flags, BGP_NEXTHOP_REGISTERED))
@@ -261,7 +272,7 @@ int bgp_find_or_add_nexthop(struct bgp *bgp, afi_t afi, struct bgp_info *ri,
 	 * ability to detect nexthops.  So when we have a view
 	 * just tell everyone the nexthop is valid
 	 */
-	if (bgp->inst_type == BGP_INSTANCE_TYPE_VIEW)
+	if (bgp_route->inst_type == BGP_INSTANCE_TYPE_VIEW)
 		return 1;
 	else
 		return (bgp_isvalid_nexthop(bnc));
@@ -396,14 +407,27 @@ void bgp_parse_nexthop_update(int command, vrf_id_t vrf_id)
 		bnc->metric = nhr.metric;
 		bnc->nexthop_num = nhr.nexthop_num;
 
+		bnc->flags &= ~BGP_NEXTHOP_LABELED_VALID; /* check below */
+
 		for (i = 0; i < nhr.nexthop_num; i++) {
+			int num_labels = 0;
+
 			nexthop = nexthop_from_zapi_nexthop(&nhr.nexthops[i]);
 
+			/* There is at least one label-switched path */
+			if (nexthop->nh_label &&
+				nexthop->nh_label->num_labels) {
+
+				bnc->flags |= BGP_NEXTHOP_LABELED_VALID;
+				num_labels = nexthop->nh_label->num_labels;
+			}
+
 			if (BGP_DEBUG(nht, NHT)) {
 				char buf[NEXTHOP_STRLEN];
 				zlog_debug(
-					"    nhop via %s",
-					nexthop2str(nexthop, buf, sizeof(buf)));
+					"    nhop via %s (%d labels)",
+					nexthop2str(nexthop, buf, sizeof(buf)),
+					num_labels);
 			}
 
 			if (nhlist_tail) {
@@ -422,7 +446,8 @@ void bgp_parse_nexthop_update(int command, vrf_id_t vrf_id)
 				continue;
 
 			for (oldnh = bnc->nexthop; oldnh; oldnh = oldnh->next)
-				if (nexthop_same_no_recurse(oldnh, nexthop))
+				if (nexthop_same_no_recurse(oldnh, nexthop) &&
+				    nexthop_labels_match(oldnh, nexthop))
 					break;
 
 			if (!oldnh)
@@ -552,6 +577,15 @@ static void sendmsg_zebra_rnh(struct bgp_nexthop_cache *bnc, int command)
 		|| CHECK_FLAG(bnc->flags, BGP_STATIC_ROUTE_EXACT_MATCH)))
 		exact_match = true;
 
+	if (BGP_DEBUG(zebra, ZEBRA)) {
+		char buf[PREFIX2STR_BUFFER];
+
+		prefix2str(p, buf, PREFIX2STR_BUFFER);
+		zlog_debug("%s: sending cmd %s for %s (vrf %s)",
+			__func__, zserv_command_string(command), buf,
+			bnc->bgp->name);
+	}
+
 	ret = zclient_send_rnh(zclient, command, p, exact_match,
 			       bnc->bgp->vrf_id);
 	/* TBD: handle the failure */
@@ -618,11 +652,11 @@ static void evaluate_paths(struct bgp_nexthop_cache *bnc)
 {
 	struct bgp_node *rn;
 	struct bgp_info *path;
-	struct bgp *bgp = bnc->bgp;
 	int afi;
 	struct peer *peer = (struct peer *)bnc->nht_info;
 	struct bgp_table *table;
 	safi_t safi;
+	struct bgp *bgp_path;
 
 	if (BGP_DEBUG(nht, NHT)) {
 		char buf[PREFIX2STR_BUFFER];
@@ -635,7 +669,8 @@ static void evaluate_paths(struct bgp_nexthop_cache *bnc)
 	LIST_FOREACH (path, &(bnc->paths), nh_thread) {
 		if (!(path->type == ZEBRA_ROUTE_BGP
 		      && ((path->sub_type == BGP_ROUTE_NORMAL)
-			  || (path->sub_type == BGP_ROUTE_STATIC))))
+			  || (path->sub_type == BGP_ROUTE_STATIC)
+			  || (path->sub_type == BGP_ROUTE_IMPORTED))))
 			continue;
 
 		rn = path->net;
@@ -644,19 +679,55 @@ static void evaluate_paths(struct bgp_nexthop_cache *bnc)
 		table = bgp_node_table(rn);
 		safi = table->safi;
 
-		/* Path becomes valid/invalid depending on whether the nexthop
+		/*
+		 * handle routes from other VRFs (they can have a
+		 * nexthop in THIS VRF). bgp_path is the bgp instance
+		 * that owns the route referencing this nexthop.
+		 */
+		bgp_path = table->bgp;
+
+		/*
+		 * Path becomes valid/invalid depending on whether the nexthop
 		 * reachable/unreachable.
+		 *
+		 * In case of unicast routes that were imported from vpn
+		 * and that have labels, they are valid only if there are
+		 * nexthops with labels
 		 */
+
+		int bnc_is_valid_nexthop = 0;
+
+		if (safi == SAFI_UNICAST &&
+			path->sub_type == BGP_ROUTE_IMPORTED &&
+			path->extra &&
+			path->extra->num_labels) {
+
+			bnc_is_valid_nexthop =
+				bgp_isvalid_labeled_nexthop(bnc) ? 1 : 0;
+		} else {
+			bnc_is_valid_nexthop =
+				bgp_isvalid_nexthop(bnc) ? 1 : 0;
+		}
+
+		if (BGP_DEBUG(nht, NHT)) {
+			char buf[PREFIX_STRLEN];
+
+			prefix2str(&rn->p, buf, PREFIX_STRLEN);
+			zlog_debug("%s: prefix %s (vrf %s) %svalid",
+				__func__, buf, bgp_path->name,
+				(bnc_is_valid_nexthop ? "" : "not "));
+		}
+
 		if ((CHECK_FLAG(path->flags, BGP_INFO_VALID) ? 1 : 0)
-		    != (bgp_isvalid_nexthop(bnc) ? 1 : 0)) {
+		    != bnc_is_valid_nexthop) {
 			if (CHECK_FLAG(path->flags, BGP_INFO_VALID)) {
-				bgp_aggregate_decrement(bgp, &rn->p, path, afi,
-							safi);
+				bgp_aggregate_decrement(bgp_path, &rn->p,
+							path, afi, safi);
 				bgp_info_unset_flag(rn, path, BGP_INFO_VALID);
 			} else {
 				bgp_info_set_flag(rn, path, BGP_INFO_VALID);
-				bgp_aggregate_increment(bgp, &rn->p, path, afi,
-							safi);
+				bgp_aggregate_increment(bgp_path, &rn->p,
+							path, afi, safi);
 			}
 		}
 
@@ -671,7 +742,7 @@ static void evaluate_paths(struct bgp_nexthop_cache *bnc)
 		    || CHECK_FLAG(bnc->change_flags, BGP_NEXTHOP_CHANGED))
 			SET_FLAG(path->flags, BGP_INFO_IGP_CHANGED);
 
-		bgp_process(bgp, rn, afi, safi);
+		bgp_process(bgp_path, rn, afi, safi);
 	}
 
 	if (peer && !CHECK_FLAG(bnc->flags, BGP_NEXTHOP_PEER_NOTIFIED)) {
diff --git a/bgpd/bgp_nht.h b/bgpd/bgp_nht.h
index 4b297f410c..a821a42c20 100644
--- a/bgpd/bgp_nht.h
+++ b/bgpd/bgp_nht.h
@@ -39,14 +39,16 @@ extern int bgp_find_nexthop(struct bgp_info *p, int connected);
  *  object. If not found, create a new object and register with ZEBRA for
  *  nexthop notification.
  * ARGUMENTS:
- *   bgp - BGP instance
+ *   bgp_route - BGP instance of route
+ *   bgp_nexthop - BGP instance of nexthop
  *   a - afi: AFI_IP or AF_IP6
  *   p - path for which the nexthop object is being looked up
  *   peer - The BGP peer associated with this NHT
  *   connected - True if NH MUST be a connected route
  */
-extern int bgp_find_or_add_nexthop(struct bgp *bgp, afi_t a, struct bgp_info *p,
-				   struct peer *peer, int connected);
+extern int bgp_find_or_add_nexthop(struct bgp *bgp_route,
+			struct bgp *bgp_nexthop, afi_t a, struct bgp_info *p,
+			struct peer *peer, int connected);
 
 /**
  * bgp_unlink_nexthop() - Unlink the nexthop object from the path structure.
diff --git a/bgpd/bgp_route.c b/bgpd/bgp_route.c
index e65354038d..944ae5b5dc 100644
--- a/bgpd/bgp_route.c
+++ b/bgpd/bgp_route.c
@@ -115,7 +115,7 @@ struct bgp_node *bgp_afi_node_get(struct bgp_table *table, afi_t afi,
 		prn = bgp_node_get(table, (struct prefix *)prd);
 
 		if (prn->info == NULL)
-			prn->info = bgp_table_init(afi, safi);
+			prn->info = bgp_table_init(table->bgp, afi, safi);
 		else
 			bgp_unlock_node(prn);
 		table = prn->info;
@@ -1327,8 +1327,10 @@ void bgp_attr_add_gshut_community(struct attr *attr)
 
 static void subgroup_announce_reset_nhop(uint8_t family, struct attr *attr)
 {
-	if (family == AF_INET)
+	if (family == AF_INET) {
 		attr->nexthop.s_addr = 0;
+		attr->mp_nexthop_global_in.s_addr = 0;
+	}
 	if (family == AF_INET6)
 		memset(&attr->mp_nexthop_global, 0, IPV6_MAX_BYTELEN);
 }
@@ -1751,7 +1753,22 @@ int subgroup_announce_check(struct bgp_node *rn, struct bgp_info *ri,
 						 ? AF_INET6
 						 : p->family),
 					attr);
+		} else if (CHECK_FLAG(ri->flags, BGP_INFO_ANNC_NH_SELF)) {
+			/*
+			 * This flag is used for leaked vpn-vrf routes
+			 */
+			int family = p->family;
+
+			if (peer_cap_enhe(peer, afi, safi))
+				family = AF_INET6;
+
+			if (bgp_debug_update(NULL, p, subgrp->update_group, 0))
+				zlog_debug(
+					"%s: BGP_INFO_ANNC_NH_SELF, family=%s",
+					__func__, family2str(family));
+			subgroup_announce_reset_nhop(family, attr);
 		}
+
 		/* If IPv6/MP and nexthop does not have any override and happens
 		 * to
 		 * be a link-local address, reset it so that we don't pass along
@@ -3161,8 +3178,13 @@ int bgp_update(struct peer *peer, struct prefix *p, uint32_t addpath_id,
 			else
 				connected = 0;
 
-			if (bgp_find_or_add_nexthop(bgp, afi, ri, NULL,
-						    connected)
+			struct bgp *bgp_nexthop = bgp;
+
+			if (ri->extra && ri->extra->bgp_orig)
+				bgp_nexthop = ri->extra->bgp_orig;
+
+			if (bgp_find_or_add_nexthop(bgp, bgp_nexthop, afi,
+							ri, NULL, connected)
 			    || CHECK_FLAG(peer->flags, PEER_FLAG_IS_RFAPI_HD))
 				bgp_info_set_flag(rn, ri, BGP_INFO_VALID);
 			else {
@@ -3289,7 +3311,7 @@ int bgp_update(struct peer *peer, struct prefix *p, uint32_t addpath_id,
 		else
 			connected = 0;
 
-		if (bgp_find_or_add_nexthop(bgp, afi, new, NULL, connected)
+		if (bgp_find_or_add_nexthop(bgp, bgp, afi, new, NULL, connected)
 		    || CHECK_FLAG(peer->flags, PEER_FLAG_IS_RFAPI_HD))
 			bgp_info_set_flag(rn, new, BGP_INFO_VALID);
 		else {
@@ -3716,12 +3738,14 @@ static wq_item_status bgp_clear_route_node(struct work_queue *wq, void *data)
 			/* Handle withdraw for VRF route-leaking and L3VPN */
 			if (SAFI_UNICAST == safi
 			    && (bgp->inst_type == BGP_INSTANCE_TYPE_VRF ||
-				bgp->inst_type == BGP_INSTANCE_TYPE_DEFAULT))
+				bgp->inst_type == BGP_INSTANCE_TYPE_DEFAULT)) {
 				vpn_leak_from_vrf_withdraw(bgp_get_default(),
 							   bgp, ri);
+			}
 			if (SAFI_MPLS_VPN == safi &&
-			    bgp->inst_type == BGP_INSTANCE_TYPE_DEFAULT)
+			    bgp->inst_type == BGP_INSTANCE_TYPE_DEFAULT) {
 				vpn_leak_to_vrf_withdraw(bgp, ri);
+			}
 
 			bgp_rib_remove(rn, ri, peer, afi, safi);
 		}
@@ -4372,8 +4396,14 @@ void bgp_static_update(struct bgp *bgp, struct prefix *p,
 			if (bgp_flag_check(bgp, BGP_FLAG_IMPORT_CHECK)
 			    && (safi == SAFI_UNICAST
 				|| safi == SAFI_LABELED_UNICAST)) {
-				if (bgp_find_or_add_nexthop(bgp, afi, ri, NULL,
-							    0))
+
+				struct bgp *bgp_nexthop = bgp;
+
+				if (ri->extra && ri->extra->bgp_orig)
+					bgp_nexthop = ri->extra->bgp_orig;
+
+				if (bgp_find_or_add_nexthop(bgp, bgp_nexthop,
+					    afi, ri, NULL, 0))
 					bgp_info_set_flag(rn, ri,
 							  BGP_INFO_VALID);
 				else {
@@ -4424,7 +4454,7 @@ void bgp_static_update(struct bgp *bgp, struct prefix *p,
 	/* Nexthop reachability check. */
 	if (bgp_flag_check(bgp, BGP_FLAG_IMPORT_CHECK)
 	    && (safi == SAFI_UNICAST || safi == SAFI_LABELED_UNICAST)) {
-		if (bgp_find_or_add_nexthop(bgp, afi, new, NULL, 0))
+		if (bgp_find_or_add_nexthop(bgp, bgp, afi, new, NULL, 0))
 			bgp_info_set_flag(rn, new, BGP_INFO_VALID);
 		else {
 			if (BGP_DEBUG(nht, NHT)) {
@@ -5072,7 +5102,7 @@ int bgp_static_set_safi(afi_t afi, safi_t safi, struct vty *vty,
 	}
 	prn = bgp_node_get(bgp->route[afi][safi], (struct prefix *)&prd);
 	if (prn->info == NULL)
-		prn->info = bgp_table_init(afi, safi);
+		prn->info = bgp_table_init(bgp, afi, safi);
 	else
 		bgp_unlock_node(prn);
 	table = prn->info;
@@ -5170,7 +5200,7 @@ int bgp_static_unset_safi(afi_t afi, safi_t safi, struct vty *vty,
 
 	prn = bgp_node_get(bgp->route[afi][safi], (struct prefix *)&prd);
 	if (prn->info == NULL)
-		prn->info = bgp_table_init(afi, safi);
+		prn->info = bgp_table_init(bgp, afi, safi);
 	else
 		bgp_unlock_node(prn);
 	table = prn->info;
@@ -11368,7 +11398,7 @@ void bgp_route_init(void)
 
 	/* Init BGP distance table. */
 	FOREACH_AFI_SAFI (afi, safi)
-		bgp_distance_table[afi][safi] = bgp_table_init(afi, safi);
+		bgp_distance_table[afi][safi] = bgp_table_init(NULL, afi, safi);
 
 	/* IPv4 BGP commands. */
 	install_element(BGP_NODE, &bgp_table_map_cmd);
diff --git a/bgpd/bgp_route.h b/bgpd/bgp_route.h
index debd6d1ff3..1e788b00f1 100644
--- a/bgpd/bgp_route.h
+++ b/bgpd/bgp_route.h
@@ -190,6 +190,7 @@ struct bgp_info {
 #define BGP_INFO_MULTIPATH      (1 << 11)
 #define BGP_INFO_MULTIPATH_CHG  (1 << 12)
 #define BGP_INFO_RIB_ATTR_CHG   (1 << 13)
+#define BGP_INFO_ANNC_NH_SELF   (1 << 14)
 
 	/* BGP route type.  This can be static, RIP, OSPF, BGP etc.  */
 	uint8_t type;
diff --git a/bgpd/bgp_table.c b/bgpd/bgp_table.c
index 261ab9f76d..613b924d0d 100644
--- a/bgpd/bgp_table.c
+++ b/bgpd/bgp_table.c
@@ -90,7 +90,7 @@ route_table_delegate_t bgp_table_delegate = {.create_node = bgp_node_create,
 /*
  * bgp_table_init
  */
-struct bgp_table *bgp_table_init(afi_t afi, safi_t safi)
+struct bgp_table *bgp_table_init(struct bgp *bgp, afi_t afi, safi_t safi)
 {
 	struct bgp_table *rt;
 
@@ -103,6 +103,11 @@ struct bgp_table *bgp_table_init(afi_t afi, safi_t safi)
 	 */
 	rt->route_table->info = rt;
 
+	/*
+	 * pointer to bgp instance allows working back from bgp_info to bgp
+	 */
+	rt->bgp = bgp;
+
 	bgp_table_lock(rt);
 	rt->afi = afi;
 	rt->safi = safi;
diff --git a/bgpd/bgp_table.h b/bgpd/bgp_table.h
index 9afc2adbb0..388c247227 100644
--- a/bgpd/bgp_table.h
+++ b/bgpd/bgp_table.h
@@ -26,6 +26,9 @@
 #include "queue.h"
 
 struct bgp_table {
+	/* table belongs to this instance */
+	struct bgp *bgp;
+
 	/* afi/safi of this table */
 	afi_t afi;
 	safi_t safi;
@@ -75,7 +78,7 @@ typedef struct bgp_table_iter_t_ {
 	route_table_iter_t rt_iter;
 } bgp_table_iter_t;
 
-extern struct bgp_table *bgp_table_init(afi_t, safi_t);
+extern struct bgp_table *bgp_table_init(struct bgp *bgp, afi_t, safi_t);
 extern void bgp_table_lock(struct bgp_table *);
 extern void bgp_table_unlock(struct bgp_table *);
 extern void bgp_table_finish(struct bgp_table **);
diff --git a/bgpd/bgp_zebra.c b/bgpd/bgp_zebra.c
index 94d141300a..023a866315 100644
--- a/bgpd/bgp_zebra.c
+++ b/bgpd/bgp_zebra.c
@@ -1047,6 +1047,7 @@ void bgp_zebra_announce(struct bgp_node *rn, struct prefix *p,
 	route_tag_t tag;
 	mpls_label_t label;
 	int nh_othervrf = 0;
+	char buf_prefix[PREFIX_STRLEN];	/* filled in if we are debugging */
 
 	/* Don't try to install if we're not connected to Zebra or Zebra doesn't
 	 * know of this instance.
@@ -1057,6 +1058,9 @@ void bgp_zebra_announce(struct bgp_node *rn, struct prefix *p,
 	if (bgp->main_zebra_update_hold)
 		return;
 
+	if (bgp_debug_zebra(p))
+		prefix2str(&api.prefix, buf_prefix, sizeof(buf_prefix));
+
 	/*
 	 * vrf leaking support (will have only one nexthop)
 	 */
@@ -1139,9 +1143,6 @@ void bgp_zebra_announce(struct bgp_node *rn, struct prefix *p,
 			struct in_addr *nexthop;
 
 			if (bgp_debug_zebra(&api.prefix)) {
-				char buf_prefix[PREFIX_STRLEN];
-				prefix2str(&api.prefix, buf_prefix,
-					   sizeof(buf_prefix));
 				if (mpinfo->extra) {
 					zlog_debug(
 						"%s: p=%s, bgp_is_valid_label: %d",
@@ -1329,6 +1330,16 @@ void bgp_zebra_announce(struct bgp_node *rn, struct prefix *p,
 		}
 	}
 
+	if (bgp_debug_zebra(p)) {
+		int recursion_flag = 0;
+
+		if (CHECK_FLAG(api.flags, ZEBRA_FLAG_ALLOW_RECURSION))
+			recursion_flag = 1;
+
+		zlog_debug("%s: %s: announcing to zebra (recursion %sset)",
+			__func__, buf_prefix,
+			(recursion_flag ? "" : "NOT "));
+	}
 	zclient_route_send(valid_nh_count ? ZEBRA_ROUTE_ADD
 					  : ZEBRA_ROUTE_DELETE,
 			   zclient, &api);
diff --git a/bgpd/bgpd.c b/bgpd/bgpd.c
index 70dc11a7b1..c46111e1fb 100644
--- a/bgpd/bgpd.c
+++ b/bgpd/bgpd.c
@@ -2905,9 +2905,9 @@ static struct bgp *bgp_create(as_t *as, const char *name,
 	bgp->group->cmp = (int (*)(void *, void *))peer_group_cmp;
 
 	FOREACH_AFI_SAFI (afi, safi) {
-		bgp->route[afi][safi] = bgp_table_init(afi, safi);
-		bgp->aggregate[afi][safi] = bgp_table_init(afi, safi);
-		bgp->rib[afi][safi] = bgp_table_init(afi, safi);
+		bgp->route[afi][safi] = bgp_table_init(bgp, afi, safi);
+		bgp->aggregate[afi][safi] = bgp_table_init(bgp, afi, safi);
+		bgp->rib[afi][safi] = bgp_table_init(bgp, afi, safi);
 
 		/* Enable maximum-paths */
 		bgp_maximum_paths_set(bgp, afi, safi, BGP_PEER_EBGP,