bgpd: Implement enhanced route refresh capability

16:40:49 BGP: 192.168.0.2: sending route-refresh (BoRR) for IPv4/unicast
16:40:51 BGP: 192.168.0.2: sending route-refresh (EoRR) for IPv4/unicast

Signed-off-by: Donatas Abraitis <donatas.abraitis@gmail.com>

12 files changed, 425 insertions, 46 deletions

diff --git a/bgpd/bgp_debug.c b/bgpd/bgp_debug.c
index 2c076fb80b..dc48323c1a 100644
--- a/bgpd/bgp_debug.c
+++ b/bgpd/bgp_debug.c
@@ -118,7 +118,7 @@ static const struct message bgp_notify_msg[] = {
 	{BGP_NOTIFY_HOLD_ERR, "Hold Timer Expired"},
 	{BGP_NOTIFY_FSM_ERR, "Neighbor Events Error"},
 	{BGP_NOTIFY_CEASE, "Cease"},
-	{BGP_NOTIFY_CAPABILITY_ERR, "CAPABILITY Message Error"},
+	{BGP_NOTIFY_ROUTE_REFRESH_ERR, "ROUTE-REFRESH Message Error"},
 	{0}};
 
 static const struct message bgp_notify_head_msg[] = {
@@ -166,11 +166,9 @@ static const struct message bgp_notify_cease_msg[] = {
 	{BGP_NOTIFY_CEASE_OUT_OF_RESOURCE, "/Out of Resource"},
 	{0}};
 
-static const struct message bgp_notify_capability_msg[] = {
+static const struct message bgp_notify_route_refresh_msg[] = {
 	{BGP_NOTIFY_SUBCODE_UNSPECIFIC, "/Unspecific"},
-	{BGP_NOTIFY_CAPABILITY_INVALID_ACTION, "/Invalid Action Value"},
-	{BGP_NOTIFY_CAPABILITY_INVALID_LENGTH, "/Invalid Capability Length"},
-	{BGP_NOTIFY_CAPABILITY_MALFORMED_CODE, "/Malformed Capability Value"},
+	{BGP_NOTIFY_ROUTE_REFRESH_INVALID_MSG_LEN, "/Invalid Message Length"},
 	{0}};
 
 static const struct message bgp_notify_fsm_msg[] = {
@@ -487,8 +485,8 @@ const char *bgp_notify_subcode_str(char code, char subcode)
 	case BGP_NOTIFY_CEASE:
 		return lookup_msg(bgp_notify_cease_msg, subcode,
 				  "Unrecognized Error Subcode");
-	case BGP_NOTIFY_CAPABILITY_ERR:
-		return lookup_msg(bgp_notify_capability_msg, subcode,
+	case BGP_NOTIFY_ROUTE_REFRESH_ERR:
+		return lookup_msg(bgp_notify_route_refresh_msg, subcode,
 				  "Unrecognized Error Subcode");
 	}
 	return "";
diff --git a/bgpd/bgp_fsm.c b/bgpd/bgp_fsm.c
index a4d17cac40..cec4a9339a 100644
--- a/bgpd/bgp_fsm.c
+++ b/bgpd/bgp_fsm.c
@@ -469,6 +469,7 @@ void bgp_timer_set(struct peer *peer)
 		BGP_TIMER_OFF(peer->t_gr_restart);
 		BGP_TIMER_OFF(peer->t_gr_stale);
 		BGP_TIMER_OFF(peer->t_pmax_restart);
+		BGP_TIMER_OFF(peer->t_refresh_stalepath);
 	/* fallthru */
 	case Clearing:
 		BGP_TIMER_OFF(peer->t_start);
@@ -1283,6 +1284,16 @@ int bgp_stop(struct peer *peer)
 					peer->nsf[afi][safi] = 0;
 		}
 
+		/* Stop route-refresh stalepath timer */
+		if (peer->t_refresh_stalepath) {
+			BGP_TIMER_OFF(peer->t_refresh_stalepath);
+
+			if (bgp_debug_neighbor_events(peer))
+				zlog_debug(
+					"%s: route-refresh restart stalepath timer stopped",
+					peer->host);
+		}
+
 		/* If peer reset before receiving EOR, decrement EOR count and
 		 * cancel the selection deferral timer if there are no
 		 * pending EOR messages to be received
@@ -2066,14 +2077,16 @@ static int bgp_establish(struct peer *peer)
 			       PEER_CAP_ORF_PREFIX_SM_ADV)) {
 			if (CHECK_FLAG(peer->af_cap[afi][safi],
 				       PEER_CAP_ORF_PREFIX_RM_RCV))
-				bgp_route_refresh_send(peer, afi, safi,
-						       ORF_TYPE_PREFIX,
-						       REFRESH_IMMEDIATE, 0);
+				bgp_route_refresh_send(
+					peer, afi, safi, ORF_TYPE_PREFIX,
+					REFRESH_IMMEDIATE, 0,
+					BGP_ROUTE_REFRESH_NORMAL);
 			else if (CHECK_FLAG(peer->af_cap[afi][safi],
 					    PEER_CAP_ORF_PREFIX_RM_OLD_RCV))
-				bgp_route_refresh_send(peer, afi, safi,
-						       ORF_TYPE_PREFIX_OLD,
-						       REFRESH_IMMEDIATE, 0);
+				bgp_route_refresh_send(
+					peer, afi, safi, ORF_TYPE_PREFIX_OLD,
+					REFRESH_IMMEDIATE, 0,
+					BGP_ROUTE_REFRESH_NORMAL);
 		}
 	}
 
diff --git a/bgpd/bgp_open.c b/bgpd/bgp_open.c
index 6cfcb9cc3d..533518cf93 100644
--- a/bgpd/bgp_open.c
+++ b/bgpd/bgp_open.c
@@ -760,6 +760,7 @@ static const struct message capcode_str[] = {
 	{CAPABILITY_CODE_REFRESH_OLD, "Route Refresh (Old)"},
 	{CAPABILITY_CODE_ORF_OLD, "ORF (Old)"},
 	{CAPABILITY_CODE_FQDN, "FQDN"},
+	{CAPABILITY_CODE_ENHANCED_RR, "Enhanced Route Refresh"},
 	{0}};
 
 /* Minimum sizes for length field of each cap (so not inc. the header) */
@@ -776,6 +777,7 @@ static const size_t cap_minsizes[] = {
 		[CAPABILITY_CODE_REFRESH_OLD] = CAPABILITY_CODE_REFRESH_LEN,
 		[CAPABILITY_CODE_ORF_OLD] = CAPABILITY_CODE_ORF_LEN,
 		[CAPABILITY_CODE_FQDN] = CAPABILITY_CODE_MIN_FQDN_LEN,
+		[CAPABILITY_CODE_ENHANCED_RR] = CAPABILITY_CODE_ENHANCED_LEN,
 };
 
 /* value the capability must be a multiple of.
@@ -796,6 +798,7 @@ static const size_t cap_modsizes[] = {
 		[CAPABILITY_CODE_REFRESH_OLD] = 1,
 		[CAPABILITY_CODE_ORF_OLD] = 1,
 		[CAPABILITY_CODE_FQDN] = 1,
+		[CAPABILITY_CODE_ENHANCED_RR] = 1,
 };
 
 /**
@@ -863,6 +866,7 @@ static int bgp_capability_parse(struct peer *peer, size_t length,
 		case CAPABILITY_CODE_DYNAMIC_OLD:
 		case CAPABILITY_CODE_ENHE:
 		case CAPABILITY_CODE_FQDN:
+		case CAPABILITY_CODE_ENHANCED_RR:
 			/* Check length. */
 			if (caphdr.length < cap_minsizes[caphdr.code]) {
 				zlog_info(
@@ -913,10 +917,13 @@ static int bgp_capability_parse(struct peer *peer, size_t length,
 				ret = 0; /* Don't return error for this */
 			}
 		} break;
+		case CAPABILITY_CODE_ENHANCED_RR:
 		case CAPABILITY_CODE_REFRESH:
 		case CAPABILITY_CODE_REFRESH_OLD: {
 			/* BGP refresh capability */
-			if (caphdr.code == CAPABILITY_CODE_REFRESH_OLD)
+			if (caphdr.code == CAPABILITY_CODE_ENHANCED_RR)
+				SET_FLAG(peer->cap, PEER_CAP_ENHANCED_RR_RCV);
+			else if (caphdr.code == CAPABILITY_CODE_REFRESH_OLD)
 				SET_FLAG(peer->cap, PEER_CAP_REFRESH_OLD_RCV);
 			else
 				SET_FLAG(peer->cap, PEER_CAP_REFRESH_NEW_RCV);
@@ -1450,6 +1457,13 @@ void bgp_open_capability(struct stream *s, struct peer *peer)
 	stream_putc(s, CAPABILITY_CODE_REFRESH);
 	stream_putc(s, CAPABILITY_CODE_REFRESH_LEN);
 
+	/* Enhanced Route Refresh. */
+	SET_FLAG(peer->cap, PEER_CAP_ENHANCED_RR_ADV);
+	stream_putc(s, BGP_OPEN_OPT_CAP);
+	stream_putc(s, CAPABILITY_CODE_ENHANCED_LEN + 2);
+	stream_putc(s, CAPABILITY_CODE_ENHANCED_RR);
+	stream_putc(s, CAPABILITY_CODE_ENHANCED_LEN);
+
 	/* AS4 */
 	SET_FLAG(peer->cap, PEER_CAP_AS4_ADV);
 	stream_putc(s, BGP_OPEN_OPT_CAP);
diff --git a/bgpd/bgp_open.h b/bgpd/bgp_open.h
index 5250a68581..471ac05c7c 100644
--- a/bgpd/bgp_open.h
+++ b/bgpd/bgp_open.h
@@ -49,6 +49,7 @@ struct graceful_restart_af {
 #define CAPABILITY_CODE_DYNAMIC_OLD    66 /* Dynamic Capability, deprecated since 2003 */
 #define CAPABILITY_CODE_DYNAMIC        67 /* Dynamic Capability */
 #define CAPABILITY_CODE_ADDPATH        69 /* Addpath Capability */
+#define CAPABILITY_CODE_ENHANCED_RR    70 /* Enhanced Route Refresh capability */
 #define CAPABILITY_CODE_FQDN           73 /* Advertise hostname capability */
 #define CAPABILITY_CODE_ENHE            5 /* Extended Next Hop Encoding */
 #define CAPABILITY_CODE_REFRESH_OLD   128 /* Route Refresh Capability(cisco) */
@@ -63,6 +64,7 @@ struct graceful_restart_af {
 #define CAPABILITY_CODE_ADDPATH_LEN     4
 #define CAPABILITY_CODE_ENHE_LEN        6 /* NRLI AFI = 2, SAFI = 2, Nexthop AFI = 2 */
 #define CAPABILITY_CODE_MIN_FQDN_LEN    2
+#define CAPABILITY_CODE_ENHANCED_LEN    0
 #define CAPABILITY_CODE_ORF_LEN         5
 
 /* Cooperative Route Filtering Capability.  */
diff --git a/bgpd/bgp_packet.c b/bgpd/bgp_packet.c
index b2b9e04bc3..26be5c33db 100644
--- a/bgpd/bgp_packet.c
+++ b/bgpd/bgp_packet.c
@@ -444,13 +444,45 @@ int bgp_generate_updgrp_packets(struct thread *thread)
 			 * yet.
 			 */
 			if (!next_pkt || !next_pkt->buffer) {
-				/* Make sure we supress BGP UPDATES
-				 * for normal processing later again.
-				 */
-				if (!paf->t_announce_route)
+				if (!paf->t_announce_route) {
+					/* Make sure we supress BGP UPDATES
+					 * for normal processing later again.
+					 */
 					UNSET_FLAG(paf->subgroup->sflags,
 						   SUBGRP_STATUS_FORCE_UPDATES);
 
+					/* If route-refresh BoRR message was
+					 * already sent and we are done with
+					 * re-announcing tables for a decent
+					 * afi/safi, we ready to send
+					 * EoRR request.
+					 */
+					if (CHECK_FLAG(
+						    peer->af_sflags[afi][safi],
+						    PEER_STATUS_BORR_SEND)) {
+						bgp_route_refresh_send(
+							peer, afi, safi, 0, 0,
+							0,
+							BGP_ROUTE_REFRESH_EORR);
+
+						SET_FLAG(peer->af_sflags[afi]
+									[safi],
+							 PEER_STATUS_EORR_SEND);
+						UNSET_FLAG(
+							peer->af_sflags[afi]
+								       [safi],
+							PEER_STATUS_BORR_SEND);
+
+						if (bgp_debug_neighbor_events(
+							    peer))
+							zlog_debug(
+								"%s sending route-refresh (EoRR) for %s/%s",
+								peer->host,
+								afi2str(afi),
+								safi2str(safi));
+					}
+				}
+
 				if (CHECK_FLAG(peer->cap,
 					       PEER_CAP_RESTART_RCV)) {
 					if (!(PAF_SUBGRP(paf))->t_coalesce
@@ -816,7 +848,7 @@ void bgp_notify_send(struct peer *peer, uint8_t code, uint8_t sub_code)
  */
 void bgp_route_refresh_send(struct peer *peer, afi_t afi, safi_t safi,
 			    uint8_t orf_type, uint8_t when_to_refresh,
-			    int remove)
+			    int remove, uint8_t subtype)
 {
 	struct stream *s;
 	struct bgp_filter *filter;
@@ -842,7 +874,10 @@ void bgp_route_refresh_send(struct peer *peer, afi_t afi, safi_t safi,
 
 	/* Encode Route Refresh message. */
 	stream_putw(s, pkt_afi);
-	stream_putc(s, 0);
+	if (subtype)
+		stream_putc(s, subtype);
+	else
+		stream_putc(s, 0);
 	stream_putc(s, pkt_safi);
 
 	if (orf_type == ORF_TYPE_PREFIX || orf_type == ORF_TYPE_PREFIX_OLD)
@@ -1460,6 +1495,29 @@ static int bgp_keepalive_receive(struct peer *peer, bgp_size_t size)
 	return Receive_KEEPALIVE_message;
 }
 
+static int bgp_refresh_stalepath_timer_expire(struct thread *thread)
+{
+	struct peer_af *paf;
+
+	paf = THREAD_ARG(thread);
+
+	afi_t afi = paf->afi;
+	safi_t safi = paf->safi;
+	struct peer *peer = paf->peer;
+
+	peer->t_refresh_stalepath = NULL;
+
+	if (peer->nsf[afi][safi])
+		bgp_clear_stale_route(peer, afi, safi);
+
+	if (bgp_debug_neighbor_events(peer))
+		zlog_debug("%s: route-refresh (BoRR) timer for %s/%s expired",
+			   peer->host, afi2str(afi), safi2str(safi));
+
+	bgp_timer_set(peer);
+
+	return 0;
+}
 
 /**
  * Process BGP UPDATE message for peer.
@@ -1888,6 +1946,9 @@ static int bgp_route_refresh_receive(struct peer *peer, bgp_size_t size)
 	struct peer_af *paf;
 	struct update_group *updgrp;
 	struct peer *updgrp_peer;
+	uint8_t subtype;
+	bgp_size_t msg_length =
+		size - (BGP_MSG_ROUTE_REFRESH_MIN_SIZE - BGP_HEADER_SIZE);
 
 	/* If peer does not have the capability, send notification. */
 	if (!CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_ADV)) {
@@ -1915,7 +1976,7 @@ static int bgp_route_refresh_receive(struct peer *peer, bgp_size_t size)
 
 	/* Parse packet. */
 	pkt_afi = stream_getw(s);
-	(void)stream_getc(s);
+	subtype = stream_getc(s);
 	pkt_safi = stream_getc(s);
 
 	if (bgp_debug_update(peer, NULL, NULL, 0))
@@ -1938,8 +1999,34 @@ static int bgp_route_refresh_receive(struct peer *peer, bgp_size_t size)
 		uint8_t orf_type;
 		uint16_t orf_len;
 
-		if (size - (BGP_MSG_ROUTE_REFRESH_MIN_SIZE - BGP_HEADER_SIZE)
-		    < 5) {
+		if (subtype) {
+			/* If the length, excluding the fixed-size message
+			 * header, of the received ROUTE-REFRESH message with
+			 * Message Subtype 1 and 2 is not 4, then the BGP
+			 * speaker MUST send a NOTIFICATION message with the
+			 * Error Code of "ROUTE-REFRESH Message Error" and the
+			 * subcode of "Invalid Message Length".
+			 */
+			if (msg_length != 4) {
+				zlog_err(
+					"%s Enhanced Route Refresh message length error",
+					peer->host);
+				bgp_notify_send(
+					peer, BGP_NOTIFY_ROUTE_REFRESH_ERR,
+					BGP_NOTIFY_ROUTE_REFRESH_INVALID_MSG_LEN);
+			}
+
+			/* When the BGP speaker receives a ROUTE-REFRESH message
+			 * with a "Message Subtype" field other than 0, 1, or 2,
+			 * it MUST ignore the received ROUTE-REFRESH message.
+			 */
+			if (subtype > 2)
+				zlog_err(
+					"%s Enhanced Route Refresh invalid subtype",
+					peer->host);
+		}
+
+		if (msg_length < 5) {
 			zlog_info("%s ORF route refresh length error",
 				  peer->host);
 			bgp_notify_send(peer, BGP_NOTIFY_CEASE,
@@ -2139,6 +2226,120 @@ static int bgp_route_refresh_receive(struct peer *peer, bgp_size_t size)
 				   SUBGRP_STATUS_DEFAULT_ORIGINATE);
 	}
 
+	if (subtype == BGP_ROUTE_REFRESH_BORR) {
+		/* A BGP speaker that has received the Graceful Restart
+		 * Capability from its neighbor MUST ignore any BoRRs for
+		 * an <AFI, SAFI> from the neighbor before the speaker
+		 * receives the EoR for the given <AFI, SAFI> from the
+		 * neighbor.
+		 */
+		if (CHECK_FLAG(peer->cap, PEER_CAP_RESTART_RCV)
+		    && !CHECK_FLAG(peer->af_sflags[afi][safi],
+				   PEER_STATUS_EOR_RECEIVED)) {
+			if (bgp_debug_neighbor_events(peer))
+				zlog_debug(
+					"%s rcvd route-refresh (BoRR) for %s/%s before EoR",
+					peer->host, afi2str(afi),
+					safi2str(safi));
+			return BGP_PACKET_NOOP;
+		}
+
+		if (peer->t_refresh_stalepath) {
+			if (bgp_debug_neighbor_events(peer))
+				zlog_debug(
+					"%s rcvd route-refresh (BoRR) for %s/%s, whereas BoRR already received",
+					peer->host, afi2str(afi),
+					safi2str(safi));
+			return BGP_PACKET_NOOP;
+		}
+
+		SET_FLAG(peer->af_sflags[afi][safi], PEER_STATUS_BORR_RECEIVED);
+		UNSET_FLAG(peer->af_sflags[afi][safi],
+			   PEER_STATUS_EORR_RECEIVED);
+
+		/* When a BGP speaker receives a BoRR message from
+		 * a peer, it MUST mark all the routes with the given
+		 * Address Family Identifier and Subsequent Address
+		 * Family Identifier, <AFI, SAFI> [RFC2918], from
+		 * that peer as stale.
+		 */
+		if (peer_active_nego(peer)) {
+			SET_FLAG(peer->af_sflags[afi][safi],
+				 PEER_STATUS_ENHANCED_REFRESH);
+			bgp_set_stale_route(peer, afi, safi);
+		}
+
+		if (peer->status == Established)
+			thread_add_timer(bm->master,
+					 bgp_refresh_stalepath_timer_expire,
+					 paf, peer->bgp->stalepath_time,
+					 &peer->t_refresh_stalepath);
+
+		if (bgp_debug_neighbor_events(peer))
+			zlog_debug(
+				"%s rcvd route-refresh (BoRR) for %s/%s, triggering timer for %u seconds",
+				peer->host, afi2str(afi), safi2str(safi),
+				peer->bgp->stalepath_time);
+	} else if (subtype == BGP_ROUTE_REFRESH_EORR) {
+		if (!peer->t_refresh_stalepath) {
+			zlog_err(
+				"%s rcvd route-refresh (EoRR) for %s/%s, whereas no BoRR received",
+				peer->host, afi2str(afi), safi2str(safi));
+			return BGP_PACKET_NOOP;
+		}
+
+		BGP_TIMER_OFF(peer->t_refresh_stalepath);
+
+		SET_FLAG(peer->af_sflags[afi][safi], PEER_STATUS_EORR_RECEIVED);
+		UNSET_FLAG(peer->af_sflags[afi][safi],
+			   PEER_STATUS_BORR_RECEIVED);
+
+		if (bgp_debug_neighbor_events(peer))
+			zlog_debug(
+				"%s rcvd route-refresh (EoRR) for %s/%s, stopping BoRR timer",
+				peer->host, afi2str(afi), safi2str(safi));
+
+		if (peer->nsf[afi][safi])
+			bgp_clear_stale_route(peer, afi, safi);
+	} else {
+		/* In response to a "normal route refresh request" from the
+		 * peer, the speaker MUST send a BoRR message.
+		 */
+		if (CHECK_FLAG(peer->cap, PEER_CAP_ENHANCED_RR_RCV)) {
+			/* For a BGP speaker that supports the BGP Graceful
+			 * Restart, it MUST NOT send a BoRR for an <AFI, SAFI>
+			 * to a neighbor before it sends the EoR for the
+			 * <AFI, SAFI> to the neighbor.
+			 */
+			if (!CHECK_FLAG(peer->af_sflags[afi][safi],
+					PEER_STATUS_EOR_SEND)) {
+				if (bgp_debug_neighbor_events(peer))
+					zlog_debug(
+						"%s rcvd route-refresh (REQUEST) for %s/%s before EoR",
+						peer->host, afi2str(afi),
+						safi2str(safi));
+				return BGP_PACKET_NOOP;
+			}
+
+			bgp_route_refresh_send(peer, afi, safi, 0, 0, 0,
+					       BGP_ROUTE_REFRESH_BORR);
+
+			if (bgp_debug_neighbor_events(peer))
+				zlog_debug(
+					"%s sending route-refresh (BoRR) for %s/%s",
+					peer->host, afi2str(afi),
+					safi2str(safi));
+
+			/* Set flag Ready-To-Send to know when we can send EoRR
+			 * message.
+			 */
+			SET_FLAG(peer->af_sflags[afi][safi],
+				 PEER_STATUS_BORR_SEND);
+			UNSET_FLAG(peer->af_sflags[afi][safi],
+				   PEER_STATUS_EORR_SEND);
+		}
+	}
+
 	/* Perform route refreshment to the peer */
 	bgp_announce_route(peer, afi, safi);
 
diff --git a/bgpd/bgp_packet.h b/bgpd/bgp_packet.h
index e83f7d950c..525859a2da 100644
--- a/bgpd/bgp_packet.h
+++ b/bgpd/bgp_packet.h
@@ -62,8 +62,9 @@ extern void bgp_open_send(struct peer *);
 extern void bgp_notify_send(struct peer *, uint8_t, uint8_t);
 extern void bgp_notify_send_with_data(struct peer *, uint8_t, uint8_t,
 				      uint8_t *, size_t);
-extern void bgp_route_refresh_send(struct peer *, afi_t, safi_t, uint8_t,
-				   uint8_t, int);
+extern void bgp_route_refresh_send(struct peer *peer, afi_t afi, safi_t safi,
+				   uint8_t orf_type, uint8_t when_to_refresh,
+				   int remove, uint8_t subtype);
 extern void bgp_capability_send(struct peer *, afi_t, safi_t, int, int);
 
 extern int bgp_capability_receive(struct peer *, bgp_size_t);
diff --git a/bgpd/bgp_route.c b/bgpd/bgp_route.c
index 60ad8d20e9..7ffc6945b9 100644
--- a/bgpd/bgp_route.c
+++ b/bgpd/bgp_route.c
@@ -4595,8 +4595,10 @@ static wq_item_status bgp_clear_route_node(struct work_queue *wq, void *data)
 			continue;
 
 		/* graceful restart STALE flag set. */
-		if (CHECK_FLAG(peer->sflags, PEER_STATUS_NSF_WAIT)
-		    && peer->nsf[afi][safi]
+		if (((CHECK_FLAG(peer->sflags, PEER_STATUS_NSF_WAIT)
+		      && peer->nsf[afi][safi])
+		     || CHECK_FLAG(peer->af_sflags[afi][safi],
+				   PEER_STATUS_ENHANCED_REFRESH))
 		    && !CHECK_FLAG(pi->flags, BGP_PATH_STALE)
 		    && !CHECK_FLAG(pi->flags, BGP_PATH_UNUSEABLE))
 			bgp_path_info_set_flag(dest, pi, BGP_PATH_STALE);
@@ -4847,7 +4849,7 @@ void bgp_clear_stale_route(struct peer *peer, afi_t afi, safi_t safi)
 	struct bgp_path_info *pi;
 	struct bgp_table *table;
 
-	if (safi == SAFI_MPLS_VPN) {
+	if (safi == SAFI_MPLS_VPN || safi == SAFI_ENCAP || safi == SAFI_EVPN) {
 		for (dest = bgp_table_top(peer->bgp->rib[afi][safi]); dest;
 		     dest = bgp_route_next(dest)) {
 			struct bgp_dest *rm;
@@ -4886,6 +4888,81 @@ void bgp_clear_stale_route(struct peer *peer, afi_t afi, safi_t safi)
 	}
 }
 
+void bgp_set_stale_route(struct peer *peer, afi_t afi, safi_t safi)
+{
+	struct bgp_dest *dest, *ndest;
+	struct bgp_path_info *pi;
+	struct bgp_table *table;
+
+	if (safi == SAFI_MPLS_VPN || safi == SAFI_ENCAP || safi == SAFI_EVPN) {
+		for (dest = bgp_table_top(peer->bgp->rib[afi][safi]); dest;
+		     dest = bgp_route_next(dest)) {
+			table = bgp_dest_get_bgp_table_info(dest);
+			if (!table)
+				continue;
+
+			for (ndest = bgp_table_top(table); ndest;
+			     ndest = bgp_route_next(ndest)) {
+				for (pi = bgp_dest_get_bgp_path_info(ndest); pi;
+				     pi = pi->next) {
+					if (pi->peer != peer)
+						continue;
+
+					if ((CHECK_FLAG(
+						    peer->af_sflags[afi][safi],
+						    PEER_STATUS_ENHANCED_REFRESH))
+					    && !CHECK_FLAG(pi->flags,
+							   BGP_PATH_STALE)
+					    && !CHECK_FLAG(
+						       pi->flags,
+						       BGP_PATH_UNUSEABLE)) {
+						if (bgp_debug_neighbor_events(
+							    peer))
+							zlog_debug(
+								"%s: route-refresh for %s/%s, marking prefix %pFX as stale",
+								peer->host,
+								afi2str(afi),
+								safi2str(safi),
+								bgp_dest_get_prefix(
+									ndest));
+
+						bgp_path_info_set_flag(
+							ndest, pi,
+							BGP_PATH_STALE);
+					}
+				}
+			}
+		}
+	} else {
+		for (dest = bgp_table_top(peer->bgp->rib[afi][safi]); dest;
+		     dest = bgp_route_next(dest)) {
+			for (pi = bgp_dest_get_bgp_path_info(dest); pi;
+			     pi = pi->next) {
+				if (pi->peer != peer)
+					continue;
+
+				if ((CHECK_FLAG(peer->af_sflags[afi][safi],
+						PEER_STATUS_ENHANCED_REFRESH))
+				    && !CHECK_FLAG(pi->flags, BGP_PATH_STALE)
+				    && !CHECK_FLAG(pi->flags,
+						   BGP_PATH_UNUSEABLE)) {
+					if (bgp_debug_neighbor_events(peer))
+						zlog_debug(
+							"%s: route-refresh for %s/%s, marking prefix %pFX as stale",
+							peer->host,
+							afi2str(afi),
+							safi2str(safi),
+							bgp_dest_get_prefix(
+								dest));
+
+					bgp_path_info_set_flag(dest, pi,
+							       BGP_PATH_STALE);
+				}
+			}
+		}
+	}
+}
+
 bool bgp_outbound_policy_exists(struct peer *peer, struct bgp_filter *filter)
 {
 	if (peer->sort == BGP_PEER_IBGP)
diff --git a/bgpd/bgp_route.h b/bgpd/bgp_route.h
index ec08eb9c65..bdbf4743ab 100644
--- a/bgpd/bgp_route.h
+++ b/bgpd/bgp_route.h
@@ -575,6 +575,7 @@ extern void bgp_clear_route(struct peer *, afi_t, safi_t);
 extern void bgp_clear_route_all(struct peer *);
 extern void bgp_clear_adj_in(struct peer *, afi_t, safi_t);
 extern void bgp_clear_stale_route(struct peer *, afi_t, safi_t);
+extern void bgp_set_stale_route(struct peer *peer, afi_t afi, safi_t safi);
 extern bool bgp_outbound_policy_exists(struct peer *, struct bgp_filter *);
 extern bool bgp_inbound_policy_exists(struct peer *, struct bgp_filter *);
 
diff --git a/bgpd/bgp_routemap.c b/bgpd/bgp_routemap.c
index 637eaca397..0f4f26e3ee 100644
--- a/bgpd/bgp_routemap.c
+++ b/bgpd/bgp_routemap.c
@@ -3503,8 +3503,9 @@ static void bgp_route_map_process_peer(const char *rmap_name,
 					zlog_debug(
 						"Processing route_map %s update on peer %s (inbound, route-refresh)",
 						rmap_name, peer->host);
-				bgp_route_refresh_send(peer, afi, safi, 0, 0,
-						       0);
+				bgp_route_refresh_send(
+					peer, afi, safi, 0, 0, 0,
+					BGP_ROUTE_REFRESH_NORMAL);
 			}
 		}
 	}
diff --git a/bgpd/bgp_vty.c b/bgpd/bgp_vty.c
index 4cdd4d2e62..1fdcb2c219 100644
--- a/bgpd/bgp_vty.c
+++ b/bgpd/bgp_vty.c
@@ -13153,6 +13153,37 @@ static void bgp_show_peer(struct vty *vty, struct peer *p, bool use_json,
 							"received");
 				}
 
+				/* Enhanced Route Refresh */
+				if (CHECK_FLAG(p->cap, PEER_CAP_ENHANCED_RR_ADV)
+				    || CHECK_FLAG(p->cap,
+						  PEER_CAP_ENHANCED_RR_RCV)) {
+					if (CHECK_FLAG(p->cap,
+						       PEER_CAP_ENHANCED_RR_ADV)
+					    && CHECK_FLAG(
+						    p->cap,
+						    PEER_CAP_ENHANCED_RR_RCV))
+						json_object_string_add(
+							json_cap,
+							"enhancedRouteRefresh",
+							"advertisedAndReceived");
+					else if (
+						CHECK_FLAG(
+							p->cap,
+							PEER_CAP_ENHANCED_RR_ADV))
+						json_object_string_add(
+							json_cap,
+							"enhancedRouteRefresh",
+							"advertised");
+					else if (
+						CHECK_FLAG(
+							p->cap,
+							PEER_CAP_ENHANCED_RR_RCV))
+						json_object_string_add(
+							json_cap,
+							"enhancedRouteRefresh",
+							"received");
+				}
+
 				/* Multiprotocol Extensions */
 				json_object *json_multi = NULL;
 				json_multi = json_object_new_object();
@@ -13525,6 +13556,28 @@ static void bgp_show_peer(struct vty *vty, struct peer *p, bool use_json,
 					vty_out(vty, "\n");
 				}
 
+				/* Enhanced Route Refresh */
+				if (CHECK_FLAG(p->cap, PEER_CAP_ENHANCED_RR_ADV)
+				    || CHECK_FLAG(p->cap,
+						  PEER_CAP_ENHANCED_RR_RCV)) {
+					vty_out(vty,
+						"    Enhanced Route Refresh:");
+					if (CHECK_FLAG(
+						    p->cap,
+						    PEER_CAP_ENHANCED_RR_ADV))
+						vty_out(vty, " advertised");
+					if (CHECK_FLAG(
+						    p->cap,
+						    PEER_CAP_ENHANCED_RR_RCV))
+						vty_out(vty, " %sreceived",
+							CHECK_FLAG(
+								p->cap,
+								PEER_CAP_REFRESH_ADV)
+								? "and "
+								: "");
+					vty_out(vty, "\n");
+				}
+
 				/* Multiprotocol Extensions */
 				FOREACH_AFI_SAFI (afi, safi)
 					if (p->afc_adv[afi][safi]
diff --git a/bgpd/bgpd.c b/bgpd/bgpd.c
index b6afa391f3..3ae9cd59b6 100644
--- a/bgpd/bgpd.c
+++ b/bgpd/bgpd.c
@@ -4010,7 +4010,8 @@ void peer_change_action(struct peer *peer, afi_t afi, safi_t safi,
 	} else if (type == peer_change_reset_in) {
 		if (CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_OLD_RCV)
 		    || CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_NEW_RCV))
-			bgp_route_refresh_send(peer, afi, safi, 0, 0, 0);
+			bgp_route_refresh_send(peer, afi, safi, 0, 0, 0,
+					       BGP_ROUTE_REFRESH_NORMAL);
 		else {
 			if ((peer->doppelganger)
 			    && (peer->doppelganger->status != Deleted)
@@ -5083,7 +5084,8 @@ static void peer_on_policy_change(struct peer *peer, afi_t afi, safi_t safi,
 			bgp_soft_reconfig_in(peer, afi, safi);
 		else if (CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_OLD_RCV)
 			 || CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_NEW_RCV))
-			bgp_route_refresh_send(peer, afi, safi, 0, 0, 0);
+			bgp_route_refresh_send(peer, afi, safi, 0, 0, 0,
+					       BGP_ROUTE_REFRESH_NORMAL);
 	}
 }
 
@@ -7331,19 +7333,23 @@ int peer_clear_soft(struct peer *peer, afi_t afi, safi_t safi,
 					       PEER_STATUS_ORF_PREFIX_SEND))
 					bgp_route_refresh_send(
 						peer, afi, safi, prefix_type,
-						REFRESH_DEFER, 1);
-				bgp_route_refresh_send(peer, afi, safi,
-						       prefix_type,
-						       REFRESH_IMMEDIATE, 0);
+						REFRESH_DEFER, 1,
+						BGP_ROUTE_REFRESH_NORMAL);
+				bgp_route_refresh_send(
+					peer, afi, safi, prefix_type,
+					REFRESH_IMMEDIATE, 0,
+					BGP_ROUTE_REFRESH_NORMAL);
 			} else {
 				if (CHECK_FLAG(peer->af_sflags[afi][safi],
 					       PEER_STATUS_ORF_PREFIX_SEND))
 					bgp_route_refresh_send(
 						peer, afi, safi, prefix_type,
-						REFRESH_IMMEDIATE, 1);
+						REFRESH_IMMEDIATE, 1,
+						BGP_ROUTE_REFRESH_NORMAL);
 				else
-					bgp_route_refresh_send(peer, afi, safi,
-							       0, 0, 0);
+					bgp_route_refresh_send(
+						peer, afi, safi, 0, 0, 0,
+						BGP_ROUTE_REFRESH_NORMAL);
 			}
 			return 0;
 		}
@@ -7362,8 +7368,9 @@ int peer_clear_soft(struct peer *peer, afi_t afi, safi_t safi,
 			   message to the peer. */
 			if (CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_OLD_RCV)
 			    || CHECK_FLAG(peer->cap, PEER_CAP_REFRESH_NEW_RCV))
-				bgp_route_refresh_send(peer, afi, safi, 0, 0,
-						       0);
+				bgp_route_refresh_send(
+					peer, afi, safi, 0, 0, 0,
+					BGP_ROUTE_REFRESH_NORMAL);
 			else
 				return BGP_ERR_SOFT_RECONFIG_UNCONFIGURED;
 		}
diff --git a/bgpd/bgpd.h b/bgpd/bgpd.h
index 16210bed15..85188dd841 100644
--- a/bgpd/bgpd.h
+++ b/bgpd/bgpd.h
@@ -1063,6 +1063,8 @@ struct peer {
 #define PEER_CAP_ENHE_RCV                   (1U << 14) /* Extended nexthop received */
 #define PEER_CAP_HOSTNAME_ADV               (1U << 15) /* hostname advertised */
 #define PEER_CAP_HOSTNAME_RCV               (1U << 16) /* hostname received */
+#define PEER_CAP_ENHANCED_RR_ADV (1U << 17) /* enhanced rr advertised */
+#define PEER_CAP_ENHANCED_RR_RCV (1U << 18) /* enhanced rr received */
 
 	/* Capability flags (reset in bgp_stop) */
 	uint32_t af_cap[AFI_MAX][SAFI_MAX];
@@ -1264,6 +1266,11 @@ struct peer {
 #define PEER_STATUS_PREFIX_LIMIT      (1U << 3) /* exceed prefix-limit */
 #define PEER_STATUS_EOR_SEND          (1U << 4) /* end-of-rib send to peer */
 #define PEER_STATUS_EOR_RECEIVED      (1U << 5) /* end-of-rib received from peer */
+#define PEER_STATUS_ENHANCED_REFRESH (1U << 6) /* Enhanced Route Refresh */
+#define PEER_STATUS_BORR_SEND (1U << 7) /* BoRR send to peer */
+#define PEER_STATUS_BORR_RECEIVED (1U << 8) /* BoRR received from peer */
+#define PEER_STATUS_EORR_SEND (1U << 9) /* EoRR send to peer */
+#define PEER_STATUS_EORR_RECEIVED (1U << 10) /* EoRR received from peer */
 
 	/* Configured timer values. */
 	_Atomic uint32_t holdtime;
@@ -1297,6 +1304,7 @@ struct peer {
 	struct thread *t_gr_stale;
 	struct thread *t_generate_updgrp_packets;
 	struct thread *t_process_packet;
+	struct thread *t_refresh_stalepath;
 
 	/* Thread flags. */
 	_Atomic uint32_t thread_flags;
@@ -1621,7 +1629,7 @@ struct bgp_nlri {
 #define BGP_NOTIFY_HOLD_ERR                      4
 #define BGP_NOTIFY_FSM_ERR                       5
 #define BGP_NOTIFY_CEASE                         6
-#define BGP_NOTIFY_CAPABILITY_ERR                7
+#define BGP_NOTIFY_ROUTE_REFRESH_ERR             7
 
 /* Subcodes for BGP Finite State Machine Error */
 #define BGP_NOTIFY_FSM_ERR_SUBCODE_UNSPECIFIC  0
@@ -1669,10 +1677,13 @@ struct bgp_nlri {
 #define BGP_NOTIFY_CEASE_COLLISION_RESOLUTION    7
 #define BGP_NOTIFY_CEASE_OUT_OF_RESOURCE         8
 
-/* BGP_NOTIFY_CAPABILITY_ERR sub codes (draft-ietf-idr-dynamic-cap-02). */
-#define BGP_NOTIFY_CAPABILITY_INVALID_ACTION     1
-#define BGP_NOTIFY_CAPABILITY_INVALID_LENGTH     2
-#define BGP_NOTIFY_CAPABILITY_MALFORMED_CODE     3
+/* BGP_NOTIFY_ROUTE_REFRESH_ERR sub codes (RFC 7313). */
+#define BGP_NOTIFY_ROUTE_REFRESH_INVALID_MSG_LEN 1
+
+/* BGP route refresh optional subtypes. */
+#define BGP_ROUTE_REFRESH_NORMAL 0
+#define BGP_ROUTE_REFRESH_BORR 1
+#define BGP_ROUTE_REFRESH_EORR 2
 
 /* BGP timers default value.  */
 #define BGP_INIT_START_TIMER                     1