zebra: proper refcounting for rmac/nh entries

Signed-off-by: Mitesh Kanjariya <mitesh@cumulusnetworks.comy>

10 files changed, 129 insertions, 49 deletions

diff --git a/zebra/connected.c b/zebra/connected.c
index 18dc6a970b..b5661caeda 100644
--- a/zebra/connected.c
+++ b/zebra/connected.c
@@ -396,10 +396,10 @@ void connected_down(struct interface *ifp, struct connected *ifc)
 	 * head.
 	 */
 	rib_delete(afi, SAFI_UNICAST, ifp->vrf_id, ZEBRA_ROUTE_CONNECT, 0, 0,
-		   &p, NULL, &nh, 0, 0, false);
+		   &p, NULL, &nh, 0, 0, false, NULL);
 
 	rib_delete(afi, SAFI_MULTICAST, ifp->vrf_id, ZEBRA_ROUTE_CONNECT, 0,
-		   0, &p, NULL, &nh, 0, 0, false);
+		   0, &p, NULL, &nh, 0, 0, false, NULL);
 
 	if (IS_ZEBRA_DEBUG_RIB_DETAILED) {
 		char buf[PREFIX_STRLEN];
diff --git a/zebra/kernel_socket.c b/zebra/kernel_socket.c
index 89c933f90f..ce4226e616 100644
--- a/zebra/kernel_socket.c
+++ b/zebra/kernel_socket.c
@@ -1039,7 +1039,7 @@ void rtm_read(struct rt_msghdr *rtm)
 		if (rtm->rtm_type == RTM_CHANGE)
 			rib_delete(AFI_IP, SAFI_UNICAST, VRF_DEFAULT,
 				   ZEBRA_ROUTE_KERNEL, 0, zebra_flags, &p, NULL,
-				   NULL, 0, 0, true);
+				   NULL, 0, 0, true, NULL);
 
 		if (!nh.type) {
 			nh.type = NEXTHOP_TYPE_IPV4;
@@ -1054,7 +1054,7 @@ void rtm_read(struct rt_msghdr *rtm)
 		else
 			rib_delete(AFI_IP, SAFI_UNICAST, VRF_DEFAULT,
 				   ZEBRA_ROUTE_KERNEL, 0, zebra_flags, &p, NULL,
-				   &nh, 0, 0, true);
+				   &nh, 0, 0, true, NULL);
 	}
 	if (dest.sa.sa_family == AF_INET6) {
 		/* One day we might have a debug section here like one in the
@@ -1085,7 +1085,7 @@ void rtm_read(struct rt_msghdr *rtm)
 		if (rtm->rtm_type == RTM_CHANGE)
 			rib_delete(AFI_IP6, SAFI_UNICAST, VRF_DEFAULT,
 				   ZEBRA_ROUTE_KERNEL, 0, zebra_flags, &p, NULL,
-				   NULL, 0, 0, true);
+				   NULL, 0, 0, true, NULL);
 
 		if (!nh.type) {
 			nh.type = ifindex ? NEXTHOP_TYPE_IPV6_IFINDEX
@@ -1102,7 +1102,7 @@ void rtm_read(struct rt_msghdr *rtm)
 		else
 			rib_delete(AFI_IP6, SAFI_UNICAST, VRF_DEFAULT,
 				   ZEBRA_ROUTE_KERNEL, 0, zebra_flags, &p, NULL,
-				   &nh, 0, 0, true);
+				   &nh, 0, 0, true, NULL);
 	}
 }
 
diff --git a/zebra/redistribute.c b/zebra/redistribute.c
index cbf77765a3..3c6a2a7daf 100644
--- a/zebra/redistribute.c
+++ b/zebra/redistribute.c
@@ -570,7 +570,7 @@ int zebra_del_import_table_entry(struct route_node *rn, struct route_entry *re)
 
 	rib_delete(afi, SAFI_UNICAST, re->vrf_id, ZEBRA_ROUTE_TABLE,
 		   re->table, re->flags, &p, NULL, re->nexthop,
-		   zebrad.rtm_table_default, re->metric, false);
+		   zebrad.rtm_table_default, re->metric, false, NULL);
 
 	return 0;
 }
diff --git a/zebra/rib.h b/zebra/rib.h
index 4b82e8d8d5..393322586b 100644
--- a/zebra/rib.h
+++ b/zebra/rib.h
@@ -304,7 +304,8 @@ extern int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *,
 extern void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 		       u_short instance, int flags, struct prefix *p,
 		       struct prefix_ipv6 *src_p, const struct nexthop *nh,
-		       u_int32_t table_id, u_int32_t metric, bool fromkernel);
+		       u_int32_t table_id, u_int32_t metric, bool fromkernel,
+		       struct ethaddr *rmac);
 
 extern struct route_entry *rib_match(afi_t afi, safi_t safi, vrf_id_t,
 				     union g_addr *,
diff --git a/zebra/rt_netlink.c b/zebra/rt_netlink.c
index cbe736e00c..c5ca57e5b4 100644
--- a/zebra/rt_netlink.c
+++ b/zebra/rt_netlink.c
@@ -535,13 +535,13 @@ static int netlink_route_change_read_unicast(struct sockaddr_nl *snl,
 				memcpy(&nh.gate, gate, sz);
 			rib_delete(afi, SAFI_UNICAST, vrf_id,
 				   proto, 0, flags, &p, NULL, &nh,
-				   table, metric, true);
+				   table, metric, true, NULL);
 		} else {
 			/* XXX: need to compare the entire list of nexthops
 			 * here for NLM_F_APPEND stupidity */
 			rib_delete(afi, SAFI_UNICAST, vrf_id,
 				   proto, 0, flags, &p, NULL, NULL,
-				   table, metric, true);
+				   table, metric, true, NULL);
 		}
 	}
 
diff --git a/zebra/zebra_rib.c b/zebra/zebra_rib.c
index ed1a4953bf..804c65023c 100644
--- a/zebra/zebra_rib.c
+++ b/zebra/zebra_rib.c
@@ -50,6 +50,7 @@
 #include "zebra/zebra_rnh.h"
 #include "zebra/interface.h"
 #include "zebra/connected.h"
+#include "zebra/zebra_vxlan.h"
 
 DEFINE_HOOK(rib_update, (struct route_node * rn, const char *reason),
 	    (rn, reason))
@@ -2319,7 +2320,8 @@ int rib_add_multipath(afi_t afi, safi_t safi, struct prefix *p,
 void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 		u_short instance, int flags, struct prefix *p,
 		struct prefix_ipv6 *src_p, const struct nexthop *nh,
-		u_int32_t table_id, u_int32_t metric, bool fromkernel)
+		u_int32_t table_id, u_int32_t metric, bool fromkernel,
+		struct ethaddr *rmac)
 {
 	struct route_table *table;
 	struct route_node *rn;
@@ -2472,6 +2474,22 @@ void rib_delete(afi_t afi, safi_t safi, vrf_id_t vrf_id, int type,
 
 			return;
 		}
+
+		if (CHECK_FLAG(flags, ZEBRA_FLAG_EVPN_TYPE2_ROUTE)) {
+			struct nexthop *tmp_nh;
+
+			for (ALL_NEXTHOPS(re->nexthop, tmp_nh)) {
+				struct ipaddr vtep_ip;
+
+				memset(&vtep_ip, 0, sizeof(struct ipaddr));
+				vtep_ip.ipa_type = IPADDR_V4;
+				memcpy(&(vtep_ip.ipaddr_v4),
+				       &(tmp_nh->gate.ipv4),
+				       sizeof(struct in_addr));
+				zebra_vxlan_evpn_vrf_route_del(re->vrf_id, rmac,
+							       &vtep_ip, p);
+			}
+		}
 		rib_delnode(rn, same);
 	}
 
diff --git a/zebra/zebra_vxlan.c b/zebra/zebra_vxlan.c
index fa31a702b4..1c57374c69 100644
--- a/zebra/zebra_vxlan.c
+++ b/zebra/zebra_vxlan.c
@@ -48,6 +48,7 @@
 #include "zebra/zebra_l2.h"
 #include "lib/json.h"
 
+DEFINE_MTYPE_STATIC(ZEBRA, HOST_PREFIX, "host prefix");
 DEFINE_MTYPE_STATIC(ZEBRA, ZVNI, "VNI hash");
 DEFINE_MTYPE_STATIC(ZEBRA, ZL3VNI, "L3 VNI hash");
 DEFINE_MTYPE_STATIC(ZEBRA, ZVNI_VTEP, "VNI remote VTEP");
@@ -666,14 +667,14 @@ static void zl3vni_print_nh_hash(struct hash_backet *backet,
 		vty_out(vty, "%-15s %-17s %6d\n",
 			ipaddr2str(&(n->ip), buf2, sizeof(buf2)),
 			prefix_mac2str(&n->emac, buf1, sizeof(buf1)),
-			n->nh_refcnt);
+			listcount(n->host_list));
 	} else {
 		json_object_string_add(json_nh, "vtep-ip",
 				       inet_ntoa(n->r_vtep_ip));
 		json_object_string_add(json_nh, "rmac",
 				       prefix_mac2str(&n->emac, buf1,
 						      sizeof(buf1)));
-		json_object_int_add(json_nh, "refCnt", n->nh_refcnt);
+		json_object_int_add(json_nh, "refCnt", listcount(n->host_list));
 	}
 }
 
@@ -806,14 +807,15 @@ static void zl3vni_print_rmac_hash(struct hash_backet *backet,
 		vty_out(vty, "%-17s %-21s %-6d\n",
 			prefix_mac2str(&zrmac->macaddr, buf, sizeof(buf)),
 					inet_ntoa(zrmac->fwd_info.r_vtep_ip),
-					zrmac->rmac_refcnt);
+					listcount(zrmac->host_list));
 	} else {
 		json_object_string_add(json_rmac, "rmac",
 				       prefix_mac2str(&zrmac->macaddr, buf,
 						      sizeof(buf)));
 		json_object_string_add(json_rmac, "vtep-ip",
 				       inet_ntoa(zrmac->fwd_info.r_vtep_ip));
-		json_object_int_add(json_rmac, "refcnt", zrmac->rmac_refcnt);
+		json_object_int_add(json_rmac, "refcnt",
+				    listcount(zrmac->host_list));
 		json_object_array_add(json, json_rmac);
 	}
 }
@@ -2756,6 +2758,47 @@ static void zvni_cleanup_all(struct hash_backet *backet, void *zvrf)
 	zvni_del(zvni);
 }
 
+static int is_host_present_in_host_list(struct list *list,
+					struct prefix *host)
+{
+	struct listnode *node = NULL;
+	struct prefix *p = NULL;
+
+	for (ALL_LIST_ELEMENTS_RO(list, node, p)) {
+		if (prefix_same(p, host))
+			return 1;
+	}
+	return 0;
+}
+
+static void host_list_add_host(struct list *list,
+			       struct prefix *host)
+{
+	struct prefix *p = NULL;
+
+	p = XCALLOC(MTYPE_HOST_PREFIX, sizeof(struct prefix));
+	memcpy(p, host, sizeof(struct prefix));
+
+	listnode_add_sort(list, p);
+}
+
+static void host_list_delete_host(struct list *list,
+				  struct prefix *host)
+{
+	struct listnode *node = NULL, *nnode = NULL, *node_to_del = NULL;
+	struct prefix *p = NULL;
+
+	for (ALL_LIST_ELEMENTS(list, node, nnode, p)) {
+		if (prefix_same(p, host)) {
+			XFREE(MTYPE_HOST_PREFIX, p);
+			node_to_del = node;
+		}
+	}
+
+	if (node_to_del)
+		list_delete_node(list, node_to_del);
+}
+
 /*
  * Look up MAC hash entry.
  */
@@ -2800,8 +2843,8 @@ static zebra_mac_t *zl3vni_rmac_add(zebra_l3vni_t *zl3vni,
 	zrmac = hash_get(zl3vni->rmac_table, &tmp_rmac, zl3vni_rmac_alloc);
 	assert(zrmac);
 
-	zrmac->neigh_list = list_new();
-	zrmac->neigh_list->cmp = (int (*)(void *, void *))neigh_cmp;
+	zrmac->host_list = list_new();
+	zrmac->host_list->cmp = (int (*)(void *, void *))prefix_cmp;
 
 	SET_FLAG(zrmac->flags, ZEBRA_MAC_REMOTE);
 	SET_FLAG(zrmac->flags, ZEBRA_MAC_REMOTE_RMAC);
@@ -2882,9 +2925,10 @@ static int zl3vni_rmac_uninstall(zebra_l3vni_t *zl3vni,
 }
 
 /* handle rmac add */
-static int zebra_vxlan_l3vni_remote_rmac_add(zebra_l3vni_t *zl3vni,
-					     struct ethaddr *rmac,
-					     struct ipaddr *vtep_ip)
+static int zl3vni_remote_rmac_add(zebra_l3vni_t *zl3vni,
+				  struct ethaddr *rmac,
+				  struct ipaddr *vtep_ip,
+				  struct prefix *host_prefix)
 {
 	char buf[ETHER_ADDR_STRLEN];
 	char buf1[INET6_ADDRSTRLEN];
@@ -2909,14 +2953,17 @@ static int zebra_vxlan_l3vni_remote_rmac_add(zebra_l3vni_t *zl3vni,
 		/* install rmac in kernel */
 		zl3vni_rmac_install(zl3vni, zrmac);
 	}
-	zrmac->rmac_refcnt++;
+
+	if (!is_host_present_in_host_list(zrmac->host_list, host_prefix))
+		host_list_add_host(zrmac->host_list, host_prefix);
 	return 0;
 }
 
 
 /* handle rmac delete */
-static int zebra_vxlan_l3vni_remote_rmac_del(zebra_l3vni_t *zl3vni,
-					     struct ethaddr *rmac)
+static int zl3vni_remote_rmac_del(zebra_l3vni_t *zl3vni,
+				  struct ethaddr *rmac,
+				  struct prefix *host_prefix)
 {
 	zebra_mac_t *zrmac = NULL;
 
@@ -2924,8 +2971,8 @@ static int zebra_vxlan_l3vni_remote_rmac_del(zebra_l3vni_t *zl3vni,
 	if (!zrmac)
 		return -1;
 
-	zrmac->rmac_refcnt--;
-	if (!zrmac->rmac_refcnt) {
+	host_list_delete_host(zrmac->host_list, host_prefix);
+	if (list_isempty(zrmac->host_list)) {
 
 		/* uninstall from kernel */
 		zl3vni_rmac_uninstall(zl3vni, zrmac);
@@ -2982,6 +3029,9 @@ static zebra_neigh_t *zl3vni_nh_add(zebra_l3vni_t *zl3vni,
 	n = hash_get(zl3vni->nh_table, &tmp_n, zl3vni_nh_alloc);
 	assert(n);
 
+	n->host_list = list_new();
+	n->host_list->cmp = (int (*)(void *, void *))prefix_cmp;
+
 	memcpy(&n->emac, mac, ETH_ALEN);
 	SET_FLAG(n->flags, ZEBRA_NEIGH_REMOTE);
 	SET_FLAG(n->flags, ZEBRA_NEIGH_REMOTE_NH);
@@ -3037,9 +3087,10 @@ static int zl3vni_nh_uninstall(zebra_l3vni_t *zl3vni,
 }
 
 /* add remote vtep as a neigh entry */
-static int zebra_vxlan_l3vni_remote_nh_add(zebra_l3vni_t *zl3vni,
-			     struct ipaddr *vtep_ip,
-			     struct ethaddr *rmac)
+static int zl3vni_remote_nh_add(zebra_l3vni_t *zl3vni,
+				struct ipaddr *vtep_ip,
+				struct ethaddr *rmac,
+				struct prefix *host_prefix)
 {
 	char buf[ETHER_ADDR_STRLEN];
 	char buf1[INET6_ADDRSTRLEN];
@@ -3063,13 +3114,17 @@ static int zebra_vxlan_l3vni_remote_nh_add(zebra_l3vni_t *zl3vni,
 		/* install the nh neigh in kernel */
 		zl3vni_nh_install(zl3vni, nh);
 	}
-	nh->nh_refcnt++;
+
+	if (!is_host_present_in_host_list(nh->host_list, host_prefix))
+		host_list_add_host(nh->host_list, host_prefix);
+
 	return 0;
 }
 
 /* handle nh neigh delete */
-static int zebra_vxlan_l3vni_remote_nh_del(zebra_l3vni_t *zl3vni,
-			     struct ipaddr *vtep_ip)
+static int zl3vni_remote_nh_del(zebra_l3vni_t *zl3vni,
+				struct ipaddr *vtep_ip,
+				struct prefix *host_prefix)
 {
 	zebra_neigh_t *nh = NULL;
 
@@ -3077,8 +3132,8 @@ static int zebra_vxlan_l3vni_remote_nh_del(zebra_l3vni_t *zl3vni,
 	if (!nh)
 		return -1;
 
-	nh->nh_refcnt--;
-	if (!nh->nh_refcnt) {
+	host_list_delete_host(nh->host_list, host_prefix);
+	if (list_isempty(nh->host_list)) {
 
 		/* uninstall from kernel */
 		zl3vni_nh_uninstall(zl3vni, nh);
@@ -3533,7 +3588,8 @@ static int zebra_vxlan_handle_vni_transition(struct zebra_vrf *zvrf,
 /* handle evpn route in vrf table */
 void zebra_vxlan_evpn_vrf_route_add(vrf_id_t vrf_id,
 				   struct ethaddr *rmac,
-				   struct ipaddr *ip)
+				   struct ipaddr *vtep_ip,
+				   struct prefix *host_prefix)
 {
 	zebra_l3vni_t *zl3vni = NULL;
 
@@ -3542,16 +3598,17 @@ void zebra_vxlan_evpn_vrf_route_add(vrf_id_t vrf_id,
 		return;
 
 	/* add the next hop neighbor */
-	zebra_vxlan_l3vni_remote_nh_add(zl3vni, ip, rmac);
+	zl3vni_remote_nh_add(zl3vni, vtep_ip, rmac, host_prefix);
 
 	/* add the rmac */
-	zebra_vxlan_l3vni_remote_rmac_add(zl3vni, rmac, ip);
+	zl3vni_remote_rmac_add(zl3vni, rmac, vtep_ip, host_prefix);
 }
 
 /* handle evpn vrf route delete */
 void zebra_vxlan_evpn_vrf_route_del(vrf_id_t vrf_id,
 				   struct ethaddr *rmac,
-				   struct ipaddr *ip)
+				   struct ipaddr *vtep_ip,
+				   struct prefix *host_prefix)
 {
 	zebra_l3vni_t *zl3vni = NULL;
 
@@ -3560,10 +3617,10 @@ void zebra_vxlan_evpn_vrf_route_del(vrf_id_t vrf_id,
 		return;
 
 	/* delete the next hop entry */
-	zebra_vxlan_l3vni_remote_nh_del(zl3vni, ip);
+	zl3vni_remote_nh_del(zl3vni, vtep_ip, host_prefix);
 
 	/* delete the rmac entry */
-	zebra_vxlan_l3vni_remote_rmac_del(zl3vni, rmac);
+	zl3vni_remote_rmac_del(zl3vni, rmac, host_prefix);
 }
 
 
diff --git a/zebra/zebra_vxlan.h b/zebra/zebra_vxlan.h
index 6022901277..6afca1c73a 100644
--- a/zebra/zebra_vxlan.h
+++ b/zebra/zebra_vxlan.h
@@ -147,9 +147,11 @@ extern void zebra_vxlan_ns_init(struct zebra_ns *zns);
 extern void zebra_vxlan_ns_disable(struct zebra_ns *zns);
 extern void zebra_vxlan_evpn_vrf_route_add(vrf_id_t vrf_id,
 					   struct ethaddr *rmac,
-					   struct ipaddr *ip);
+					   struct ipaddr *ip,
+					   struct prefix *host_prefix);
 extern void zebra_vxlan_evpn_vrf_route_del(vrf_id_t vrf_id,
 					   struct ethaddr *rmac,
-					   struct ipaddr *ip);
+					   struct ipaddr *vtep_ip,
+					   struct prefix *host_prefix);
 
 #endif /* _ZEBRA_VXLAN_H */
diff --git a/zebra/zebra_vxlan_private.h b/zebra/zebra_vxlan_private.h
index c7bb7668ae..bde70ba25a 100644
--- a/zebra/zebra_vxlan_private.h
+++ b/zebra/zebra_vxlan_private.h
@@ -242,8 +242,8 @@ struct zebra_mac_t_ {
 	/* List of neigh associated with this mac */
 	struct list *neigh_list;
 
-	/* Refcnt of number of mac-ips */
-	u_int32_t rmac_refcnt;
+	/* list of hosts pointing to this remote RMAC */
+	struct list *host_list;
 };
 
 /*
@@ -315,8 +315,8 @@ struct zebra_neigh_t_ {
 	/* Remote VTEP IP - applicable only for remote neighbors. */
 	struct in_addr r_vtep_ip;
 
-	/* refcnt of remote mac-ip referring to a NH neigh */
-	u_int32_t nh_refcnt;
+	/* list of hosts pointing to this remote NH entry */
+	struct list *host_list;
 };
 
 /*
diff --git a/zebra/zserv.c b/zebra/zserv.c
index db09941944..7f3d7c7fcd 100644
--- a/zebra/zserv.c
+++ b/zebra/zserv.c
@@ -1192,7 +1192,8 @@ static int zread_route_add(struct zserv *client, u_short length,
 					zebra_vxlan_evpn_vrf_route_add(
 								vrf_id,
 								&api.rmac,
-								&vtep_ip);
+								&vtep_ip,
+								&api.prefix);
 				}
 				break;
 			}
@@ -1297,7 +1298,7 @@ static int zread_route_del(struct zserv *client, u_short length,
 
 	rib_delete(afi, api.safi, zvrf_id(zvrf), api.type, api.instance,
 		   api.flags, &api.prefix, src_p, NULL, zvrf->table_id,
-		   api.metric, false);
+		   api.metric, false, &api.rmac);
 
 	/* Stats */
 	switch (api.prefix.family) {
@@ -1498,7 +1499,7 @@ static int zread_ipv4_delete(struct zserv *client, u_short length,
 	table_id = zvrf->table_id;
 
 	rib_delete(AFI_IP, api.safi, zvrf_id(zvrf), api.type, api.instance,
-		   api.flags, &p, NULL, NULL, table_id, 0, false);
+		   api.flags, &p, NULL, NULL, table_id, 0, false, NULL);
 	client->v4_route_del_cnt++;
 
 stream_failure:
@@ -1916,7 +1917,8 @@ static int zread_ipv6_delete(struct zserv *client, u_short length,
 		src_pp = NULL;
 
 	rib_delete(AFI_IP6, api.safi, zvrf_id(zvrf), api.type, api.instance,
-		   api.flags, &p, src_pp, NULL, client->rtm_table, 0, false);
+		   api.flags, &p, src_pp, NULL, client->rtm_table, 0, false,
+		   NULL);
 
 	client->v6_route_del_cnt++;