bgpd: Use L3NHGs for symmetric IRB host routes

Two L3 next groups are installed per-VRF per-ES for v4 and v6. These
NHGs are used as an indirect destination for symmetric IRB host routes.

Using L3NHGs allows for efficient failover of an ES (similar to the
use of L2NHGs) i.e. when an ES goes down the number of dataplane
updates are limited to 2xN (where N is the number of tenant VRFs
associated with the ES) instead of updating all host-routes behind the
ES.

Signed-off-by: Anuradha Karuppiah <anuradhak@cumulusnetworks.com>

diff --git a/bgpd/bgp_evpn.c b/bgpd/bgp_evpn.c
index 2f1217945ea469a3d937065d53ee817903ec3c16..15d647f4df24ef86db9d721997eed0ab8641b745 100644 (file)
--- a/bgpd/bgp_evpn.c
+++ b/bgpd/bgp_evpn.c
@@ -2897,7 +2897,7 @@ bgp_evpn_skip_vrf_import_of_local_es(const struct prefix_evpn *evp,
                        nh = pi->attr->nexthop;
                else
                        nh.s_addr = 0;
-               if (!bgp_evpn_es_is_vtep_active(esi, nh)) {
+               if (install && !bgp_evpn_es_is_vtep_active(esi, nh)) {
                        if (BGP_DEBUG(evpn_mh, EVPN_MH_RT)) {
                                char esi_buf[ESI_STR_LEN];
 

diff --git a/bgpd/bgp_evpn_mh.c b/bgpd/bgp_evpn_mh.c
index f559fc7f4a91e274b5ea366122b24ca1e071de48..0407085d4cc3a1a7b96e3e4ee403d4cd771a1761 100644 (file)
--- a/bgpd/bgp_evpn_mh.c
+++ b/bgpd/bgp_evpn_mh.c
@@ -1236,19 +1236,14 @@ static void bgp_evpn_es_vtep_re_eval_active(struct bgp *bgp,
                /* send remote ES to zebra */
                bgp_zebra_send_remote_es_vtep(bgp, es_vtep, new_active);
 
-               /* If VTEP becomes active update the NHG first and then
-                * the exploded routes. If VTEP becomes inactive update
-                * routes first. This ordering is done to avoid deleting
-                * the NHG while there are dependent routes against
-                * it.
+               /* The NHG is updated first for efficient failover handling.
+                * Note the NHG can be de-activated while there are bgp
+                * routes referencing it. Zebra is capable of handling that
+                * elegantly by holding the NHG till all routes using it are
+                * removed.
                 */
-               if (new_active) {
-                       bgp_evpn_l3nhg_update_on_vtep_chg(es_vtep->es);
-                       bgp_evpn_es_path_all_update(es_vtep, true /*active*/);
-               } else {
-                       bgp_evpn_es_path_all_update(es_vtep, false /*active*/);
-                       bgp_evpn_l3nhg_update_on_vtep_chg(es_vtep->es);
-               }
+               bgp_evpn_l3nhg_update_on_vtep_chg(es_vtep->es);
+               bgp_evpn_es_path_all_update(es_vtep, new_active);
 
                /* queue up the es for background consistency checks */
                bgp_evpn_es_cons_checks_pend_add(es_vtep->es);
@@ -2158,37 +2153,116 @@ void bgp_evpn_es_show_esi(struct vty *vty, esi_t *esi, bool uj)
  * in that VRF.
  */
 /******************************** L3 NHG management *************************/
-static void bgp_evpn_l3nhg_zebra_add(struct bgp_evpn_es_vrf *es_vrf)
+static void bgp_evpn_l3nhg_zebra_add_v4_or_v6(struct bgp_evpn_es_vrf *es_vrf,
+                                             bool v4_nhg)
 {
-       uint32_t nh_cnt = 0;
+       uint32_t nhg_id = v4_nhg ? es_vrf->nhg_id : es_vrf->v6_nhg_id;
+       struct bgp_evpn_es *es = es_vrf->es;
        struct listnode *node;
        struct bgp_evpn_es_vtep *es_vtep;
-       struct bgp_evpn_es *es = es_vrf->es;
+       struct nexthop nh;
+       struct zapi_nexthop *api_nh;
+       struct zapi_nhg api_nhg = {};
+
+       /* Skip installation of L3-NHG if host routes used */
+       if (!nhg_id)
+               return;
 
        if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-               zlog_debug("es %s vrf %u nhg 0x%x to zebra", es->esi_str,
-                          es_vrf->bgp_vrf->vrf_id, es_vrf->nhg_id);
+               zlog_debug("es %s vrf %u %s nhg %u to zebra", es->esi_str,
+                          es_vrf->bgp_vrf->vrf_id,
+                          v4_nhg ? "v4_nhg" : "v6_nhg", nhg_id);
+
+       /* only the gateway ip changes for each NH. rest of the params
+        * are constant
+        */
+       memset(&nh, 0, sizeof(nh));
+       nh.vrf_id = es_vrf->bgp_vrf->vrf_id;
+       nh.flags = NEXTHOP_FLAG_ONLINK;
+       nh.ifindex = es_vrf->bgp_vrf->l3vni_svi_ifindex;
+       nh.weight = 1;
+       nh.type =
+               v4_nhg ? NEXTHOP_TYPE_IPV4_IFINDEX : NEXTHOP_TYPE_IPV6_IFINDEX;
+
+       api_nhg.id = nhg_id;
        for (ALL_LIST_ELEMENTS_RO(es->es_vtep_list, node, es_vtep)) {
-               if (CHECK_FLAG(es_vtep->flags, BGP_EVPNES_VTEP_ACTIVE)) {
-                       ++nh_cnt;
-                       if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-                               zlog_debug("nhg 0x%x vtep %pI4 dev 0x%x",
-                                          es_vrf->nhg_id, &es_vtep->vtep_ip,
-                                          es_vrf->bgp_vrf->l3vni_svi_ifindex);
-               }
+               if (!CHECK_FLAG(es_vtep->flags, BGP_EVPNES_VTEP_ACTIVE))
+                       continue;
+
+               /* overwrite the gw */
+               if (v4_nhg)
+                       nh.gate.ipv4 = es_vtep->vtep_ip;
+               else
+                       ipv4_to_ipv4_mapped_ipv6(&nh.gate.ipv6,
+                                                es_vtep->vtep_ip);
+
+               /* convert to zapi format */
+               api_nh = &api_nhg.nexthops[api_nhg.nexthop_num];
+               zapi_nexthop_from_nexthop(api_nh, &nh);
+
+               ++api_nhg.nexthop_num;
+               if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
+                       zlog_debug("nhg %u vtep %pI4 l3-svi %d", api_nhg.id,
+                                  &es_vtep->vtep_ip,
+                                  es_vrf->bgp_vrf->l3vni_svi_ifindex);
        }
 
-       /* XXX - program NHG in zebra */
+       if (!api_nhg.nexthop_num)
+               return;
+
+       if (api_nhg.nexthop_num > MULTIPATH_NUM)
+               return;
+
+       zclient_nhg_send(zclient, ZEBRA_NHG_ADD, &api_nhg);
 }
 
-static void bgp_evpn_l3nhg_zebra_del(struct bgp_evpn_es_vrf *es_vrf)
+static bool bgp_evpn_l3nhg_zebra_ok(struct bgp_evpn_es_vrf *es_vrf)
+{
+       if (!bgp_mh_info->host_routes_use_l3nhg && !bgp_mh_info->install_l3nhg)
+               return false;
+
+       /* Check socket. */
+       if (!zclient || zclient->sock < 0)
+               return false;
+
+       return true;
+}
+
+static void bgp_evpn_l3nhg_zebra_add(struct bgp_evpn_es_vrf *es_vrf)
 {
+       if (!bgp_evpn_l3nhg_zebra_ok(es_vrf))
+               return;
+
+       bgp_evpn_l3nhg_zebra_add_v4_or_v6(es_vrf, true /*v4_nhg*/);
+       bgp_evpn_l3nhg_zebra_add_v4_or_v6(es_vrf, false /*v4_nhg*/);
+}
+
+static void bgp_evpn_l3nhg_zebra_del_v4_or_v6(struct bgp_evpn_es_vrf *es_vrf,
+                                             bool v4_nhg)
+{
+       struct zapi_nhg api_nhg = {};
+
+       api_nhg.id = v4_nhg ? es_vrf->nhg_id : es_vrf->v6_nhg_id;
+
+       /* Skip installation of L3-NHG if host routes used */
+       if (!api_nhg.id)
+               return;
+
        if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-               zlog_debug("es %s vrf %u nhg 0x%x to zebra",
+               zlog_debug("es %s vrf %u %s nhg %u to zebra",
                           es_vrf->es->esi_str, es_vrf->bgp_vrf->vrf_id,
-                          es_vrf->nhg_id);
+                          v4_nhg ? "v4_nhg" : "v6_nhg", api_nhg.id);
+
+       zclient_nhg_send(zclient, ZEBRA_NHG_DEL, &api_nhg);
+}
 
-       /* XXX - program NHG in zebra */
+static void bgp_evpn_l3nhg_zebra_del(struct bgp_evpn_es_vrf *es_vrf)
+{
+       if (!bgp_evpn_l3nhg_zebra_ok(es_vrf))
+               return;
+
+       bgp_evpn_l3nhg_zebra_del_v4_or_v6(es_vrf, true /*v4_nhg*/);
+       bgp_evpn_l3nhg_zebra_del_v4_or_v6(es_vrf, false /*v4_nhg*/);
 }
 
 static void bgp_evpn_l3nhg_deactivate(struct bgp_evpn_es_vrf *es_vrf)
@@ -2197,7 +2271,7 @@ static void bgp_evpn_l3nhg_deactivate(struct bgp_evpn_es_vrf *es_vrf)
                return;
 
        if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-               zlog_debug("es %s vrf %u nhg 0x%x de-activate",
+               zlog_debug("es %s vrf %u nhg %u de-activate",
                           es_vrf->es->esi_str, es_vrf->bgp_vrf->vrf_id,
                           es_vrf->nhg_id);
        bgp_evpn_l3nhg_zebra_del(es_vrf);
@@ -2216,7 +2290,7 @@ static void bgp_evpn_l3nhg_activate(struct bgp_evpn_es_vrf *es_vrf, bool update)
                        return;
        } else {
                if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-                       zlog_debug("es %s vrf %u nhg 0x%x activate",
+                       zlog_debug("es %s vrf %u nhg %u activate",
                                   es_vrf->es->esi_str, es_vrf->bgp_vrf->vrf_id,
                                   es_vrf->nhg_id);
                es_vrf->flags |= BGP_EVPNES_VRF_NHG_ACTIVE;
@@ -2289,9 +2363,11 @@ static struct bgp_evpn_es_vrf *bgp_evpn_es_vrf_create(struct bgp_evpn_es *es,
 
        /* setup the L3 NHG id for the ES */
        es_vrf->nhg_id = bgp_l3nhg_id_alloc();
+       es_vrf->v6_nhg_id = bgp_l3nhg_id_alloc();
+
        if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-               zlog_debug("es %s vrf %u nhg 0x%x create", es->esi_str,
-                          bgp_vrf->vrf_id, es_vrf->nhg_id);
+               zlog_debug("es %s vrf %u nhg %u v6_nhg %d create", es->esi_str,
+                          bgp_vrf->vrf_id, es_vrf->nhg_id, es_vrf->v6_nhg_id);
        bgp_evpn_l3nhg_activate(es_vrf, false /* update */);
 
        return es_vrf;
@@ -2304,7 +2380,7 @@ static void bgp_evpn_es_vrf_delete(struct bgp_evpn_es_vrf *es_vrf)
        struct bgp *bgp_vrf = es_vrf->bgp_vrf;
 
        if (BGP_DEBUG(evpn_mh, EVPN_MH_ES))
-               zlog_debug("es %s vrf %u nhg 0x%x delete", es->esi_str,
+               zlog_debug("es %s vrf %u nhg %u delete", es->esi_str,
                           bgp_vrf->vrf_id, es_vrf->nhg_id);
 
        /* Remove the NHG resources */
@@ -2312,6 +2388,9 @@ static void bgp_evpn_es_vrf_delete(struct bgp_evpn_es_vrf *es_vrf)
        if (es_vrf->nhg_id)
                bgp_l3nhg_id_free(es_vrf->nhg_id);
        es_vrf->nhg_id = 0;
+       if (es_vrf->v6_nhg_id)
+               bgp_l3nhg_id_free(es_vrf->v6_nhg_id);
+       es_vrf->v6_nhg_id = 0;
 
        /* remove from the ES's VRF list */
        list_delete_node(es->es_vrf_list, &es_vrf->es_listnode);
@@ -2448,7 +2527,7 @@ bool bgp_evpn_path_es_use_nhg(struct bgp *bgp_vrf, struct bgp_path_info *pi,
                return true;
 
        /* this needs to be set the v6NHG if v6route */
-       if (evp->family == AF_INET6)
+       if (is_evpn_prefix_ipaddr_v6(evp))
                *nhg_p = es_vrf->v6_nhg_id;
        else
                *nhg_p = es_vrf->nhg_id;
@@ -3624,6 +3703,8 @@ void bgp_evpn_mh_init(void)
        /* config knobs - XXX add cli to control it */
        bgp_mh_info->ead_evi_adv_for_down_links = true;
        bgp_mh_info->consistency_checking = true;
+       bgp_mh_info->install_l3nhg = true;
+       bgp_mh_info->host_routes_use_l3nhg = false;
 
        if (bgp_mh_info->consistency_checking)
                thread_add_timer(bm->master, bgp_evpn_run_consistency_checks,

diff --git a/bgpd/bgp_evpn_mh.h b/bgpd/bgp_evpn_mh.h
index bde46e272bbac7b79407041172f0d712de7aa2a5..1ccc504cffcd15aa4e9f85459fb6ebcf6d9d9494 100644 (file)
--- a/bgpd/bgp_evpn_mh.h
+++ b/bgpd/bgp_evpn_mh.h
@@ -257,6 +257,8 @@ struct bgp_evpn_mh_info {
        bool ead_evi_adv_for_down_links;
        /* Enable ES consistency checking */
        bool consistency_checking;
+       /* Use L3 NHGs for host routes in symmetric IRB */
+       bool install_l3nhg;
        bool host_routes_use_l3nhg;
 };
 

diff --git a/bgpd/bgp_nht.c b/bgpd/bgp_nht.c
index f4acf379fc155c7f54a5d506eff03eb5d8b5b2a5..29ab3d9c6c953d2e19a9f2cbad198cb093fe9b77 100644 (file)
--- a/bgpd/bgp_nht.c
+++ b/bgpd/bgp_nht.c
@@ -31,6 +31,7 @@
 #include "nexthop.h"
 #include "vrf.h"
 #include "filter.h"
+#include "nexthop_group.h"
 
 #include "bgpd/bgpd.h"
 #include "bgpd/bgp_table.h"
@@ -975,32 +976,74 @@ void bgp_nht_dereg_enhe_cap_intfs(struct peer *peer)
  * failover of remote ES links.
  ***************************************************************************/
 static bitfield_t bgp_nh_id_bitmap;
+static uint32_t bgp_l3nhg_start;
 
-uint32_t bgp_l3nhg_id_alloc(void)
+/* XXX - currently we do nothing on the callbacks */
+static void bgp_l3nhg_add_cb(const char *name)
+{
+}
+static void bgp_l3nhg_add_nexthop_cb(const struct nexthop_group_cmd *nhgc,
+                                    const struct nexthop *nhop)
+{
+}
+static void bgp_l3nhg_del_nexthop_cb(const struct nexthop_group_cmd *nhgc,
+                                    const struct nexthop *nhop)
+{
+}
+static void bgp_l3nhg_del_cb(const char *name)
 {
-       uint32_t nhg_id = 0;
-
-       bf_assign_index(bgp_nh_id_bitmap, nhg_id);
-
-       return nhg_id;
 }
 
-void bgp_l3nhg_id_free(uint32_t nhg_id)
+static void bgp_l3nhg_zebra_init(void)
 {
-       if (!nhg_id)
+       static bool bgp_l3nhg_zebra_inited;
+       if (bgp_l3nhg_zebra_inited)
                return;
 
-       bf_release_index(bgp_nh_id_bitmap, nhg_id);
+       bgp_l3nhg_zebra_inited = true;
+       bgp_l3nhg_start = zclient_get_nhg_start(ZEBRA_ROUTE_BGP);
+       nexthop_group_init(bgp_l3nhg_add_cb, bgp_l3nhg_add_nexthop_cb,
+                          bgp_l3nhg_del_nexthop_cb, bgp_l3nhg_del_cb);
 }
 
+
+#define min(A, B) ((A) < (B) ? (A) : (B))
 void bgp_l3nhg_init(void)
 {
-#define BGP_NH_ID_MAX (16 * 1024)
-       bf_init(bgp_nh_id_bitmap, BGP_NH_ID_MAX);
+       uint32_t id_max;
+
+       id_max = min(ZEBRA_NHG_PROTO_SPACING - 1, 16 * 1024);
+       bf_init(bgp_nh_id_bitmap, id_max);
        bf_assign_zero_index(bgp_nh_id_bitmap);
+
+       if (BGP_DEBUG(nht, NHT) || BGP_DEBUG(evpn_mh, EVPN_MH_ES))
+               zlog_debug("bgp l3_nhg range %u - %u", bgp_l3nhg_start + 1,
+                          bgp_l3nhg_start + id_max);
 }
 
 void bgp_l3nhg_finish(void)
 {
        bf_free(bgp_nh_id_bitmap);
 }
+
+uint32_t bgp_l3nhg_id_alloc(void)
+{
+       uint32_t nhg_id = 0;
+
+       bgp_l3nhg_zebra_init();
+       bf_assign_index(bgp_nh_id_bitmap, nhg_id);
+       if (nhg_id)
+               nhg_id += bgp_l3nhg_start;
+
+       return nhg_id;
+}
+
+void bgp_l3nhg_id_free(uint32_t nhg_id)
+{
+       if (!nhg_id || (nhg_id <= bgp_l3nhg_start))
+               return;
+
+       nhg_id -= bgp_l3nhg_start;
+
+       bf_release_index(bgp_nh_id_bitmap, nhg_id);
+}

diff --git a/bgpd/bgp_zebra.c b/bgpd/bgp_zebra.c
index eb0883f775d56b2b9d08c2fae0a41cf073815f62..12bc2e41bc36e14e9f8a6620311132a97367eb72 100644 (file)
--- a/bgpd/bgp_zebra.c
+++ b/bgpd/bgp_zebra.c
@@ -1260,10 +1260,14 @@ void bgp_zebra_announce(struct bgp_dest *dest, const struct prefix *p,
                cum_bw = bgp_path_info_mpath_cumbw(info);
 
        /* EVPN MAC-IP routes are installed with a L3 NHG id */
-       if (bgp_evpn_path_es_use_nhg(bgp, info, &nhg_id))
+       if (bgp_evpn_path_es_use_nhg(bgp, info, &nhg_id)) {
                mpinfo = NULL;
-       else
+               api.nhgid = nhg_id;
+               if (nhg_id)
+                       SET_FLAG(api.message, ZAPI_MESSAGE_NHG);
+       } else {
                mpinfo = info;
+       }
 
        for (; mpinfo; mpinfo = bgp_path_info_mpath_next(mpinfo)) {
                uint32_t nh_weight;