*: Handle VRF configuration when VRF gets inactivated and activated

A VRF is active only when the corresponding VRF device is present in the
kernel. However, when the kernel VRF device is removed, the VRF container in
FRR should go away only if there is no user configuration for it. Otherwise,
when the VRF device is created again so that the VRF becomes active, FRR
cannot take the correct actions. Example configuration for the VRF includes
static routes and EVPN L3 VNI.

Note that a VRF is currently considered to be "configured" as soon as the
operator has issued the "vrf <name>" command in FRR. Such a configured VRF
is not deleted upon VRF device removal, it is only made inactive. A VRF that
is "configured" can be deleted only upon operator action and only if the VRF
has been deactivated i.e., the VRF device removed from the kernel. This is
an existing restriction.

To implement this change, the VRF disable and delete actions have been modified.

Signed-off-by: Vivek Venkatraman <vivek@cumulusnetworks.com>
Reviewed-by:   Donald Sharp <sharpd@cumulusnetworks.com>
Reviewed-by:   Mitesh Kanjariya <mkanjariya@cumulusnetworks.com>
Reviewed-by:   Don Slice <dslice@cumulusnetworks.com>
Ticket: CM-18553, CM-18918, CM-10139
Reviewed By: CCR-7022
Testing Done:
1. vrf and pim-vrf automation tests
2. Multiple VRF delete and readd (ifdown, ifup-with-depends)
3. FRR stop, start, restart
4. Networking restart
5. Configuration delete and readd

Some of the above tests run in different sequences (manually).

diff --git a/lib/vrf.c b/lib/vrf.c
index 2fa3a9c0ef512eb67496f20c28172031c6f48584..bc081796c55cac3ca0df58ac6fd022b9588ea914 100644 (file)
--- a/lib/vrf.c
+++ b/lib/vrf.c
@@ -141,7 +141,9 @@ struct vrf *vrf_get(vrf_id_t vrf_id, const char *name)
        return vrf;
 }
 
-/* Delete a VRF. This is called in vrf_terminate(). */
+/* Delete a VRF. This is called when the underlying VRF goes away, a
+ * pre-configured VRF is deleted or when shutting down (vrf_terminate()).
+ */
 void vrf_delete(struct vrf *vrf)
 {
        if (debug_vrf)
@@ -150,6 +152,23 @@ void vrf_delete(struct vrf *vrf)
        if (vrf_is_enabled(vrf))
                vrf_disable(vrf);
 
+       /* If the VRF is user configured, it'll stick around, just remove
+        * the ID mapping. Interfaces assigned to this VRF should've been
+        * removed already as part of the VRF going down.
+        */
+       if (vrf_is_user_cfged(vrf)) {
+               if (vrf->vrf_id != VRF_UNKNOWN) {
+                       /* Delete any VRF interfaces - should be only
+                        * the VRF itself, other interfaces should've
+                        * been moved out of the VRF.
+                        */
+                       if_terminate(vrf);
+                       RB_REMOVE(vrf_id_head, &vrfs_by_id, vrf);
+                       vrf->vrf_id = VRF_UNKNOWN;
+               }
+               return;
+       }
+
        if (vrf_master.vrf_delete_hook)
                (*vrf_master.vrf_delete_hook)(vrf);
 
@@ -172,14 +191,6 @@ struct vrf *vrf_lookup_by_id(vrf_id_t vrf_id)
        return (RB_FIND(vrf_id_head, &vrfs_by_id, &vrf));
 }
 
-/*
- * Check whether the VRF is enabled.
- */
-static int vrf_is_enabled(struct vrf *vrf)
-{
-       return vrf && CHECK_FLAG(vrf->status, VRF_ACTIVE);
-}
-
 /*
  * Enable a VRF - that is, let the VRF be ready to use.
  * The VRF_ENABLE_HOOK callback will be called to inform
@@ -445,6 +456,9 @@ DEFUN_NOSH (vrf,
 
        vrfp = vrf_get(VRF_UNKNOWN, vrfname);
 
+       /* Mark as user configured. */
+       SET_FLAG(vrfp->status, VRF_CONFIGURED);
+
        VTY_PUSH_CONTEXT(VRF_NODE, vrfp);
 
        return CMD_SUCCESS;
@@ -473,6 +487,8 @@ DEFUN_NOSH (no_vrf,
                return CMD_WARNING_CONFIG_FAILED;
        }
 
+       /* Clear configured flag and invoke delete. */
+       UNSET_FLAG(vrfp->status, VRF_CONFIGURED);
        vrf_delete(vrfp);
 
        return CMD_SUCCESS;

diff --git a/lib/vrf.h b/lib/vrf.h
index 7e625769e7f332c447b6e84387ec7c3c42a3ffd7..89d2316354ccc2136212c3fdb5ad1937bc9ff4aa 100644 (file)
--- a/lib/vrf.h
+++ b/lib/vrf.h
@@ -75,7 +75,8 @@ struct vrf {
 
        /* Zebra internal VRF status */
        u_char status;
-#define VRF_ACTIVE     (1 << 0)
+#define VRF_ACTIVE     (1 << 0) /* VRF is up in kernel */
+#define VRF_CONFIGURED (1 << 1) /* VRF is configured by user in frr */
 
        /* Interfaces belonging to this VRF */
        struct if_name_head ifaces_by_name;
@@ -119,6 +120,20 @@ extern vrf_id_t vrf_name_to_id(const char *);
                (V) = vrf->vrf_id;                                             \
        } while (0)
 
+/*
+ * Check whether the VRF is enabled.
+ */
+static inline int vrf_is_enabled(struct vrf *vrf)
+{
+       return vrf && CHECK_FLAG(vrf->status, VRF_ACTIVE);
+}
+
+/* check if the vrf is user configured */
+static inline int vrf_is_user_cfged(struct vrf *vrf)
+{
+       return vrf && CHECK_FLAG(vrf->status, VRF_CONFIGURED);
+}
+
 /*
  * Utilities to obtain the user data
  */

diff --git a/zebra/zebra_mpls.c b/zebra/zebra_mpls.c
index 61051ba87e966148983c57c4b4305c299bd9f361..22c81b57841320a20476be61c55ec979590b4920 100644 (file)
--- a/zebra/zebra_mpls.c
+++ b/zebra/zebra_mpls.c
@@ -2875,6 +2875,16 @@ int zebra_mpls_write_label_block_config(struct vty *vty, struct zebra_vrf *zvrf)
        return 1;
 }
 
+/*
+ * Called when VRF becomes inactive, cleans up information but keeps
+ * the table itself.
+ * NOTE: Currently supported only for default VRF.
+ */
+void zebra_mpls_cleanup_tables(struct zebra_vrf *zvrf)
+{
+       hash_iterate(zvrf->lsp_table, lsp_uninstall_from_kernel, NULL);
+}
+
 /*
  * Called upon process exiting, need to delete LSP forwarding
  * entries from the kernel.

diff --git a/zebra/zebra_mpls.h b/zebra/zebra_mpls.h
index 22c771c34809aa86706180a480c00056d40b4b1b..900474a6111fdf85b8dbfcb1b9ae510293631c1e 100644 (file)
--- a/zebra/zebra_mpls.h
+++ b/zebra/zebra_mpls.h
@@ -378,6 +378,13 @@ void zebra_mpls_print_lsp_table(struct vty *vty, struct zebra_vrf *zvrf,
  */
 int zebra_mpls_write_lsp_config(struct vty *vty, struct zebra_vrf *zvrf);
 
+/*
+ * Called when VRF becomes inactive, cleans up information but keeps
+ * the table itself.
+ * NOTE: Currently supported only for default VRF.
+ */
+void zebra_mpls_cleanup_tables(struct zebra_vrf *zvrf);
+
 /*
  * Called upon process exiting, need to delete LSP forwarding
  * entries from the kernel.

diff --git a/zebra/zebra_ns.c b/zebra/zebra_ns.c
index b3b9c6d18a4c9f1144d2c75ad86127def4129f22..c48a6f7bd8ad377e8b7e882281a5f0f8cc147ea1 100644 (file)
--- a/zebra/zebra_ns.c
+++ b/zebra/zebra_ns.c
@@ -41,6 +41,7 @@ struct zebra_ns *zebra_ns_lookup(ns_id_t ns_id)
        return dzns;
 }
 
+/* Do global enable actions - open sockets, read kernel config etc. */
 int zebra_ns_enable(ns_id_t ns_id, void **info)
 {
        struct zebra_ns *zns = (struct zebra_ns *)(*info);
@@ -49,8 +50,6 @@ int zebra_ns_enable(ns_id_t ns_id, void **info)
        rtadv_init(zns);
 #endif
 
-       zns->if_table = route_table_init();
-       zebra_vxlan_ns_init(zns);
        kernel_init(zns);
        interface_list(zns);
        route_read(zns);
@@ -79,8 +78,14 @@ int zebra_ns_init(void)
 
        ns_init();
 
+       /* Do any needed per-NS data structure allocation. */
+       dzns->if_table = route_table_init();
+       zebra_vxlan_ns_init(dzns);
+
+       /* Register zebra VRF callbacks, create and activate default VRF. */
        zebra_vrf_init();
 
+       /* Default NS is activated */
        zebra_ns_enable(NS_DEFAULT, (void **)&dzns);
 
        return 0;

diff --git a/zebra/zebra_vrf.c b/zebra/zebra_vrf.c
index 000beba870ec717e21bba205a84e945a0a120601..6c9171c95cde753e03d5715eaf7ffcd5638ec686 100644 (file)
--- a/zebra/zebra_vrf.c
+++ b/zebra/zebra_vrf.c
@@ -82,7 +82,7 @@ static int zebra_vrf_new(struct vrf *vrf)
        struct zebra_vrf *zvrf;
 
        if (IS_ZEBRA_DEBUG_EVENT)
-               zlog_info("ZVRF %s with id %u", vrf->name, vrf->vrf_id);
+               zlog_info("VRF %s created, id %u", vrf->name, vrf->vrf_id);
 
        zvrf = zebra_vrf_alloc();
        zvrf->zns = zebra_ns_lookup(
@@ -106,9 +106,16 @@ static int zebra_vrf_enable(struct vrf *vrf)
        safi_t safi;
 
        assert(zvrf);
+       if (IS_ZEBRA_DEBUG_EVENT)
+               zlog_debug("VRF %s id %u is now active",
+                          zvrf_name(zvrf), zvrf_id(zvrf));
 
+       /* Inform clients that the VRF is now active. This is an
+        * add for the clients.
+        */
        zebra_vrf_add_update(zvrf);
 
+       /* Install any static routes configured for this VRF. */
        for (afi = AFI_IP; afi < AFI_MAX; afi++)
                for (safi = SAFI_UNICAST; safi < SAFI_MAX; safi++) {
                        stable = zvrf->stable[afi][safi];
@@ -132,6 +139,9 @@ static int zebra_vrf_enable(struct vrf *vrf)
                                }
                }
 
+       /* Kick off any VxLAN-EVPN processing. */
+       zebra_vxlan_vrf_enable(zvrf);
+
        return 0;
 }
 
@@ -142,13 +152,16 @@ static int zebra_vrf_disable(struct vrf *vrf)
        struct route_table *stable;
        struct route_node *rn;
        struct static_route *si;
+       u_int32_t table_id;
        afi_t afi;
        safi_t safi;
 
-       if (IS_ZEBRA_DEBUG_KERNEL)
-               zlog_debug("VRF %s id %u is now disabled.", zvrf_name(zvrf),
-                          zvrf_id(zvrf));
+       assert(zvrf);
+       if (IS_ZEBRA_DEBUG_EVENT)
+               zlog_debug("VRF %s id %u is now inactive",
+                          zvrf_name(zvrf), zvrf_id(zvrf));
 
+       /* Uninstall any static routes configured for this VRF. */
        for (afi = AFI_IP; afi < AFI_MAX; afi++)
                for (safi = SAFI_UNICAST; safi < SAFI_MAX; safi++) {
                        stable = zvrf->stable[afi][safi];
@@ -161,23 +174,15 @@ static int zebra_vrf_disable(struct vrf *vrf)
                                                afi, safi, &rn->p, NULL, si);
                }
 
-       return 0;
-}
-
-static int zebra_vrf_delete(struct vrf *vrf)
-{
-       struct zebra_vrf *zvrf = vrf->info;
-       struct route_table *table;
-       u_int32_t table_id;
-       afi_t afi;
-       safi_t safi;
-       unsigned i;
-
-       assert(zvrf);
+       /* Stop any VxLAN-EVPN processing. */
+       zebra_vxlan_vrf_disable(zvrf);
 
+       /* Inform clients that the VRF is now inactive. This is a
+        * delete for the clients.
+        */
        zebra_vrf_delete_update(zvrf);
 
-       /* uninstall everything */
+       /* uninstall all routes */
        if (!CHECK_FLAG(zvrf->flags, ZEBRA_VRF_RETAIN)) {
                struct interface *ifp;
 
@@ -197,16 +202,32 @@ static int zebra_vrf_delete(struct vrf *vrf)
                                                                [table_id]);
                }
 
-               /* Cleanup Vxlan table and update kernel */
-               zebra_vxlan_close_tables(zvrf);
-
-               zebra_mpls_close_tables(zvrf);
+               /* Cleanup Vxlan, MPLS and PW tables. */
+               zebra_vxlan_cleanup_tables(zvrf);
+               zebra_mpls_cleanup_tables(zvrf);
                zebra_pw_exit(zvrf);
 
                FOR_ALL_INTERFACES (vrf, ifp)
                        if_nbr_ipv6ll_to_ipv4ll_neigh_del_all(ifp);
        }
 
+       return 0;
+}
+
+static int zebra_vrf_delete(struct vrf *vrf)
+{
+       struct zebra_vrf *zvrf = vrf->info;
+       struct route_table *table;
+       u_int32_t table_id;
+       afi_t afi;
+       safi_t safi;
+       unsigned i;
+
+       assert(zvrf);
+       if (IS_ZEBRA_DEBUG_EVENT)
+               zlog_debug("VRF %s id %u deleted",
+                          zvrf_name(zvrf), zvrf_id(zvrf));
+
        /* clean-up work queues */
        for (i = 0; i < MQ_SIZE; i++) {
                struct listnode *lnode, *nnode;
@@ -224,6 +245,10 @@ static int zebra_vrf_delete(struct vrf *vrf)
                }
        }
 
+       /* Free Vxlan and MPLS. */
+       zebra_vxlan_close_tables(zvrf);
+       zebra_mpls_close_tables(zvrf);
+
        /* release allocated memory */
        for (afi = AFI_IP; afi <= AFI_IP6; afi++) {
                void *table_info;
@@ -251,7 +276,7 @@ static int zebra_vrf_delete(struct vrf *vrf)
                route_table_finish(zvrf->import_check_table[afi]);
        }
 
-       /* cleanup evpn states for vrf */
+       /* Cleanup EVPN states for vrf */
        zebra_vxlan_vrf_delete(zvrf);
 
        list_delete_all_node(zvrf->rid_all_sorted_list);
@@ -482,7 +507,7 @@ static int vrf_config_write(struct vty *vty)
                        vty_out(vty, "!\n");
                }
 
-               if (strcmp(zvrf_name(zvrf), VRF_DEFAULT_NAME)) {
+               if (vrf_is_user_cfged(vrf)) {
                        vty_out(vty, "vrf %s\n", zvrf_name(zvrf));
 
                static_config(vty, zvrf, AFI_IP, SAFI_UNICAST, "ip route");

diff --git a/zebra/zebra_vxlan.c b/zebra/zebra_vxlan.c
index 7b79c5be4d397586acb17c1d65025447f68c6cd5..51b4f23151b2379a92db994bdf04aefc02249d62 100644 (file)
--- a/zebra/zebra_vxlan.c
+++ b/zebra/zebra_vxlan.c
@@ -2866,17 +2866,19 @@ static int zvni_vtep_uninstall(zebra_vni_t *zvni, struct in_addr *vtep_ip)
 /*
  * Cleanup VNI/VTEP and update kernel
  */
-static void zvni_cleanup_all(struct hash_backet *backet, void *zvrf)
+static void zvni_cleanup_all(struct hash_backet *backet, void *arg)
 {
        zebra_vni_t *zvni = NULL;
        zebra_l3vni_t *zl3vni = NULL;
+       struct zebra_vrf *zvrf = (struct zebra_vrf *)arg;
 
        zvni = (zebra_vni_t *)backet->data;
        if (!zvni)
                return;
 
        /* remove from l3-vni list */
-       zl3vni = zl3vni_from_vrf(zvni->vrf_id);
+       if (zvrf->l3vni)
+               zl3vni = zl3vni_lookup(zvrf->l3vni);
        if (zl3vni)
                listnode_delete(zl3vni->l2vnis, zvni);
 
@@ -6590,17 +6592,48 @@ int zebra_vxlan_process_vrf_vni_cmd(struct zebra_vrf *zvrf,
        return 0;
 }
 
-int zebra_vxlan_vrf_delete(struct zebra_vrf *zvrf)
+int zebra_vxlan_vrf_enable(struct zebra_vrf *zvrf)
+{
+       zebra_l3vni_t *zl3vni = NULL;
+
+       if (zvrf->l3vni)
+               zl3vni = zl3vni_lookup(zvrf->l3vni);
+       if (!zl3vni)
+               return 0;
+
+       zl3vni->vrf_id = zvrf_id(zvrf);
+       if (is_l3vni_oper_up(zl3vni))
+               zebra_vxlan_process_l3vni_oper_up(zl3vni);
+       return 0;
+}
+
+int zebra_vxlan_vrf_disable(struct zebra_vrf *zvrf)
 {
        zebra_l3vni_t *zl3vni = NULL;
 
-       zl3vni = zl3vni_from_vrf(zvrf_id(zvrf));
+       if (zvrf->l3vni)
+               zl3vni = zl3vni_lookup(zvrf->l3vni);
        if (!zl3vni)
                return 0;
 
+       zl3vni->vrf_id = VRF_UNKNOWN;
        zebra_vxlan_process_l3vni_oper_down(zl3vni);
+       return 0;
+}
+
+int zebra_vxlan_vrf_delete(struct zebra_vrf *zvrf)
+{
+       zebra_l3vni_t *zl3vni = NULL;
+       vni_t vni;
+
+       if (zvrf->l3vni)
+               zl3vni = zl3vni_lookup(zvrf->l3vni);
+       if (!zl3vni)
+               return 0;
+
+       vni = zl3vni->vni;
        zl3vni_del(zl3vni);
-       zebra_vxlan_handle_vni_transition(zvrf, zl3vni->vni, 0);
+       zebra_vxlan_handle_vni_transition(zvrf, vni, 0);
 
        return 0;
 }
@@ -6853,6 +6886,14 @@ void zebra_vxlan_init_tables(struct zebra_vrf *zvrf)
                                      "Zebra VRF VNI Table");
 }
 
+/* Cleanup VNI info, but don't free the table. */
+void zebra_vxlan_cleanup_tables(struct zebra_vrf *zvrf)
+{
+       if (!zvrf)
+               return;
+       hash_iterate(zvrf->vni_table, zvni_cleanup_all, zvrf);
+}
+
 /* Close all VNI handling */
 void zebra_vxlan_close_tables(struct zebra_vrf *zvrf)
 {

diff --git a/zebra/zebra_vxlan.h b/zebra/zebra_vxlan.h
index 2a505911005faa80e179178641e4742e31127e7f..d9801a8b6059d4bcbeec9f569aa249a8781653e2 100644 (file)
--- a/zebra/zebra_vxlan.h
+++ b/zebra/zebra_vxlan.h
@@ -54,6 +54,9 @@ is_evpn_enabled()
 
 extern ifindex_t get_l3vni_svi_ifindex(vrf_id_t vrf_id);
 extern int zebra_vxlan_vrf_delete(struct zebra_vrf *zvrf);
+extern int zebra_vxlan_vrf_enable(struct zebra_vrf *zvrf);
+extern int zebra_vxlan_vrf_disable(struct zebra_vrf *zvrf);
+extern int zebra_vxlan_vrf_delete(struct zebra_vrf *zvrf);
 extern void zebra_vxlan_print_specific_nh_l3vni(struct vty *vty, vni_t l3vni,
                                                struct ipaddr *ip, u_char uj);
 extern void zebra_vxlan_print_evpn(struct vty *vty, u_char uj);
@@ -154,6 +157,7 @@ extern int zebra_vxlan_process_vrf_vni_cmd(struct zebra_vrf *zvrf, vni_t vni,
                                           int err_str_sz, int add);
 extern void zebra_vxlan_init_tables(struct zebra_vrf *zvrf);
 extern void zebra_vxlan_close_tables(struct zebra_vrf *);
+extern void zebra_vxlan_cleanup_tables(struct zebra_vrf *);
 extern void zebra_vxlan_ns_init(struct zebra_ns *zns);
 extern void zebra_vxlan_ns_disable(struct zebra_ns *zns);
 extern void zebra_vxlan_evpn_vrf_route_add(vrf_id_t vrf_id,

diff --git a/zebra/zebra_vxlan_null.c b/zebra/zebra_vxlan_null.c
index fa6ed77e536afc9619d3a76e7b77d7abacf98998..0eb880e848abd8539efff69079237fd18e79a2ff 100644 (file)
--- a/zebra/zebra_vxlan_null.c
+++ b/zebra/zebra_vxlan_null.c
@@ -207,3 +207,7 @@ void zebra_vxlan_init_tables(struct zebra_vrf *zvrf)
 void zebra_vxlan_close_tables(struct zebra_vrf *zvrf)
 {
 }
+
+void zebra_vxlan_cleanup_tables(struct zebra_vrf *zvrf)
+{
+}