*: Handle VRF configuration when VRF gets inactivated and activated

A VRF is active only when the corresponding VRF device is present in the
kernel. However, when the kernel VRF device is removed, the VRF container in
FRR should go away only if there is no user configuration for it. Otherwise,
when the VRF device is created again so that the VRF becomes active, FRR
cannot take the correct actions. Example configuration for the VRF includes
static routes and EVPN L3 VNI.

Note that a VRF is currently considered to be "configured" as soon as the
operator has issued the "vrf <name>" command in FRR. Such a configured VRF
is not deleted upon VRF device removal, it is only made inactive. A VRF that
is "configured" can be deleted only upon operator action and only if the VRF
has been deactivated i.e., the VRF device removed from the kernel. This is
an existing restriction.

To implement this change, the VRF disable and delete actions have been modified.

Signed-off-by: Vivek Venkatraman <vivek@cumulusnetworks.com>
Reviewed-by:   Donald Sharp <sharpd@cumulusnetworks.com>
Reviewed-by:   Mitesh Kanjariya <mkanjariya@cumulusnetworks.com>
Reviewed-by:   Don Slice <dslice@cumulusnetworks.com>

Ticket: CM-18553, CM-18918, CM-10139
Reviewed By: CCR-7022
Testing Done:
1. vrf and pim-vrf automation tests
2. Multiple VRF delete and readd (ifdown, ifup-with-depends)
3. FRR stop, start, restart
4. Networking restart
5. Configuration delete and readd

Some of the above tests run in different sequences (manually).

2 files changed, 41 insertions, 10 deletions

diff --git a/lib/vrf.c b/lib/vrf.c
index 2fa3a9c0ef..bc081796c5 100644
--- a/lib/vrf.c
+++ b/lib/vrf.c
@@ -141,7 +141,9 @@ struct vrf *vrf_get(vrf_id_t vrf_id, const char *name)
 	return vrf;
 }
 
-/* Delete a VRF. This is called in vrf_terminate(). */
+/* Delete a VRF. This is called when the underlying VRF goes away, a
+ * pre-configured VRF is deleted or when shutting down (vrf_terminate()).
+ */
 void vrf_delete(struct vrf *vrf)
 {
 	if (debug_vrf)
@@ -150,6 +152,23 @@ void vrf_delete(struct vrf *vrf)
 	if (vrf_is_enabled(vrf))
 		vrf_disable(vrf);
 
+	/* If the VRF is user configured, it'll stick around, just remove
+	 * the ID mapping. Interfaces assigned to this VRF should've been
+	 * removed already as part of the VRF going down.
+	 */
+	if (vrf_is_user_cfged(vrf)) {
+		if (vrf->vrf_id != VRF_UNKNOWN) {
+			/* Delete any VRF interfaces - should be only
+			 * the VRF itself, other interfaces should've
+			 * been moved out of the VRF.
+			 */
+			if_terminate(vrf);
+			RB_REMOVE(vrf_id_head, &vrfs_by_id, vrf);
+			vrf->vrf_id = VRF_UNKNOWN;
+		}
+		return;
+	}
+
 	if (vrf_master.vrf_delete_hook)
 		(*vrf_master.vrf_delete_hook)(vrf);
 
@@ -173,14 +192,6 @@ struct vrf *vrf_lookup_by_id(vrf_id_t vrf_id)
 }
 
 /*
- * Check whether the VRF is enabled.
- */
-static int vrf_is_enabled(struct vrf *vrf)
-{
-	return vrf && CHECK_FLAG(vrf->status, VRF_ACTIVE);
-}
-
-/*
  * Enable a VRF - that is, let the VRF be ready to use.
  * The VRF_ENABLE_HOOK callback will be called to inform
  * that they can allocate resources in this VRF.
@@ -445,6 +456,9 @@ DEFUN_NOSH (vrf,
 
 	vrfp = vrf_get(VRF_UNKNOWN, vrfname);
 
+	/* Mark as user configured. */
+	SET_FLAG(vrfp->status, VRF_CONFIGURED);
+
 	VTY_PUSH_CONTEXT(VRF_NODE, vrfp);
 
 	return CMD_SUCCESS;
@@ -473,6 +487,8 @@ DEFUN_NOSH (no_vrf,
 		return CMD_WARNING_CONFIG_FAILED;
 	}
 
+	/* Clear configured flag and invoke delete. */
+	UNSET_FLAG(vrfp->status, VRF_CONFIGURED);
 	vrf_delete(vrfp);
 
 	return CMD_SUCCESS;
diff --git a/lib/vrf.h b/lib/vrf.h
index 7e625769e7..89d2316354 100644
--- a/lib/vrf.h
+++ b/lib/vrf.h
@@ -75,7 +75,8 @@ struct vrf {
 
 	/* Zebra internal VRF status */
 	u_char status;
-#define VRF_ACTIVE     (1 << 0)
+#define VRF_ACTIVE     (1 << 0) /* VRF is up in kernel */
+#define VRF_CONFIGURED (1 << 1) /* VRF is configured by user in frr */
 
 	/* Interfaces belonging to this VRF */
 	struct if_name_head ifaces_by_name;
@@ -120,6 +121,20 @@ extern vrf_id_t vrf_name_to_id(const char *);
 	} while (0)
 
 /*
+ * Check whether the VRF is enabled.
+ */
+static inline int vrf_is_enabled(struct vrf *vrf)
+{
+	return vrf && CHECK_FLAG(vrf->status, VRF_ACTIVE);
+}
+
+/* check if the vrf is user configured */
+static inline int vrf_is_user_cfged(struct vrf *vrf)
+{
+	return vrf && CHECK_FLAG(vrf->status, VRF_CONFIGURED);
+}
+
+/*
  * Utilities to obtain the user data
  */