pimd: fix channel_oil and upstream RPF in sync

During PIM Neighbor change/UP event, pim_scan_oil api
scans all channel oil to see any rpf impacted. Instead of
passing current upstream's RPF it passes current RPF as 0 and
does query to rib for nexhtop (without ECMP/Rebalance). This creates
inconsist RPF between Upstream and Channel oil.
In Channel Oil keep backward pointer to upstream DB and fetch up's
RPF and passed to channel_oil scan.
Decrement channel_oil ref_count in upstream_del when decrementing
up ref_count and it is not the last.
Created ECMP based FIB lookup API.

Testing Done:
Performed following testing on tester setup:
5 x LHR, 4 x MSDP Spines, 6 Sources each sending to 1023 groups from one of the spines.
Total send rate 8Mpps.
Test that caused problems was to reboot every device at the same time.
After fix performed 5 iterations of reboot devices and show no sign of the problem.

Signed-off-by: Chirag Shah <chirag@cumulusnetworks.com>

1 files changed, 63 insertions, 84 deletions

diff --git a/pimd/pim_zebra.c b/pimd/pim_zebra.c
index 80e7d77642..baf296623f 100644
--- a/pimd/pim_zebra.c
+++ b/pimd/pim_zebra.c
@@ -53,7 +53,6 @@
 
 static struct zclient *zclient = NULL;
 
-static int fib_lookup_if_vif_index(struct in_addr addr);
 
 /* Router-id update message from zebra. */
 static int pim_router_id_update_zebra(int command, struct zclient *zclient,
@@ -469,7 +468,28 @@ pim_scan_individual_oil (struct channel_oil *c_oil, int in_vif_index)
   if (in_vif_index)
     input_iface_vif_index = in_vif_index;
   else
-    input_iface_vif_index = fib_lookup_if_vif_index (vif_source);
+    {
+      struct prefix src, grp;
+
+      src.family = AF_INET;
+      src.prefixlen = IPV4_MAX_BITLEN;
+      src.u.prefix4 = vif_source;
+      grp.family = AF_INET;
+      grp.prefixlen = IPV4_MAX_BITLEN;
+      grp.u.prefix4 = c_oil->oil.mfcc_mcastgrp;
+
+      if (PIM_DEBUG_ZEBRA)
+        {
+          char source_str[INET_ADDRSTRLEN];
+          char group_str[INET_ADDRSTRLEN];
+          pim_inet4_dump("<source?>", c_oil->oil.mfcc_origin, source_str, sizeof(source_str));
+          pim_inet4_dump("<group?>", c_oil->oil.mfcc_mcastgrp, group_str, sizeof(group_str));
+          zlog_debug ("%s: channel_oil (%s, %s) upstream info is not present.",
+                      __PRETTY_FUNCTION__, source_str, group_str);
+        }
+      input_iface_vif_index = pim_ecmp_fib_lookup_if_vif_index(vif_source, &src, &grp);
+    }
+
   if (input_iface_vif_index < 1)
     {
       if (PIM_DEBUG_ZEBRA)
@@ -557,12 +577,25 @@ void pim_scan_oil()
   struct listnode    *node;
   struct listnode    *nextnode;
   struct channel_oil *c_oil;
+  ifindex_t          ifindex;
+  int                vif_index = 0;
 
   qpim_scan_oil_last = pim_time_monotonic_sec();
   ++qpim_scan_oil_events;
 
   for (ALL_LIST_ELEMENTS(pim_channel_oil_list, node, nextnode, c_oil))
-    pim_scan_individual_oil (c_oil, 0);
+    {
+      if (c_oil->up && c_oil->up->rpf.source_nexthop.interface)
+        {
+          ifindex = c_oil->up->rpf.source_nexthop.interface->ifindex;
+          vif_index = pim_if_find_vifindex_by_ifindex (ifindex);
+          /* Pass Current selected NH vif index to mroute download */
+          if (vif_index)
+            pim_scan_individual_oil (c_oil, vif_index);
+        }
+      else
+        pim_scan_individual_oil (c_oil, 0);
+    }
 }
 
 static int on_rpf_cache_refresh(struct thread *t)
@@ -699,67 +732,6 @@ void igmp_anysource_forward_stop(struct igmp_group *group)
     igmp_source_forward_stop (source);
 }
 
-static int fib_lookup_if_vif_index(struct in_addr addr)
-{
-  struct pim_zlookup_nexthop nexthop_tab[MULTIPATH_NUM];
-  int num_ifindex;
-  int vif_index;
-  ifindex_t first_ifindex;
-
-  num_ifindex = zclient_lookup_nexthop(nexthop_tab,
-				       MULTIPATH_NUM, addr,
-				       PIM_NEXTHOP_LOOKUP_MAX);
-  if (num_ifindex < 1) {
-    if (PIM_DEBUG_ZEBRA)
-      {
-	char addr_str[INET_ADDRSTRLEN];
-	pim_inet4_dump("<addr?>", addr, addr_str, sizeof(addr_str));
-	zlog_debug("%s %s: could not find nexthop ifindex for address %s",
-		   __FILE__, __PRETTY_FUNCTION__,
-		   addr_str);
-      }
-    return -1;
-  }
-  
-  first_ifindex = nexthop_tab[0].ifindex;
-  
-  if (num_ifindex > 1) {
-    if (PIM_DEBUG_ZEBRA)
-      {
-	char addr_str[INET_ADDRSTRLEN];
-	pim_inet4_dump("<addr?>", addr, addr_str, sizeof(addr_str));
-	zlog_debug("%s %s: FIXME ignoring multiple nexthop ifindex'es num_ifindex=%d for address %s (using only ifindex=%d)",
-		   __FILE__, __PRETTY_FUNCTION__,
-		   num_ifindex, addr_str, first_ifindex);
-      }
-    /* debug warning only, do not return */
-  }
-  
-  if (PIM_DEBUG_ZEBRA) {
-    char addr_str[INET_ADDRSTRLEN];
-    pim_inet4_dump("<ifaddr?>", addr, addr_str, sizeof(addr_str));
-    zlog_debug("%s %s: found nexthop ifindex=%d (interface %s) for address %s",
-	       __FILE__, __PRETTY_FUNCTION__,
-	       first_ifindex, ifindex2ifname(first_ifindex, VRF_DEFAULT), addr_str);
-  }
-
-  vif_index = pim_if_find_vifindex_by_ifindex(first_ifindex);
-
-  if (vif_index < 0) {
-    if (PIM_DEBUG_ZEBRA)
-      {
-	char addr_str[INET_ADDRSTRLEN];
-	pim_inet4_dump("<addr?>", addr, addr_str, sizeof(addr_str));
-	zlog_debug("%s %s: low vif_index=%d < 1 nexthop for address %s",
-		   __FILE__, __PRETTY_FUNCTION__,
-		   vif_index, addr_str);
-      }
-    return -2;
-  }
-
-  return vif_index;
-}
-
 static void
 igmp_source_forward_reevaluate_one(struct igmp_source *source)
 {
@@ -873,6 +845,7 @@ void igmp_source_forward_start(struct igmp_source *source)
     int ret = 0;
     struct pim_nexthop_cache out_pnc;
     struct pim_nexthop nexthop;
+    struct pim_upstream *up = NULL;
 
     if (!pim_rp_set_upstream_addr (&vif_source, source->source_addr, sg.grp))
       return;
@@ -883,20 +856,23 @@ void igmp_source_forward_start(struct igmp_source *source)
     nht_p.u.prefix4 = vif_source;
     memset (&out_pnc, 0, sizeof (struct pim_nexthop_cache));
 
+    src.family = AF_INET;
+    src.prefixlen = IPV4_MAX_BITLEN;
+    src.u.prefix4 = vif_source;   //RP or Src address
+    grp.family = AF_INET;
+    grp.prefixlen = IPV4_MAX_BITLEN;
+    grp.u.prefix4 = sg.grp;
+
     if ((ret = pim_find_or_track_nexthop (&nht_p, NULL, NULL, &out_pnc)) == 1)
       {
         if (out_pnc.nexthop_num)
           {
-            src.family = AF_INET;
-            src.prefixlen = IPV4_MAX_BITLEN;
-            src.u.prefix4 = vif_source;   //RP or Src address
-            grp.family = AF_INET;
-            grp.prefixlen = IPV4_MAX_BITLEN;
-            grp.u.prefix4 = sg.grp;
-            memset (&nexthop, 0, sizeof (nexthop));
+            up = pim_upstream_find (&sg);
+            memset (&nexthop, 0, sizeof (struct pim_nexthop));
+            if (up)
+              memcpy (&nexthop, &up->rpf.source_nexthop, sizeof (struct pim_nexthop));
             //Compute PIM RPF using Cached nexthop
-            pim_ecmp_nexthop_search (&out_pnc, &nexthop,
-                                  &src, &grp, 0);
+            pim_ecmp_nexthop_search (&out_pnc, &nexthop, &src, &grp, 0);
             if (nexthop.interface)
               input_iface_vif_index = pim_if_find_vifindex_by_ifindex (nexthop.interface->ifindex);
           }
@@ -914,7 +890,7 @@ void igmp_source_forward_start(struct igmp_source *source)
           }
       }
     else
-      input_iface_vif_index = fib_lookup_if_vif_index(vif_source);
+      input_iface_vif_index = pim_ecmp_fib_lookup_if_vif_index(vif_source, &src, &grp);
 
     if (PIM_DEBUG_ZEBRA)
       {
@@ -1095,7 +1071,6 @@ void pim_forward_start(struct pim_ifchannel *ch)
       struct prefix nht_p, src, grp;
       int ret = 0;
       struct pim_nexthop_cache out_pnc;
-      struct pim_nexthop nexthop;
 
       /* Register addr with Zebra NHT */
       nht_p.family = AF_INET;
@@ -1106,8 +1081,7 @@ void pim_forward_start(struct pim_ifchannel *ch)
       grp.u.prefix4 = up->sg.grp;
       memset (&out_pnc, 0, sizeof (struct pim_nexthop_cache));
 
-      if ((ret =
-                  pim_find_or_track_nexthop (&nht_p, NULL, NULL, &out_pnc)) == 1)
+      if ((ret = pim_find_or_track_nexthop (&nht_p, NULL, NULL, &out_pnc)) == 1)
         {
           if (out_pnc.nexthop_num)
             {
@@ -1117,11 +1091,14 @@ void pim_forward_start(struct pim_ifchannel *ch)
               grp.family = AF_INET;
               grp.prefixlen = IPV4_MAX_BITLEN;
               grp.u.prefix4 = up->sg.grp;
-              memset (&nexthop, 0, sizeof (nexthop));
               //Compute PIM RPF using Cached nexthop
-              pim_ecmp_nexthop_search (&out_pnc, &nexthop, &src, &grp, 0);
-              input_iface_vif_index =
-                  pim_if_find_vifindex_by_ifindex (nexthop.interface->ifindex);
+              if (pim_ecmp_nexthop_search (&out_pnc, &up->rpf.source_nexthop, &src, &grp, 0) == 0)
+                input_iface_vif_index = pim_if_find_vifindex_by_ifindex (up->rpf.source_nexthop.interface->ifindex);
+              else
+                {
+                  if (PIM_DEBUG_TRACE)
+                    zlog_debug ("%s: Nexthop selection failed for %s ", __PRETTY_FUNCTION__, up->sg_str);
+                }
             }
           else
             {
@@ -1137,7 +1114,7 @@ void pim_forward_start(struct pim_ifchannel *ch)
             }
         }
       else
-          input_iface_vif_index = fib_lookup_if_vif_index (up->upstream_addr);
+        input_iface_vif_index = pim_ecmp_fib_lookup_if_vif_index(up->upstream_addr, &src, &grp);
 
       if (input_iface_vif_index < 1)
         {
@@ -1153,8 +1130,10 @@ void pim_forward_start(struct pim_ifchannel *ch)
         }
       if (PIM_DEBUG_TRACE)
         {
-          zlog_debug ("%s: NHT entry %s update channel_oil vif_index %d ",
-                      __PRETTY_FUNCTION__, up->sg_str, input_iface_vif_index);
+          struct interface *in_intf = pim_if_find_by_vif_index (input_iface_vif_index);
+          zlog_debug ("%s: Update channel_oil IIF %s VIFI %d entry %s ",
+                      __PRETTY_FUNCTION__, in_intf ? in_intf->name : "NIL",
+                      input_iface_vif_index, up->sg_str);
         }
       up->channel_oil = pim_channel_oil_add (&up->sg, input_iface_vif_index);
       if (!up->channel_oil)