Merge "Hold packets while doing ECMP resolution" into R3.1

dp-core/vr_flow.c

@@ -899,6 +899,11 @@ vr_trap_flow(struct vrouter *router, struct vr_flow_entry *fe,
     vr_preset(npkt);
 
     switch (fe->fe_flags & VR_FLOW_FLAG_TRAP_MASK) {
+    case VR_FLOW_FLAG_TRAP_ECMP:
+        trap_reason = AGENT_TRAP_ECMP_RESOLVE;
+        fe->fe_flags &= ~VR_FLOW_FLAG_TRAP_ECMP;
+        break;
+
     default:
         /*
          * agent needs a method to identify new flows from existing flows.
@@ -914,20 +919,20 @@ vr_trap_flow(struct vrouter *router, struct vr_flow_entry *fe,
             trap_reason = AGENT_TRAP_FLOW_ACTION_HOLD;
         }
 
-        ta.vfta_index = index;
-        if ((fe->fe_type == VP_TYPE_IP) || (fe->fe_type == VP_TYPE_IP6))
-            ta.vfta_nh_index = fe->fe_key.flow_nh_id;
-        if (stats) {
-            ta.vfta_stats = *stats;
-        } else {
-            ta.vfta_stats = fe->fe_stats;
-        }
-
-        ta.vfta_gen_id = fe->fe_gen_id;
-
         break;
     }
 
+    ta.vfta_index = index;
+    if ((fe->fe_type == VP_TYPE_IP) || (fe->fe_type == VP_TYPE_IP6))
+        ta.vfta_nh_index = fe->fe_key.flow_nh_id;
+    if (stats) {
+        ta.vfta_stats = *stats;
+    } else {
+        ta.vfta_stats = fe->fe_stats;
+    }
+
+    ta.vfta_gen_id = fe->fe_gen_id;
+
     return vr_trap(npkt, fe->fe_vrf, trap_reason, &ta);
 }
 
@@ -961,6 +966,89 @@ vr_do_flow_action(struct vrouter *router, struct vr_flow_entry *fe,
     return vr_flow_action(router, fe, index, pkt, fmd);
 }
 
+int
+vr_flow_hold(struct vrouter *router, unsigned int fe_index,
+        struct vr_packet *pkt, struct vr_forwarding_md *fmd,
+        unsigned int trap_reason)
+{
+    bool modified = false, reevaluate = false;
+    int ret = 0;
+    unsigned int i;
+
+    struct vr_flow_entry *fe = vr_flow_get_entry(router, fe_index);
+
+    if (!fe) {
+        ret = -EINVAL;
+        goto exit_hold;
+    }
+
+    /* try for some number of times */
+    for (i = 0; i < 10; i++) {
+        if ((modified = vr_flow_start_modify(router, fe))) {
+            break;
+        }
+    }
+
+    if (!modified) {
+        ret = -EBUSY;
+        goto exit_hold;
+    }
+
+    /*
+     * to take care of the cases where the ecmp index changes
+     * before the packet is put on the hold queue. For e.g.:
+     * the case of second packet, where by the time the second
+     * packet came here, agent had already decided to change
+     * the index based on the information from the first packet.
+     * It so happened that the change to the flow entry did
+     * not happen when the nexthop code checked for the index,
+     * but by the time the packet reached here, agent had
+     * already made the change.
+     */
+    if (trap_reason == AGENT_TRAP_ECMP_RESOLVE) {
+        if (fmd->fmd_ecmp_nh_index != fe->fe_ecmp_nh_index) {
+            fmd->fmd_ecmp_nh_index = fe->fe_ecmp_nh_index;
+            reevaluate = true;
+            goto exit_modified;
+        }
+    }
+
+    if (!fe->fe_hold_list) {
+        switch (trap_reason) {
+        case AGENT_TRAP_ECMP_RESOLVE:
+            fe->fe_flags |= VR_FLOW_FLAG_TRAP_ECMP;
+            break;
+
+        default:
+            break;
+        }
+
+        fe->fe_hold_list = vr_zalloc(sizeof(struct vr_flow_queue),
+                VR_FLOW_QUEUE_OBJECT);
+        if (!fe->fe_hold_list) {
+            fe->fe_flags &= ~VR_FLOW_FLAG_TRAP_ECMP;
+            ret = -ENOMEM;
+            goto exit_modified;
+        }
+
+    }
+
+    vr_enqueue_flow(router, fe, pkt, fe_index, &fe->fe_stats, fmd);
+
+exit_modified:
+    vr_flow_stop_modify(router, fe);
+    if (reevaluate) {
+        vr_reinject_packet(pkt, fmd);
+    }
+
+exit_hold:
+    if (ret) {
+        vr_pfree(pkt, VP_DROP_FLOW_UNUSABLE);
+    }
+
+    return ret;
+}
+
 static unsigned int
 vr_flow_table_hold_count(struct vrouter *router)
 {

dp-core/vr_nexthop.c

@@ -674,11 +674,13 @@ nh_composite_ecmp(struct vr_packet *pkt, struct vr_nexthop *nh,
     if (stats)
         stats->vrf_ecmp_composites++;
 
-    if (!fmd || fmd->fmd_ecmp_nh_index >= (short)nh->nh_component_cnt)
+    if (!fmd)
         goto drop;
 
     if (fmd->fmd_ecmp_nh_index >= 0) {
-        member_nh = nh->nh_component_nh[fmd->fmd_ecmp_nh_index].cnh;
+        if (fmd->fmd_ecmp_nh_index < nh->nh_component_cnt) {
+            member_nh = nh->nh_component_nh[fmd->fmd_ecmp_nh_index].cnh;
+        }
     } else if (fmd->fmd_flow_index < 0) {
         member_nh = nh_composite_ecmp_select_nh(pkt, nh, fmd);
     }
@@ -688,7 +690,8 @@ nh_composite_ecmp(struct vr_packet *pkt, struct vr_nexthop *nh,
             vr_pfree(pkt, VP_DROP_INVALID_NH);
             return 0;
         } else {
-            vr_trap(pkt, fmd->fmd_dvrf, AGENT_TRAP_ECMP_RESOLVE, &fmd->fmd_flow_index);
+            vr_flow_hold(nh->nh_router, fmd->fmd_flow_index, pkt, fmd,
+                    AGENT_TRAP_ECMP_RESOLVE);
             return 0;
         }
     }

include/vr_flow.h

@@ -402,6 +402,8 @@ extern void vr_flow_exit(struct vrouter *, bool);
 
 extern bool vr_flow_forward(struct vrouter *,
         struct vr_packet *, struct vr_forwarding_md *);
+extern int vr_flow_hold(struct vrouter *, unsigned int,
+        struct vr_packet *, struct vr_forwarding_md *, unsigned int);
 
 void *vr_flow_get_va(struct vrouter *, uint64_t);