From 227ce76ab61a3c7d88f98ce3fc6407373dc4517b Mon Sep 17 00:00:00 2001
From: "Anand H. Krishnan" <anandhk@juniper.net>
Date: Tue, 21 Jul 2015 12:30:29 +0530
Subject: [PATCH] Make deletion of nexthop an asynchronous operation

Currently, deletion of nexthop happens in the context of the vrouter
agent as a synchronous operation. To prevent traffic from accessing
invalid memory (that was a nexthop before deletion), we delay the
memory release using RCUs - the synchronous flavor of RCU.

In a scaled setup the synchronous RCU version can result in a
significant time spent doing delays, since for each delete the process
will delete, wait and then free the memory.

To get more useful work done in the time that is spent on wait, we
will make use of the asynchronous RCU, which will free agent to do
other work.

Change-Id: Ibbbef8bfeaaf40d3e087820d8121f99f9430f63b
Partial-BUG: #1465195
---
 dp-core/vr_interface.c |  2 +-
 dp-core/vr_nexthop.c   | 79 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 67 insertions(+), 14 deletions(-)

diff --git a/dp-core/vr_interface.c b/dp-core/vr_interface.c
index d4f728dd8..0039b2936 100644
--- a/dp-core/vr_interface.c
+++ b/dp-core/vr_interface.c
@@ -1329,7 +1329,7 @@ vif_free(struct vr_interface *vif)
 void
 vrouter_put_interface(struct vr_interface *vif)
 {
-    if (!--vif->vif_users)
+    if (!__sync_sub_and_fetch(&vif->vif_users, 1))
         vif_free(vif);
 
     return;
diff --git a/dp-core/vr_nexthop.c b/dp-core/vr_nexthop.c
index 553644ddc..87f587059 100644
--- a/dp-core/vr_nexthop.c
+++ b/dp-core/vr_nexthop.c
@@ -72,10 +72,59 @@ __vrouter_set_nexthop(struct vrouter *router, unsigned int index,
     return 0;
 }
 
+static void
+vrouter_free_nexthop(struct vr_nexthop *nh)
+{
+    if (nh->nh_type == NH_COMPOSITE) {
+        if (nh->nh_component_nh) {
+            vr_free(nh->nh_component_nh, VR_NEXTHOP_COMPONENT_OBJECT);
+            nh->nh_component_nh = NULL;
+        }
+    }
+
+    if (nh->nh_dev) {
+        vrouter_put_interface(nh->nh_dev);
+    }
+
+    vr_free(nh, VR_NEXTHOP_OBJECT);
+    return;
+}
+
+static void
+vrouter_free_nexthop_cb(struct vrouter *router, void *data)
+{
+    struct vr_defer_data *vdd = (struct vr_defer_data *)data;
+
+    if (!vdd)
+        return;
+
+    vrouter_free_nexthop((struct vr_nexthop *)vdd->vdd_data);
+    return;
+}
+
+static int
+vrouter_free_nexthop_defer(struct vr_nexthop *nh)
+{
+    struct vr_defer_data *defer;
+
+    defer = vr_get_defer_data(sizeof(*defer));
+    if (!defer)
+        return -ENOMEM;
+
+    defer->vdd_data = nh;
+    vr_defer(nh->nh_router, vrouter_free_nexthop_cb, (void *)defer);
+
+    return 0;
+}
+
 void
 vrouter_put_nexthop(struct vr_nexthop *nh)
 {
-    int i;
+    int i, component_cnt;
+    struct vr_nexthop *cnh;
+
+    if (!nh)
+        return;
 
     /* This function might get invoked with zero ref_cnt */
     if (nh->nh_users) {
@@ -83,23 +132,27 @@ vrouter_put_nexthop(struct vr_nexthop *nh)
     }
 
     if (!nh->nh_users ) {
-
-        if (!vr_not_ready)
-            vr_delay_op();
-
         /* If composite de-ref the internal nexthops */
         if (nh->nh_type == NH_COMPOSITE) {
-            for (i = 0; i < nh->nh_component_cnt; i++) {
-                if (nh->nh_component_nh[i].cnh)
-                    vrouter_put_nexthop(nh->nh_component_nh[i].cnh);
+            component_cnt = nh->nh_component_cnt;
+            nh->nh_component_cnt = 0;
+            for (i = 0; i < component_cnt; i++) {
+                if (nh->nh_component_nh[i].cnh) {
+                    cnh = nh->nh_component_nh[i].cnh;
+                    nh->nh_component_nh[i].cnh = NULL;
+                    vrouter_put_nexthop(cnh);
+                }
             }
-
-            vr_free(nh->nh_component_nh, VR_NEXTHOP_COMPONENT_OBJECT);
         }
-        if (nh->nh_dev) {
-            vrouter_put_interface(nh->nh_dev);
+
+        if (vr_not_ready) {
+            vrouter_free_nexthop(nh);
+        } else {
+            if (vrouter_free_nexthop_defer(nh)) {
+                vr_delay_op();
+                vrouter_free_nexthop(nh);
+            }
         }
-        vr_free(nh, VR_NEXTHOP_OBJECT);
     }
 
     return;