From 53fb29020fcaabe4b3b9512e60a9edf1a8e67101 Mon Sep 17 00:00:00 2001
From: Divakar <ddivakar@juniper.net>
Date: Thu, 3 Mar 2016 11:02:13 +0530
Subject: [PATCH] Modify hentry flags before adding to free list

Currently when a flow entry is deleted, the corresponding hash entry is
kept at the head of the free list and after successfully inserting into
the list the entry's flags are reset. If any other core is
attempting to fetch a free entry, there is a likely chance that just
added entry will be given as free entry before the flags are reset as
this entry is at the head of the list. Because of this there is a race
condition for setting the flags between deletion of entry and additin of
entry. If deletion overwrites the addition flags, the Valid flags gets
overwritten. Because of this the entry would not be in free list but
still would not be considered a valid entry. Searching this entry by
index fails as this is not a valid entry.

As a fix, the flags while inserting the entry to free list are modified
before inserting so that there would not be any race condition.

Change-Id: Ia9334d313d2d444279704e84c5193de80ca7f238
closes-bug: #1552544
---
 dp-core/vr_htable.c | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/dp-core/vr_htable.c b/dp-core/vr_htable.c
index 40af6f703..4c1602b79 100644
--- a/dp-core/vr_htable.c
+++ b/dp-core/vr_htable.c
@@ -95,12 +95,14 @@ vr_htable_put_free_oentry(struct vr_htable *table, vr_hentry_t *ent)
 
     vr_hentry_t *tmp;
 
-    if (!table || !ent)
+    if (!table || !ent || ent->hentry_index < table->ht_hentries)
         return;
 
     if (ent->hentry_flags & VR_HENTRY_FLAG_IN_FREE_LIST)
         return;
 
+    ent->hentry_flags |= VR_HENTRY_FLAG_IN_FREE_LIST;
+
     tmp = NULL;
     do {
 
@@ -113,7 +115,6 @@ vr_htable_put_free_oentry(struct vr_htable *table, vr_hentry_t *ent)
         if (__sync_bool_compare_and_swap(&table->ht_free_oentry_head,
                                                             tmp, ent)) {
             (void)__sync_sub_and_fetch(&table->ht_used_oentries, 1);
-            ent->hentry_flags |= VR_HENTRY_FLAG_IN_FREE_LIST;
             return;
         }
 
@@ -160,23 +161,22 @@ vr_htable_get_hentry_by_index(vr_htable_t htable, unsigned int index)
 }
 
 static void
-vr_htable_hentry_invalidate(struct vr_htable *table, vr_hentry_t *ent)
+vr_htable_oentry_invalidate(struct vr_htable *table, vr_hentry_t *ent)
 {
-    if (!table || !ent)
+    if (!table || !ent || (ent->hentry_index < table->ht_hentries))
         return;
 
+    /* Clear all the flags except free list */
+    ent->hentry_flags &= VR_HENTRY_FLAG_IN_FREE_LIST;
     ent->hentry_bucket_index = VR_INVALID_HENTRY_INDEX;
+    ent->hentry_next_index = VR_INVALID_HENTRY_INDEX;
+    ent->hentry_next = NULL;
 
-    if (ent->hentry_index >= table->ht_hentries) {
-        ent->hentry_next_index = VR_INVALID_HENTRY_INDEX;
-        vr_htable_put_free_oentry(table, ent);
-    }
-
-    ent->hentry_flags &= VR_HENTRY_FLAG_IN_FREE_LIST;
+    vr_htable_put_free_oentry(table, ent);
 }
 
 static void
-__vr_htable_hentry_invalidate(struct vr_htable *table, vr_hentry_t *ent)
+__vr_htable_oentry_invalidate(struct vr_htable *table, vr_hentry_t *ent)
 {
     vr_hentry_t *prev, *head_ent;
 
@@ -205,7 +205,7 @@ __vr_htable_hentry_invalidate(struct vr_htable *table, vr_hentry_t *ent)
             }
         }
 
-        vr_htable_hentry_invalidate(table, ent);
+        vr_htable_oentry_invalidate(table, ent);
     }
 
     return;
@@ -224,7 +224,7 @@ vr_htable_hentry_defer_delete(struct vrouter *router, void *arg)
 
     ent = __vr_htable_get_hentry_by_index((vr_htable_t)table,
                                         defer_data->hd_index);
-    vr_htable_hentry_invalidate(table, ent);
+    vr_htable_oentry_invalidate(table, ent);
 
     return;
 }
@@ -318,7 +318,7 @@ vr_htable_hentry_scheduled_delete(void *arg)
                          vr_htable_hentry_defer_delete, (void *)defer_data);
                 }
             } else {
-                vr_htable_hentry_invalidate(table, ent);
+                vr_htable_oentry_invalidate(table, ent);
                 ent = next;
                 continue;
             }
@@ -345,17 +345,17 @@ vr_htable_release_hentry(vr_htable_t htable, vr_hentry_t *ent)
     if (!(ent->hentry_flags & VR_HENTRY_FLAG_VALID))
         return;
 
-    if (ent->hentry_index < table->ht_hentries) {
-        vr_htable_hentry_invalidate(table, ent);
+    /* Mark it as Invalid */
+    ent->hentry_flags &= ~VR_HENTRY_FLAG_VALID;
+
+    if (ent->hentry_index < table->ht_hentries)
         return;
-    }
 
     if (vr_not_ready) {
-        __vr_htable_hentry_invalidate(table, ent);
+        __vr_htable_oentry_invalidate(table, ent);
         return;
     }
 
-    ent->hentry_flags &= ~VR_HENTRY_FLAG_VALID;
     ent->hentry_flags |= VR_HENTRY_FLAG_DELETE_MARKED;
 
     head_ent = __vr_htable_get_hentry_by_index(htable, ent->hentry_bucket_index);