Introspect enhancements to debug memory utilization

1. Update introspect to show max-flows and number of entries in
   free-list
2. Fix for free-list grow request going to wrong flow-table

Change-Id: I631f6ec1535a07cc8cd12af6e7074eb5fe5f70cb
Partial-Bug: #1571163

src/ksync/ksync_tx_queue.cc

@@ -40,7 +40,8 @@ KSyncTxQueue::KSyncTxQueue(KSyncSock *sock) :
     dequeues_(0),
     write_events_(0),
     read_events_(0),
-    busy_time_(0) {
+    busy_time_(0),
+    measure_busy_time_(false) {
     queue_len_ = 0;
     shutdown_ = false;
 }

src/vnsw/agent/cmn/agent_stats.h

@@ -8,6 +8,7 @@
 #define vnsw_agent_stats_hpp
 
 #include <stdint.h>
+#include <tbb/atomic.h>
 
 typedef boost::function<uint32_t()> FlowCountFn;
 class AgentStats {
@@ -20,6 +21,7 @@ class AgentStats {
         sandesh_http_sessions_(0U), nh_count_(0U), pkt_exceptions_(0U),
         pkt_invalid_agent_hdr_(0U), pkt_invalid_interface_(0U), 
         pkt_no_handler_(0U), pkt_fragments_dropped_(0U), pkt_dropped_(0U),
+        max_flow_count_(0),
         flow_created_(0U), flow_aged_(0U), flow_active_(0U),
         flow_drop_due_to_max_limit_(0), flow_drop_due_to_linklocal_limit_(0),
         prev_flow_created_(0U), prev_flow_aged_(0U),
@@ -31,6 +33,7 @@ class AgentStats {
         out_tpkts_(0U), out_bytes_(0U) {
         assert(singleton_ == NULL);
         singleton_ = this;
+        flow_count_ = 0;
     }
 
     virtual ~AgentStats() {singleton_ = NULL;}
@@ -67,9 +70,20 @@ class AgentStats {
     void incr_sandesh_http_sessions() {sandesh_http_sessions_++;}
     uint32_t sandesh_http_sessions() const {return sandesh_http_sessions_;}
 
-    void incr_flow_created() {flow_created_++;}
+    void incr_flow_created() {
+        flow_created_++;
+        uint32_t count = flow_count_.fetch_and_increment();
+        if (count > max_flow_count_)
+            max_flow_count_ = count + 1;
+    }
+    void decr_flow_count() {
+        flow_count_--;
+    }
+
     uint64_t flow_created() const {return flow_created_;}
 
+    uint64_t max_flow_count() const {return max_flow_count_;}
+
     void incr_flow_aged() {flow_aged_++;}
     uint64_t flow_aged() const {return flow_aged_;}
 
@@ -192,6 +206,8 @@ class AgentStats {
     uint64_t pkt_dropped_;
 
     // Flow stats
+    tbb::atomic<uint32_t> flow_count_;
+    uint32_t max_flow_count_;
     uint64_t flow_created_;
     uint64_t flow_aged_;
     uint64_t flow_active_;

src/vnsw/agent/pkt/flow_entry.cc

@@ -485,6 +485,7 @@ void intrusive_ptr_release(FlowEntry *fe) {
                 flow_table->flow_entry_map_.find(fe->key());
             assert(it != flow_table->flow_entry_map_.end());
             flow_table->flow_entry_map_.erase(it);
+            flow_table->agent()->stats()->decr_flow_count();
         }
         flow_table->free_list()->Free(fe);
     }

src/vnsw/agent/pkt/flow_event.h

@@ -75,6 +75,12 @@ class FlowEvent {
         flow_handle_(FlowEntry::kInvalidFlowHandle), table_index_(0) {
     }
 
+    FlowEvent(Event event, uint32_t table_index) :
+        event_(event), flow_(NULL), pkt_info_(), db_entry_(NULL),
+        gen_id_(0), evict_gen_id_(0),
+        flow_handle_(FlowEntry::kInvalidFlowHandle), table_index_(table_index) {
+    }
+
     FlowEvent(Event event, FlowEntry *flow, uint32_t flow_handle,
               uint8_t gen_id) :
         event_(event), flow_(flow), pkt_info_(), db_entry_(NULL),

src/vnsw/agent/pkt/flow_proto.cc

@@ -27,6 +27,7 @@ FlowProto::FlowProto(Agent *agent, boost::asio::io_service &io) :
                        agent->params()->flow_task_latency_limit(), 16),
     use_vrouter_hash_(false), ipv4_trace_filter_(), ipv6_trace_filter_(),
     stats_() {
+    linklocal_flow_count_ = 0;
     agent->SetFlowProto(this);
     set_trace(false);
     uint16_t table_count = agent->flow_thread_count();
@@ -544,8 +545,8 @@ void FlowProto::CreateAuditEntry(const FlowKey &key, uint32_t flow_handle,
 }
 
 
-void FlowProto::GrowFreeListRequest(const FlowKey &key, FlowTable *table) {
-    EnqueueFlowEvent(new FlowEvent(FlowEvent::GROW_FREE_LIST, key, false,
+void FlowProto::GrowFreeListRequest(FlowTable *table) {
+    EnqueueFlowEvent(new FlowEvent(FlowEvent::GROW_FREE_LIST,
                                    table->table_index()));
     return;
 }

src/vnsw/agent/pkt/flow_proto.h

@@ -85,7 +85,7 @@ class FlowProto : public Proto {
     bool FlowUpdateHandler(FlowEvent *req);
     bool FlowDeleteHandler(FlowEvent *req, FlowTable *table);
     bool FlowKSyncMsgHandler(FlowEvent *req, FlowTable *table);
-    void GrowFreeListRequest(const FlowKey &key, FlowTable *table);
+    void GrowFreeListRequest(FlowTable *table);
     void KSyncEventRequest(KSyncEntry *ksync_entry,
                            KSyncEntry::KSyncEvent event, uint32_t flow_handle,
                            uint8_t gen_id, int ksync_error,

src/vnsw/agent/pkt/flow_table.cc

@@ -852,7 +852,7 @@ void FlowTable::GrowFreeList() {
 
 FlowEntryFreeList::FlowEntryFreeList(FlowTable *table) :
     table_(table), max_count_(0), grow_pending_(false), total_alloc_(0),
-    total_free_(0), free_list_() {
+    total_free_(0), free_list_(), grow_count_(0) {
     uint32_t count = kInitCount;
     if (table->agent()->test_mode()) {
         count = kTestInitCount;
@@ -880,6 +880,7 @@ void FlowEntryFreeList::Grow() {
     if (free_list_.size() >= kMinThreshold)
         return;
 
+    grow_count_++;
     for (uint32_t i = 0; i < kGrowSize; i++) {
         free_list_.push_back(*new FlowEntry(table_));
         max_count_++;
@@ -901,7 +902,7 @@ FlowEntry *FlowEntryFreeList::Allocate(const FlowKey &key) {
     if (grow_pending_ == false && free_list_.size() < kMinThreshold) {
         grow_pending_ = true;
         FlowProto *proto = table_->agent()->pkt()->get_flow_proto();
-        proto->GrowFreeListRequest(key, table_);
+        proto->GrowFreeListRequest(table_);
     }
     flow->Reset(key);
     total_alloc_++;

src/vnsw/agent/pkt/flow_table.h

@@ -115,6 +115,7 @@ class FlowEntryFreeList {
     uint64_t total_alloc_;
     uint64_t total_free_;
     FreeList free_list_;
+    uint64_t grow_count_;
     DISALLOW_COPY_AND_ASSIGN(FlowEntryFreeList);
 };
 

src/vnsw/agent/pkt/pkt.sandesh

@@ -429,6 +429,7 @@ struct SandeshFlowTableInfo {
     2: u32 count;
     3: u64 total_add;
     4: u64 total_del;
+    5: u64 freelist_count;
 }
 
 /**
@@ -438,7 +439,8 @@ response sandesh SandeshFlowTableInfoResp {
     1: u64 flow_count;
     2: u64 total_added;
     3: u64 total_deleted;
-    4: list<SandeshFlowTableInfo> table_list;
+    4: u64 max_flows;
+    5: list<SandeshFlowTableInfo> table_list;
 }
 
 /**

src/vnsw/agent/pkt/pkt_sandesh_flow.cc

@@ -623,6 +623,7 @@ void SandeshFlowTableInfoRequest::HandleRequest() const {
     SandeshFlowTableInfoResp *resp = new SandeshFlowTableInfoResp();
     resp->set_flow_count(proto->FlowCount());
     resp->set_total_added(agent->stats()->flow_created());
+    resp->set_max_flows(agent->stats()->max_flow_count());
     resp->set_total_deleted(agent->stats()->flow_aged());
     std::vector<SandeshFlowTableInfo> info_list;
     for (uint16_t i = 0; i < proto->flow_table_count(); i++) {
@@ -632,6 +633,7 @@ void SandeshFlowTableInfoRequest::HandleRequest() const {
         info.set_count(table->Size());
         info.set_total_add(table->free_list()->total_alloc());
         info.set_total_del(table->free_list()->total_free());
+        info.set_freelist_count(table->free_list()->free_count());
         info_list.push_back(info);
     }
     resp->set_table_list(info_list);

src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc

@@ -689,8 +689,7 @@ FlowTableKSyncEntry *KSyncFlowEntryFreeList::Allocate(const KSyncEntry *key) {
     if (grow_pending_ == false && free_list_.size() < kMinThreshold) {
         grow_pending_ = true;
         FlowProto *proto = object_->ksync()->agent()->pkt()->get_flow_proto();
-        proto->GrowFreeListRequest(flow_key->flow_entry()->key(),
-                                   flow_key->flow_entry()->flow_table());
+        proto->GrowFreeListRequest(flow_key->flow_entry()->flow_table());
     }
 
     // Do post allocation initialization