From eb0dec8b4c5e46f53944f11b516d1b42b71389f7 Mon Sep 17 00:00:00 2001 From: Praveen K V Date: Sat, 28 May 2016 17:11:55 +0530 Subject: [PATCH] Introspect enhancements to debug memory utilization 1. Update introspect to show max-flows and number of entries in free-list 2. Fix for free-list grow request going to wrong flow-table Change-Id: I631f6ec1535a07cc8cd12af6e7074eb5fe5f70cb Partial-Bug: #1571163 --- src/ksync/ksync_tx_queue.cc | 3 ++- src/vnsw/agent/cmn/agent_stats.h | 18 +++++++++++++++++- src/vnsw/agent/pkt/flow_entry.cc | 1 + src/vnsw/agent/pkt/flow_event.h | 6 ++++++ src/vnsw/agent/pkt/flow_proto.cc | 5 +++-- src/vnsw/agent/pkt/flow_proto.h | 2 +- src/vnsw/agent/pkt/flow_table.cc | 5 +++-- src/vnsw/agent/pkt/flow_table.h | 1 + src/vnsw/agent/pkt/pkt.sandesh | 4 +++- src/vnsw/agent/pkt/pkt_sandesh_flow.cc | 2 ++ .../agent/vrouter/ksync/flowtable_ksync.cc | 3 +-- 11 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/ksync/ksync_tx_queue.cc b/src/ksync/ksync_tx_queue.cc index d92f03fd33b..351fbbf8e31 100644 --- a/src/ksync/ksync_tx_queue.cc +++ b/src/ksync/ksync_tx_queue.cc @@ -40,7 +40,8 @@ KSyncTxQueue::KSyncTxQueue(KSyncSock *sock) : dequeues_(0), write_events_(0), read_events_(0), - busy_time_(0) { + busy_time_(0), + measure_busy_time_(false) { queue_len_ = 0; shutdown_ = false; } diff --git a/src/vnsw/agent/cmn/agent_stats.h b/src/vnsw/agent/cmn/agent_stats.h index 3faa8ad143b..56b81906b5b 100644 --- a/src/vnsw/agent/cmn/agent_stats.h +++ b/src/vnsw/agent/cmn/agent_stats.h @@ -8,6 +8,7 @@ #define vnsw_agent_stats_hpp #include +#include typedef boost::function FlowCountFn; class AgentStats { @@ -20,6 +21,7 @@ class AgentStats { sandesh_http_sessions_(0U), nh_count_(0U), pkt_exceptions_(0U), pkt_invalid_agent_hdr_(0U), pkt_invalid_interface_(0U), pkt_no_handler_(0U), pkt_fragments_dropped_(0U), pkt_dropped_(0U), + max_flow_count_(0), flow_created_(0U), flow_aged_(0U), flow_active_(0U), flow_drop_due_to_max_limit_(0), flow_drop_due_to_linklocal_limit_(0), prev_flow_created_(0U), prev_flow_aged_(0U), @@ -31,6 +33,7 @@ class AgentStats { out_tpkts_(0U), out_bytes_(0U) { assert(singleton_ == NULL); singleton_ = this; + flow_count_ = 0; } virtual ~AgentStats() {singleton_ = NULL;} @@ -67,9 +70,20 @@ class AgentStats { void incr_sandesh_http_sessions() {sandesh_http_sessions_++;} uint32_t sandesh_http_sessions() const {return sandesh_http_sessions_;} - void incr_flow_created() {flow_created_++;} + void incr_flow_created() { + flow_created_++; + uint32_t count = flow_count_.fetch_and_increment(); + if (count > max_flow_count_) + max_flow_count_ = count + 1; + } + void decr_flow_count() { + flow_count_--; + } + uint64_t flow_created() const {return flow_created_;} + uint64_t max_flow_count() const {return max_flow_count_;} + void incr_flow_aged() {flow_aged_++;} uint64_t flow_aged() const {return flow_aged_;} @@ -192,6 +206,8 @@ class AgentStats { uint64_t pkt_dropped_; // Flow stats + tbb::atomic flow_count_; + uint32_t max_flow_count_; uint64_t flow_created_; uint64_t flow_aged_; uint64_t flow_active_; diff --git a/src/vnsw/agent/pkt/flow_entry.cc b/src/vnsw/agent/pkt/flow_entry.cc index dda6004e481..1511d2889ca 100644 --- a/src/vnsw/agent/pkt/flow_entry.cc +++ b/src/vnsw/agent/pkt/flow_entry.cc @@ -485,6 +485,7 @@ void intrusive_ptr_release(FlowEntry *fe) { flow_table->flow_entry_map_.find(fe->key()); assert(it != flow_table->flow_entry_map_.end()); flow_table->flow_entry_map_.erase(it); + flow_table->agent()->stats()->decr_flow_count(); } flow_table->free_list()->Free(fe); } diff --git a/src/vnsw/agent/pkt/flow_event.h b/src/vnsw/agent/pkt/flow_event.h index e31bda80633..d618312f0bb 100644 --- a/src/vnsw/agent/pkt/flow_event.h +++ b/src/vnsw/agent/pkt/flow_event.h @@ -75,6 +75,12 @@ class FlowEvent { flow_handle_(FlowEntry::kInvalidFlowHandle), table_index_(0) { } + FlowEvent(Event event, uint32_t table_index) : + event_(event), flow_(NULL), pkt_info_(), db_entry_(NULL), + gen_id_(0), evict_gen_id_(0), + flow_handle_(FlowEntry::kInvalidFlowHandle), table_index_(table_index) { + } + FlowEvent(Event event, FlowEntry *flow, uint32_t flow_handle, uint8_t gen_id) : event_(event), flow_(flow), pkt_info_(), db_entry_(NULL), diff --git a/src/vnsw/agent/pkt/flow_proto.cc b/src/vnsw/agent/pkt/flow_proto.cc index 58f6fcbe586..04ec6d7fea4 100644 --- a/src/vnsw/agent/pkt/flow_proto.cc +++ b/src/vnsw/agent/pkt/flow_proto.cc @@ -27,6 +27,7 @@ FlowProto::FlowProto(Agent *agent, boost::asio::io_service &io) : agent->params()->flow_task_latency_limit(), 16), use_vrouter_hash_(false), ipv4_trace_filter_(), ipv6_trace_filter_(), stats_() { + linklocal_flow_count_ = 0; agent->SetFlowProto(this); set_trace(false); uint16_t table_count = agent->flow_thread_count(); @@ -544,8 +545,8 @@ void FlowProto::CreateAuditEntry(const FlowKey &key, uint32_t flow_handle, } -void FlowProto::GrowFreeListRequest(const FlowKey &key, FlowTable *table) { - EnqueueFlowEvent(new FlowEvent(FlowEvent::GROW_FREE_LIST, key, false, +void FlowProto::GrowFreeListRequest(FlowTable *table) { + EnqueueFlowEvent(new FlowEvent(FlowEvent::GROW_FREE_LIST, table->table_index())); return; } diff --git a/src/vnsw/agent/pkt/flow_proto.h b/src/vnsw/agent/pkt/flow_proto.h index aed926cb0b3..4bb1756b642 100644 --- a/src/vnsw/agent/pkt/flow_proto.h +++ b/src/vnsw/agent/pkt/flow_proto.h @@ -85,7 +85,7 @@ class FlowProto : public Proto { bool FlowUpdateHandler(FlowEvent *req); bool FlowDeleteHandler(FlowEvent *req, FlowTable *table); bool FlowKSyncMsgHandler(FlowEvent *req, FlowTable *table); - void GrowFreeListRequest(const FlowKey &key, FlowTable *table); + void GrowFreeListRequest(FlowTable *table); void KSyncEventRequest(KSyncEntry *ksync_entry, KSyncEntry::KSyncEvent event, uint32_t flow_handle, uint8_t gen_id, int ksync_error, diff --git a/src/vnsw/agent/pkt/flow_table.cc b/src/vnsw/agent/pkt/flow_table.cc index d3765f8accc..0eff9952a52 100644 --- a/src/vnsw/agent/pkt/flow_table.cc +++ b/src/vnsw/agent/pkt/flow_table.cc @@ -852,7 +852,7 @@ void FlowTable::GrowFreeList() { FlowEntryFreeList::FlowEntryFreeList(FlowTable *table) : table_(table), max_count_(0), grow_pending_(false), total_alloc_(0), - total_free_(0), free_list_() { + total_free_(0), free_list_(), grow_count_(0) { uint32_t count = kInitCount; if (table->agent()->test_mode()) { count = kTestInitCount; @@ -880,6 +880,7 @@ void FlowEntryFreeList::Grow() { if (free_list_.size() >= kMinThreshold) return; + grow_count_++; for (uint32_t i = 0; i < kGrowSize; i++) { free_list_.push_back(*new FlowEntry(table_)); max_count_++; @@ -901,7 +902,7 @@ FlowEntry *FlowEntryFreeList::Allocate(const FlowKey &key) { if (grow_pending_ == false && free_list_.size() < kMinThreshold) { grow_pending_ = true; FlowProto *proto = table_->agent()->pkt()->get_flow_proto(); - proto->GrowFreeListRequest(key, table_); + proto->GrowFreeListRequest(table_); } flow->Reset(key); total_alloc_++; diff --git a/src/vnsw/agent/pkt/flow_table.h b/src/vnsw/agent/pkt/flow_table.h index 7729deaac77..4e94008cfd1 100644 --- a/src/vnsw/agent/pkt/flow_table.h +++ b/src/vnsw/agent/pkt/flow_table.h @@ -115,6 +115,7 @@ class FlowEntryFreeList { uint64_t total_alloc_; uint64_t total_free_; FreeList free_list_; + uint64_t grow_count_; DISALLOW_COPY_AND_ASSIGN(FlowEntryFreeList); }; diff --git a/src/vnsw/agent/pkt/pkt.sandesh b/src/vnsw/agent/pkt/pkt.sandesh index 40d9d296eda..8a46d91b9fc 100644 --- a/src/vnsw/agent/pkt/pkt.sandesh +++ b/src/vnsw/agent/pkt/pkt.sandesh @@ -429,6 +429,7 @@ struct SandeshFlowTableInfo { 2: u32 count; 3: u64 total_add; 4: u64 total_del; + 5: u64 freelist_count; } /** @@ -438,7 +439,8 @@ response sandesh SandeshFlowTableInfoResp { 1: u64 flow_count; 2: u64 total_added; 3: u64 total_deleted; - 4: list table_list; + 4: u64 max_flows; + 5: list table_list; } /** diff --git a/src/vnsw/agent/pkt/pkt_sandesh_flow.cc b/src/vnsw/agent/pkt/pkt_sandesh_flow.cc index e60eaff4882..6163264dd8f 100644 --- a/src/vnsw/agent/pkt/pkt_sandesh_flow.cc +++ b/src/vnsw/agent/pkt/pkt_sandesh_flow.cc @@ -623,6 +623,7 @@ void SandeshFlowTableInfoRequest::HandleRequest() const { SandeshFlowTableInfoResp *resp = new SandeshFlowTableInfoResp(); resp->set_flow_count(proto->FlowCount()); resp->set_total_added(agent->stats()->flow_created()); + resp->set_max_flows(agent->stats()->max_flow_count()); resp->set_total_deleted(agent->stats()->flow_aged()); std::vector info_list; for (uint16_t i = 0; i < proto->flow_table_count(); i++) { @@ -632,6 +633,7 @@ void SandeshFlowTableInfoRequest::HandleRequest() const { info.set_count(table->Size()); info.set_total_add(table->free_list()->total_alloc()); info.set_total_del(table->free_list()->total_free()); + info.set_freelist_count(table->free_list()->free_count()); info_list.push_back(info); } resp->set_table_list(info_list); diff --git a/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc b/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc index f4326783e2c..4d9079f377f 100644 --- a/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc +++ b/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc @@ -689,8 +689,7 @@ FlowTableKSyncEntry *KSyncFlowEntryFreeList::Allocate(const KSyncEntry *key) { if (grow_pending_ == false && free_list_.size() < kMinThreshold) { grow_pending_ = true; FlowProto *proto = object_->ksync()->agent()->pkt()->get_flow_proto(); - proto->GrowFreeListRequest(flow_key->flow_entry()->key(), - flow_key->flow_entry()->flow_table()); + proto->GrowFreeListRequest(flow_key->flow_entry()->flow_table()); } // Do post allocation initialization