From 62d9b035e3d7c8e446beac7b37ddb4678f10671a Mon Sep 17 00:00:00 2001 From: Praveen K V Date: Sat, 28 May 2016 17:11:55 +0530 Subject: [PATCH] Introspect enhancements to debug memory utilization 1. Update introspect to show max-flows and number of entries in free-list 2. Fix for free-list grow request going to wrong flow-table Conflicts: src/vnsw/agent/pkt/flow_event.h Change-Id: I631f6ec1535a07cc8cd12af6e7074eb5fe5f70cb Partial-Bug: #1571163 --- src/ksync/ksync_tx_queue.cc | 3 ++- src/vnsw/agent/cmn/agent_stats.h | 18 +++++++++++++++++- src/vnsw/agent/pkt/flow_entry.cc | 1 + src/vnsw/agent/pkt/flow_event.h | 7 ++++++- src/vnsw/agent/pkt/flow_proto.cc | 5 +++-- src/vnsw/agent/pkt/flow_proto.h | 2 +- src/vnsw/agent/pkt/flow_table.cc | 5 +++-- src/vnsw/agent/pkt/flow_table.h | 1 + src/vnsw/agent/pkt/pkt.sandesh | 4 +++- src/vnsw/agent/pkt/pkt_sandesh_flow.cc | 2 ++ .../agent/vrouter/ksync/flowtable_ksync.cc | 3 +-- 11 files changed, 40 insertions(+), 11 deletions(-) diff --git a/src/ksync/ksync_tx_queue.cc b/src/ksync/ksync_tx_queue.cc index d92f03fd33b..351fbbf8e31 100644 --- a/src/ksync/ksync_tx_queue.cc +++ b/src/ksync/ksync_tx_queue.cc @@ -40,7 +40,8 @@ KSyncTxQueue::KSyncTxQueue(KSyncSock *sock) : dequeues_(0), write_events_(0), read_events_(0), - busy_time_(0) { + busy_time_(0), + measure_busy_time_(false) { queue_len_ = 0; shutdown_ = false; } diff --git a/src/vnsw/agent/cmn/agent_stats.h b/src/vnsw/agent/cmn/agent_stats.h index 92963814427..8e1f2132b43 100644 --- a/src/vnsw/agent/cmn/agent_stats.h +++ b/src/vnsw/agent/cmn/agent_stats.h @@ -8,6 +8,7 @@ #define vnsw_agent_stats_hpp #include +#include typedef boost::function FlowCountFn; class AgentStats { @@ -19,6 +20,7 @@ class AgentStats { sandesh_http_sessions_(0U), nh_count_(0U), pkt_exceptions_(0U), pkt_invalid_agent_hdr_(0U), pkt_invalid_interface_(0U), pkt_no_handler_(0U), pkt_fragments_dropped_(0U), pkt_dropped_(0U), + max_flow_count_(0), flow_created_(0U), flow_aged_(0U), flow_active_(0U), flow_drop_due_to_max_limit_(0), flow_drop_due_to_linklocal_limit_(0), prev_flow_created_(0U), prev_flow_aged_(0U), @@ -30,6 +32,7 @@ class AgentStats { out_tpkts_(0U), out_bytes_(0U) { assert(singleton_ == NULL); singleton_ = this; + flow_count_ = 0; } virtual ~AgentStats() {singleton_ = NULL;} @@ -61,9 +64,20 @@ class AgentStats { void incr_sandesh_http_sessions() {sandesh_http_sessions_++;} uint32_t sandesh_http_sessions() const {return sandesh_http_sessions_;} - void incr_flow_created() {flow_created_++;} + void incr_flow_created() { + flow_created_++; + uint32_t count = flow_count_.fetch_and_increment(); + if (count > max_flow_count_) + max_flow_count_ = count + 1; + } + void decr_flow_count() { + flow_count_--; + } + uint64_t flow_created() const {return flow_created_;} + uint64_t max_flow_count() const {return max_flow_count_;} + void incr_flow_aged() {flow_aged_++;} uint64_t flow_aged() const {return flow_aged_;} @@ -185,6 +199,8 @@ class AgentStats { uint64_t pkt_dropped_; // Flow stats + tbb::atomic flow_count_; + uint32_t max_flow_count_; uint64_t flow_created_; uint64_t flow_aged_; uint64_t flow_active_; diff --git a/src/vnsw/agent/pkt/flow_entry.cc b/src/vnsw/agent/pkt/flow_entry.cc index 2a2a66f348b..9c63e42c4e7 100644 --- a/src/vnsw/agent/pkt/flow_entry.cc +++ b/src/vnsw/agent/pkt/flow_entry.cc @@ -484,6 +484,7 @@ void intrusive_ptr_release(FlowEntry *fe) { flow_table->flow_entry_map_.find(fe->key()); assert(it != flow_table->flow_entry_map_.end()); flow_table->flow_entry_map_.erase(it); + flow_table->agent()->stats()->decr_flow_count(); } flow_table->free_list()->Free(fe); } diff --git a/src/vnsw/agent/pkt/flow_event.h b/src/vnsw/agent/pkt/flow_event.h index 369269c347b..7abedbc5c6a 100644 --- a/src/vnsw/agent/pkt/flow_event.h +++ b/src/vnsw/agent/pkt/flow_event.h @@ -71,10 +71,15 @@ class FlowEvent { FlowEvent(Event event, FlowEntry *flow) : event_(event), flow_(flow), pkt_info_(), db_entry_(NULL), - gen_id_(0), flow_handle_(FlowEntry::kInvalidFlowHandle), table_index_(0) { } + FlowEvent(Event event, uint32_t table_index) : + event_(event), flow_(NULL), pkt_info_(), db_entry_(NULL), + gen_id_(0), flow_handle_(FlowEntry::kInvalidFlowHandle), + table_index_(table_index) { + } + FlowEvent(Event event, FlowEntry *flow, uint32_t flow_handle, uint8_t gen_id) : event_(event), flow_(flow), pkt_info_(), db_entry_(NULL), diff --git a/src/vnsw/agent/pkt/flow_proto.cc b/src/vnsw/agent/pkt/flow_proto.cc index 6d7f195e3ae..08a2851acbb 100644 --- a/src/vnsw/agent/pkt/flow_proto.cc +++ b/src/vnsw/agent/pkt/flow_proto.cc @@ -27,6 +27,7 @@ FlowProto::FlowProto(Agent *agent, boost::asio::io_service &io) : agent->params()->flow_task_latency_limit(), 16), use_vrouter_hash_(false), ipv4_trace_filter_(), ipv6_trace_filter_(), stats_() { + linklocal_flow_count_ = 0; agent->SetFlowProto(this); set_trace(false); uint16_t table_count = agent->flow_thread_count(); @@ -544,8 +545,8 @@ void FlowProto::CreateAuditEntry(const FlowKey &key, uint32_t flow_handle, } -void FlowProto::GrowFreeListRequest(const FlowKey &key, FlowTable *table) { - EnqueueFlowEvent(new FlowEvent(FlowEvent::GROW_FREE_LIST, key, false, +void FlowProto::GrowFreeListRequest(FlowTable *table) { + EnqueueFlowEvent(new FlowEvent(FlowEvent::GROW_FREE_LIST, table->table_index())); return; } diff --git a/src/vnsw/agent/pkt/flow_proto.h b/src/vnsw/agent/pkt/flow_proto.h index 07ff5d6d524..804ccf913a3 100644 --- a/src/vnsw/agent/pkt/flow_proto.h +++ b/src/vnsw/agent/pkt/flow_proto.h @@ -84,7 +84,7 @@ class FlowProto : public Proto { bool FlowUpdateHandler(FlowEvent *req); bool FlowDeleteHandler(FlowEvent *req, FlowTable *table); bool FlowKSyncMsgHandler(FlowEvent *req, FlowTable *table); - void GrowFreeListRequest(const FlowKey &key, FlowTable *table); + void GrowFreeListRequest(FlowTable *table); void KSyncEventRequest(KSyncEntry *ksync_entry, KSyncEntry::KSyncEvent event, uint32_t flow_handle, uint8_t gen_id, int ksync_error, diff --git a/src/vnsw/agent/pkt/flow_table.cc b/src/vnsw/agent/pkt/flow_table.cc index 7164491d1df..1acd589c660 100644 --- a/src/vnsw/agent/pkt/flow_table.cc +++ b/src/vnsw/agent/pkt/flow_table.cc @@ -849,7 +849,7 @@ void FlowTable::GrowFreeList() { FlowEntryFreeList::FlowEntryFreeList(FlowTable *table) : table_(table), max_count_(0), grow_pending_(false), total_alloc_(0), - total_free_(0), free_list_() { + total_free_(0), free_list_(), grow_count_(0) { uint32_t count = kInitCount; if (table->agent()->test_mode()) { count = kTestInitCount; @@ -877,6 +877,7 @@ void FlowEntryFreeList::Grow() { if (free_list_.size() >= kMinThreshold) return; + grow_count_++; for (uint32_t i = 0; i < kGrowSize; i++) { free_list_.push_back(*new FlowEntry(table_)); max_count_++; @@ -898,7 +899,7 @@ FlowEntry *FlowEntryFreeList::Allocate(const FlowKey &key) { if (grow_pending_ == false && free_list_.size() < kMinThreshold) { grow_pending_ = true; FlowProto *proto = table_->agent()->pkt()->get_flow_proto(); - proto->GrowFreeListRequest(key, table_); + proto->GrowFreeListRequest(table_); } flow->Reset(key); total_alloc_++; diff --git a/src/vnsw/agent/pkt/flow_table.h b/src/vnsw/agent/pkt/flow_table.h index ce0b6b8a178..d77771a5780 100644 --- a/src/vnsw/agent/pkt/flow_table.h +++ b/src/vnsw/agent/pkt/flow_table.h @@ -104,6 +104,7 @@ class FlowEntryFreeList { uint64_t total_alloc_; uint64_t total_free_; FreeList free_list_; + uint64_t grow_count_; DISALLOW_COPY_AND_ASSIGN(FlowEntryFreeList); }; diff --git a/src/vnsw/agent/pkt/pkt.sandesh b/src/vnsw/agent/pkt/pkt.sandesh index 40d9d296eda..8a46d91b9fc 100644 --- a/src/vnsw/agent/pkt/pkt.sandesh +++ b/src/vnsw/agent/pkt/pkt.sandesh @@ -429,6 +429,7 @@ struct SandeshFlowTableInfo { 2: u32 count; 3: u64 total_add; 4: u64 total_del; + 5: u64 freelist_count; } /** @@ -438,7 +439,8 @@ response sandesh SandeshFlowTableInfoResp { 1: u64 flow_count; 2: u64 total_added; 3: u64 total_deleted; - 4: list table_list; + 4: u64 max_flows; + 5: list table_list; } /** diff --git a/src/vnsw/agent/pkt/pkt_sandesh_flow.cc b/src/vnsw/agent/pkt/pkt_sandesh_flow.cc index 6b2eeed5edb..74f3bc646cd 100644 --- a/src/vnsw/agent/pkt/pkt_sandesh_flow.cc +++ b/src/vnsw/agent/pkt/pkt_sandesh_flow.cc @@ -623,6 +623,7 @@ void SandeshFlowTableInfoRequest::HandleRequest() const { SandeshFlowTableInfoResp *resp = new SandeshFlowTableInfoResp(); resp->set_flow_count(proto->FlowCount()); resp->set_total_added(agent->stats()->flow_created()); + resp->set_max_flows(agent->stats()->max_flow_count()); resp->set_total_deleted(agent->stats()->flow_aged()); std::vector info_list; for (uint16_t i = 0; i < proto->flow_table_count(); i++) { @@ -632,6 +633,7 @@ void SandeshFlowTableInfoRequest::HandleRequest() const { info.set_count(table->Size()); info.set_total_add(table->free_list()->total_alloc()); info.set_total_del(table->free_list()->total_free()); + info.set_freelist_count(table->free_list()->free_count()); info_list.push_back(info); } resp->set_table_list(info_list); diff --git a/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc b/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc index dbccce2db91..f9c8c8fe74f 100644 --- a/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc +++ b/src/vnsw/agent/vrouter/ksync/flowtable_ksync.cc @@ -684,8 +684,7 @@ FlowTableKSyncEntry *KSyncFlowEntryFreeList::Allocate(const KSyncEntry *key) { if (grow_pending_ == false && free_list_.size() < kMinThreshold) { grow_pending_ = true; FlowProto *proto = object_->ksync()->agent()->pkt()->get_flow_proto(); - proto->GrowFreeListRequest(flow_key->flow_entry()->key(), - flow_key->flow_entry()->flow_table()); + proto->GrowFreeListRequest(flow_key->flow_entry()->flow_table()); } // Do post allocation initialization