From 1adcf4607ef4c222e997b7d552541a45890934ef Mon Sep 17 00:00:00 2001 From: Prabhjot Singh Sethi Date: Mon, 4 Jul 2016 23:08:51 +0530 Subject: [PATCH] Set Vrouter to Drop New Flows on flow limit Once Flow Limit for a Virtual Machine is reached, set all the interfaces of the VM to drop new flows, avoiding the process of creating short flows and deleting eventually. VM will recover from this state once the number of flows for a VM falls below 90% of the configured flow limit Closes-Bug: 1595135 Change-Id: I6c357b39e78bfcc7be8b5db7220134b42d2bc595 --- src/vnsw/agent/cmn/agent.h | 4 ++ src/vnsw/agent/cmn/agent_db.h | 10 ++-- src/vnsw/agent/oper/agent.sandesh | 2 + src/vnsw/agent/oper/interface.cc | 1 + src/vnsw/agent/oper/vm.cc | 47 ++++++++++++++++++- src/vnsw/agent/oper/vm.h | 17 +++++-- src/vnsw/agent/oper/vm_interface.cc | 38 ++++++++++++--- src/vnsw/agent/oper/vm_interface.h | 17 ++++++- .../agent/pkt/test/test_pkt_flow_limits.cc | 11 ++++- .../agent/vrouter/ksync/interface_ksync.cc | 10 ++++ .../agent/vrouter/ksync/interface_ksync.h | 2 + 11 files changed, 139 insertions(+), 20 deletions(-) diff --git a/src/vnsw/agent/cmn/agent.h b/src/vnsw/agent/cmn/agent.h index 4d97436edd7..1364e051d14 100644 --- a/src/vnsw/agent/cmn/agent.h +++ b/src/vnsw/agent/cmn/agent.h @@ -53,6 +53,10 @@ typedef boost::intrusive_ptr VmEntryRef; typedef boost::intrusive_ptr VmEntryConstRef; void intrusive_ptr_release(const VmEntry* p); void intrusive_ptr_add_ref(const VmEntry* p); +typedef IntrusivePtrRef VmEntryBackRef; +typedef IntrusivePtrRef VmEntryConstBackRef; +void intrusive_ptr_add_back_ref(const IntrusiveReferrer ref, const VmEntry* p); +void intrusive_ptr_del_back_ref(const IntrusiveReferrer ref, const VmEntry* p); class VnEntry; typedef boost::intrusive_ptr VnEntryRef; diff --git a/src/vnsw/agent/cmn/agent_db.h b/src/vnsw/agent/cmn/agent_db.h index ee698688171..76add4caa59 100644 --- a/src/vnsw/agent/cmn/agent_db.h +++ b/src/vnsw/agent/cmn/agent_db.h @@ -39,14 +39,14 @@ class AgentRefCount { friend void intrusive_ptr_add_back_ref(const IntrusiveReferrer ref, const Derived* p) { const AgentRefCount *entry = (const AgentRefCount *) (p); - tbb::mutex::scoped_lock lock(entry->mutex_); + tbb::mutex::scoped_lock lock(entry->back_ref_mutex_); entry->back_ref_set_.insert(ref); } friend void intrusive_ptr_del_back_ref(const IntrusiveReferrer ref, const Derived* p) { const AgentRefCount *entry = (const AgentRefCount *) (p); - tbb::mutex::scoped_lock lock(entry->mutex_); + tbb::mutex::scoped_lock lock(entry->back_ref_mutex_); entry->back_ref_set_.erase(ref); } @@ -57,9 +57,11 @@ class AgentRefCount { AgentRefCount& operator=(const AgentRefCount&) { return *this; } virtual ~AgentRefCount() {assert(refcount_ == 0);}; void swap(AgentRefCount&) {}; -private: - mutable tbb::mutex mutex_; + + mutable tbb::mutex back_ref_mutex_; mutable std::set back_ref_set_; + +private: mutable tbb::atomic refcount_; }; diff --git a/src/vnsw/agent/oper/agent.sandesh b/src/vnsw/agent/oper/agent.sandesh index 0da8c709d84..0e7c9284efe 100644 --- a/src/vnsw/agent/oper/agent.sandesh +++ b/src/vnsw/agent/oper/agent.sandesh @@ -162,6 +162,7 @@ struct ItfSandeshData { 55: optional string service_health_check_ip; 56: optional string qos_config; //QOS config uuid 57: list alias_ip_list;// Alias IP List + 58: bool drop_new_flows; } /** @@ -232,6 +233,7 @@ struct VmSgUuid { struct VmSandeshData { 1: string uuid; // VM uuid 2: optional list sg_uuid_list; + 3: bool drop_new_flows; } /** diff --git a/src/vnsw/agent/oper/interface.cc b/src/vnsw/agent/oper/interface.cc index c39cbf6866e..bc03107dfb1 100644 --- a/src/vnsw/agent/oper/interface.cc +++ b/src/vnsw/agent/oper/interface.cc @@ -1063,6 +1063,7 @@ void Interface::SetItfSandeshData(ItfSandeshData &data) const { data.set_service_health_check_ip( vintf->service_health_check_ip().to_string()); + data.set_drop_new_flows(vintf->drop_new_flows()); break; } diff --git a/src/vnsw/agent/oper/vm.cc b/src/vnsw/agent/oper/vm.cc index d8dcaf5812d..fd00d737afc 100644 --- a/src/vnsw/agent/oper/vm.cc +++ b/src/vnsw/agent/oper/vm.cc @@ -18,7 +18,8 @@ using namespace autogen; VmTable *VmTable::vm_table_; -VmEntry::VmEntry(const uuid &id) : uuid_(id), name_("") { +VmEntry::VmEntry(const uuid &id) : uuid_(id), name_(""), + drop_new_flows_(false) { flow_count_ = 0; linklocal_flow_count_ = 0; } @@ -56,6 +57,7 @@ bool VmEntry::DBEntrySandesh(Sandesh *sresp, std::string &name) const { data.set_uuid(str_uuid); std::vector &list = const_cast&>(resp->get_vm_list()); + data.set_drop_new_flows(drop_new_flows_); list.push_back(data); return true; } @@ -63,6 +65,49 @@ bool VmEntry::DBEntrySandesh(Sandesh *sresp, std::string &name) const { return false; } +void VmEntry::update_flow_count(int val) const { + VmTable *vm_table = static_cast(get_table()); + int max_flows = vm_table->agent()->max_vm_flows(); + int tmp = flow_count_.fetch_and_add(val); + + if (max_flows == 0) { + // max_flows are not configured, + // disable drop new flows and return + SetInterfacesDropNewFlows(false); + return; + } + + if (val < 0) { + assert(tmp >= val); + if ((tmp + val) < ((max_flows * kDropNewFlowsRecoveryThreshold)/100)) { + SetInterfacesDropNewFlows(false); + } + } else { + if ((tmp + val) >= max_flows) { + SetInterfacesDropNewFlows(true); + } + } +} + +void VmEntry::SetInterfacesDropNewFlows(bool drop_new_flows) const { + if (drop_new_flows_ == drop_new_flows) { + return; + } + drop_new_flows_ = drop_new_flows; + VmTable *vm_table = static_cast(get_table()); + DBRequest req; + req.oper = DBRequest::DB_ENTRY_ADD_CHANGE; + tbb::mutex::scoped_lock lock(back_ref_mutex_); + std::set::const_iterator it = back_ref_set_.begin(); + for (; it != back_ref_set_.end(); it++) { + VmInterface *vm_intf = static_cast((*it).first); + req.key.reset(new VmInterfaceKey(AgentKey::RESYNC, + vm_intf->GetUuid(), "")); + req.data.reset(new VmInterfaceNewFlowDropData(drop_new_flows)); + vm_table->agent()->interface_table()->Enqueue(&req); + } +} + void VmEntry::SendObjectLog(AgentLogEvent::type event) const { VmObjectLogInfo info; string str; diff --git a/src/vnsw/agent/oper/vm.h b/src/vnsw/agent/oper/vm.h index 568d43e3fda..6ad404bbce3 100644 --- a/src/vnsw/agent/oper/vm.h +++ b/src/vnsw/agent/oper/vm.h @@ -33,6 +33,12 @@ struct VmData : public AgentOperDBData { class VmEntry : AgentRefCount, public AgentOperDBEntry { public: static const int kVectorIncreaseSize = 16; + + // kDropNewFlowsRecoveryThreshold is set to 90% of the Max flows for a + // VM this value represents that recovery from the drop new flows will + // happen only when total number of flow fall below this value + static const int kDropNewFlowsRecoveryThreshold = 90; + VmEntry(const uuid &id); virtual ~VmEntry(); @@ -53,11 +59,7 @@ class VmEntry : AgentRefCount, public AgentOperDBEntry { bool DBEntrySandesh(Sandesh *sresp, std::string &name) const; uint32_t flow_count() const { return flow_count_; } - void update_flow_count(int val) const { - int tmp = flow_count_.fetch_and_add(val); - if (val < 0) - assert(tmp >= val); - } + void update_flow_count(int val) const; uint32_t linklocal_flow_count() const { return linklocal_flow_count_; } void update_linklocal_flow_count(int val) const { @@ -65,12 +67,17 @@ class VmEntry : AgentRefCount, public AgentOperDBEntry { if (val < 0) assert(tmp >= val); } + + bool drop_new_flows() const { return drop_new_flows_; } + private: + void SetInterfacesDropNewFlows(bool drop_new_flows) const; friend class VmTable; uuid uuid_; std::string name_; mutable tbb::atomic flow_count_; mutable tbb::atomic linklocal_flow_count_; + mutable bool drop_new_flows_; DISALLOW_COPY_AND_ASSIGN(VmEntry); }; diff --git a/src/vnsw/agent/oper/vm_interface.cc b/src/vnsw/agent/oper/vm_interface.cc index 951f6cd880b..48c286ceedd 100644 --- a/src/vnsw/agent/oper/vm_interface.cc +++ b/src/vnsw/agent/oper/vm_interface.cc @@ -55,12 +55,12 @@ using namespace boost::uuids; using namespace autogen; VmInterface::VmInterface(const boost::uuids::uuid &uuid) : - Interface(Interface::VM_INTERFACE, uuid, "", NULL), vm_(NULL), + Interface(Interface::VM_INTERFACE, uuid, "", NULL), vm_(NULL, this), vn_(NULL), primary_ip_addr_(0), mdata_ip_(NULL), subnet_bcast_addr_(0), primary_ip6_addr_(), vm_mac_(MacAddress::kZeroMac), policy_enabled_(false), mirror_entry_(NULL), mirror_direction_(MIRROR_RX_TX), cfg_name_(""), - fabric_port_(true), need_linklocal_ip_(false), dhcp_enable_(true), - do_dhcp_relay_(false), vm_name_(), + fabric_port_(true), need_linklocal_ip_(false), drop_new_flows_(false), + dhcp_enable_(true), do_dhcp_relay_(false), vm_name_(), vm_project_uuid_(nil_uuid()), vxlan_id_(0), bridging_(false), layer3_forwarding_(true), flood_unknown_unicast_(false), mac_set_(false), ecmp_(false), ecmp6_(false), disable_policy_(false), @@ -94,12 +94,12 @@ VmInterface::VmInterface(const boost::uuids::uuid &uuid, uint16_t tx_vlan_id, uint16_t rx_vlan_id, Interface *parent, const Ip6Address &a6, DeviceType device_type, VmiType vmi_type) : - Interface(Interface::VM_INTERFACE, uuid, name, NULL), vm_(NULL), + Interface(Interface::VM_INTERFACE, uuid, name, NULL), vm_(NULL, this), vn_(NULL), primary_ip_addr_(addr), mdata_ip_(NULL), subnet_bcast_addr_(0), primary_ip6_addr_(a6), vm_mac_(mac), policy_enabled_(false), mirror_entry_(NULL), mirror_direction_(MIRROR_RX_TX), cfg_name_(""), - fabric_port_(true), need_linklocal_ip_(false), dhcp_enable_(true), - do_dhcp_relay_(false), vm_name_(vm_name), + fabric_port_(true), need_linklocal_ip_(false), drop_new_flows_(false), + dhcp_enable_(true), do_dhcp_relay_(false), vm_name_(vm_name), vm_project_uuid_(vm_project_uuid), vxlan_id_(0), bridging_(false), layer3_forwarding_(true), flood_unknown_unicast_(false), mac_set_(false), @@ -1907,6 +1907,13 @@ bool VmInterface::CopyConfig(const InterfaceTable *table, ret = true; } + bool drop_new_flows = + (vm_.get() != NULL) ? vm_->drop_new_flows() : false; + if (drop_new_flows_ != drop_new_flows) { + drop_new_flows_ = drop_new_flows; + ret = true; + } + VrfEntry *vrf = table->FindVrfRef(data->vrf_name_); if (vrf_.get() != vrf) { vrf_ = vrf; @@ -2477,6 +2484,25 @@ bool VmInterfaceHealthCheckData::OnResync(const InterfaceTable *table, return vmi->UpdateIsHealthCheckActive(); } +VmInterfaceNewFlowDropData::VmInterfaceNewFlowDropData(bool drop_new_flows) : + VmInterfaceData(NULL, NULL, DROP_NEW_FLOWS, Interface::TRANSPORT_INVALID), + drop_new_flows_(drop_new_flows) { +} + +VmInterfaceNewFlowDropData::~VmInterfaceNewFlowDropData() { +} + +bool VmInterfaceNewFlowDropData::OnResync(const InterfaceTable *table, + VmInterface *vmi, + bool *force_update) const { + if (vmi->drop_new_flows_ != drop_new_flows_) { + vmi->drop_new_flows_ = drop_new_flows_; + return true; + } + + return false; +} + ///////////////////////////////////////////////////////////////////////////// // VM Port Entry utility routines ///////////////////////////////////////////////////////////////////////////// diff --git a/src/vnsw/agent/oper/vm_interface.h b/src/vnsw/agent/oper/vm_interface.h index d150305518d..4fbd43e5001 100644 --- a/src/vnsw/agent/oper/vm_interface.h +++ b/src/vnsw/agent/oper/vm_interface.h @@ -504,6 +504,7 @@ class VmInterface : public Interface { const MacAddress &vm_mac() const { return vm_mac_; } bool fabric_port() const { return fabric_port_; } bool need_linklocal_ip() const { return need_linklocal_ip_; } + bool drop_new_flows() const { return drop_new_flows_; } bool dhcp_enable_config() const { return dhcp_enable_; } void set_dhcp_enable_config(bool dhcp_enable) { dhcp_enable_= dhcp_enable; @@ -699,6 +700,7 @@ class VmInterface : public Interface { friend struct VmInterfaceMirrorData; friend struct VmInterfaceGlobalVrouterData; friend struct VmInterfaceHealthCheckData; + friend struct VmInterfaceNewFlowDropData; bool IsMetaDataIPActive() const; bool IsIpv4Active() const; @@ -855,7 +857,7 @@ class VmInterface : public Interface { bool UpdateIsHealthCheckActive(); void CopyEcmpLoadBalance(EcmpLoadBalance &ecmp_load_balance); - VmEntryRef vm_; + VmEntryBackRef vm_; VnEntryRef vn_; Ip4Address primary_ip_addr_; std::auto_ptr mdata_ip_; @@ -868,6 +870,7 @@ class VmInterface : public Interface { std::string cfg_name_; bool fabric_port_; bool need_linklocal_ip_; + bool drop_new_flows_; // DHCP flag - set according to the dhcp option in the ifmap subnet object. // It controls whether the vrouter sends the DHCP requests from VM interface // to agent or if it would flood the request in the VN. @@ -978,7 +981,8 @@ struct VmInterfaceData : public InterfaceData { IP_ADDR, OS_OPER_STATE, GLOBAL_VROUTER, - HEALTH_CHECK + HEALTH_CHECK, + DROP_NEW_FLOWS }; VmInterfaceData(Agent *agent, IFMapNode *node, Type type, @@ -1166,4 +1170,13 @@ struct VmInterfaceHealthCheckData : public VmInterfaceData { bool *force_update) const; }; +struct VmInterfaceNewFlowDropData : public VmInterfaceData { + VmInterfaceNewFlowDropData(bool drop_new_flows); + virtual ~VmInterfaceNewFlowDropData(); + virtual bool OnResync(const InterfaceTable *table, VmInterface *vmi, + bool *force_update) const; + + bool drop_new_flows_; +}; + #endif // vnsw_agent_vm_interface_hpp diff --git a/src/vnsw/agent/pkt/test/test_pkt_flow_limits.cc b/src/vnsw/agent/pkt/test/test_pkt_flow_limits.cc index 7e63cc1ab51..1c4866197f0 100644 --- a/src/vnsw/agent/pkt/test/test_pkt_flow_limits.cc +++ b/src/vnsw/agent/pkt/test/test_pkt_flow_limits.cc @@ -604,9 +604,12 @@ TEST_F(FlowTest, FlowLimit_1) { FlowStatsTimerStartStop(agent_, true); CreateFlow(flow, 4); client->WaitForIdle(); - int nh_id = agent_->interface_table()->FindInterface - (vmi_3->id())->flow_key_nh()->id(); + VmInterface *intf = static_cast(agent_->interface_table()->\ + FindInterface(vmi_3->id())); + int nh_id = intf->flow_key_nh()->id(); EXPECT_EQ(4U, get_flow_proto()->FlowCount()); + // Validate interface is set to drop new flows after flow limit is reached + EXPECT_TRUE(intf->drop_new_flows()); FlowEntry *fe = FlowGet(VrfGet("vrf3")->vrf_id(), vm4_ip, vm1_ip, IPPROTO_TCP, 300, 200, nh_id); EXPECT_TRUE(fe != NULL && fe->is_flags_set(FlowEntry::ShortFlow) == true && @@ -618,6 +621,10 @@ TEST_F(FlowTest, FlowLimit_1) { DeleteRoute("vrf5", vm4_ip); DeleteRoute("vrf3", vm1_ip); client->WaitForIdle(); + FlushFlowTable(); + client->WaitForIdle(); + // Validate interface is not dropping new flows after flows age-out + EXPECT_FALSE(intf->drop_new_flows()); client->WaitForIdle(); agent_->set_max_vm_flows(vm_flows); } diff --git a/src/vnsw/agent/vrouter/ksync/interface_ksync.cc b/src/vnsw/agent/vrouter/ksync/interface_ksync.cc index bc26ef01538..dd66af3dfe0 100644 --- a/src/vnsw/agent/vrouter/ksync/interface_ksync.cc +++ b/src/vnsw/agent/vrouter/ksync/interface_ksync.cc @@ -41,6 +41,7 @@ InterfaceKSyncEntry::InterfaceKSyncEntry(InterfaceKSyncObject *obj, const InterfaceKSyncEntry *entry, uint32_t index) : KSyncNetlinkDBEntry(index), analyzer_name_(entry->analyzer_name_), + drop_new_flows_(entry->drop_new_flows_), dhcp_enable_(entry->dhcp_enable_), fd_(kInvalidIndex), flow_key_nh_id_(entry->flow_key_nh_id_), @@ -80,6 +81,7 @@ InterfaceKSyncEntry::InterfaceKSyncEntry(InterfaceKSyncObject *obj, const Interface *intf) : KSyncNetlinkDBEntry(kInvalidIndex), analyzer_name_(), + drop_new_flows_(false), dhcp_enable_(true), fd_(-1), flow_key_nh_id_(0), @@ -210,6 +212,11 @@ bool InterfaceKSyncEntry::Sync(DBEntry *e) { ret = true; } + if (drop_new_flows_ != vm_port->drop_new_flows()) { + drop_new_flows_ = vm_port->drop_new_flows(); + ret = true; + } + if (dhcp_enable_ != vm_port->dhcp_enabled()) { dhcp_enable_ = vm_port->dhcp_enabled(); ret = true; @@ -540,6 +547,9 @@ int InterfaceKSyncEntry::Encode(sandesh_op::type op, char *buf, int buf_len) { case Interface::VM_INTERFACE: { if (vmi_device_type_ == VmInterface::TOR) return 0; + if (drop_new_flows_) { + flags |= VIF_FLAG_DROP_NEW_FLOWS; + } if (dhcp_enable_) { flags |= VIF_FLAG_DHCP_ENABLED; } diff --git a/src/vnsw/agent/vrouter/ksync/interface_ksync.h b/src/vnsw/agent/vrouter/ksync/interface_ksync.h index 4c6167d90dc..bc2b0a54461 100644 --- a/src/vnsw/agent/vrouter/ksync/interface_ksync.h +++ b/src/vnsw/agent/vrouter/ksync/interface_ksync.h @@ -67,6 +67,7 @@ class InterfaceKSyncEntry : public KSyncNetlinkDBEntry { virtual int DeleteMsg(char *buf, int buf_len); virtual KSyncEntry *UnresolvedReference(); void FillObjectLog(sandesh_op::type op, KSyncIntfInfo &info) const; + bool drop_new_flows() const {return drop_new_flows_;} bool dhcp_enable() const {return dhcp_enable_;} bool layer3_forwarding() const {return layer3_forwarding_;} bool bridging() const {return bridging_;} @@ -78,6 +79,7 @@ class InterfaceKSyncEntry : public KSyncNetlinkDBEntry { int Encode(sandesh_op::type op, char *buf, int buf_len); string analyzer_name_; + bool drop_new_flows_; bool dhcp_enable_; uint32_t fd_; // FD opened for this uint32_t flow_key_nh_id_;