Implements change list to optimize config processing

In scaled environments there can be large number of invocations to
IFNodeToReq and IFLinkToReq APIs. The config processing routines are
written to act on the latest version of configuration. So, an invocation
of config routine will override all previous invocations. Ideally, we wnat
to invoke the config callbacks only once after all configuraiton is
received. However, we dont have any marker to identify end of configuration.

The next best design we follow is to add the objects changed into a
change list. The change-list is run from a task-trigger. The list can
potentially compress multiple changes to a node

The changelist should also take care of dependency between objects. For
example, VMInterface has reference to VirtualNetwork. So, the change list
for virtual-network should be invoked before VMInterface. The changelist
should take of all dependencies.

To simplify current design, the changelist is implemented only to objects
virtual-machine-interface, logical-interfaces and physical-device-vn

Change-Id: I86585c136f566f5bb1a3d1503c39a1b21e096764

src/vnsw/agent/cmn/agent.cc

@@ -22,6 +22,7 @@
 #include <cmn/agent.h>
 
 #include <oper/operdb_init.h>
+#include <oper/config_manager.h>
 #include <oper/interface_common.h>
 #include <oper/multicast.h>
 #include <oper/nexthop.h>
@@ -397,6 +398,8 @@ Agent::Agent() :
 
     agent_signal_.reset(
         AgentObjectFactory::Create<AgentSignal>(event_mgr_));
+
+    config_manager_.reset(new ConfigManager(this));
 }
 
 Agent::~Agent() {
@@ -439,6 +442,10 @@ void Agent::set_stats(AgentStats *stats) {
     stats_ = stats;
 }
 
+ConfigManager *Agent::config_manager() const {
+    return config_manager_.get();
+}
+
 KSync *Agent::ksync() const {
     return ksync_;
 }

src/vnsw/agent/cmn/agent.h

@@ -32,6 +32,7 @@ class TaskScheduler;
 class AgentInit;
 class AgentStatsCollector;
 class FlowStatsCollector;
+class ConfigManager;
 
 class Interface;
 typedef boost::intrusive_ptr<Interface> InterfaceRef;
@@ -760,6 +761,8 @@ class Agent {
     bool init_done() const { return init_done_; }
     void set_init_done(bool done) { init_done_ = done; }
 
+    ConfigManager *config_manager() const;
+
     AgentParam *params() const { return params_; }
 
     bool isXenMode();
@@ -841,6 +844,7 @@ class Agent {
     LoadbalancerTable *loadbalancer_table_;
     PhysicalDeviceTable *physical_device_table_;
     PhysicalDeviceVnTable *physical_device_vn_table_;
+    std::auto_ptr<ConfigManager> config_manager_;
 
     // Mirror config table
     MirrorCfgTable *mirror_cfg_table_;

src/vnsw/agent/oper/SConscript

@@ -26,6 +26,7 @@ vnswoperdb = env.Library('vnswoperdb',
                           'agent_route_encap.cc',
                           'agent_route_walker.cc',
                           'bridge_route.cc',
+                          'config_manager.cc',
                           'evpn_route.cc',
                           'global_vrouter.cc',
                           'ifmap_dependency_manager.cc',

src/vnsw/agent/oper/config_manager.cc

@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
+ */
+
+#include <boost/uuid/uuid_io.hpp>
+#include <vnc_cfg_types.h>
+#include <base/util.h>
+
+#include <ifmap/ifmap_node.h>
+#include <cmn/agent_cmn.h>
+#include <oper/operdb_init.h>
+#include <oper/ifmap_dependency_manager.h>
+#include <oper/config_manager.h>
+
+#include <oper/interface_common.h>
+#include <oper/physical_device.h>
+#include <oper/physical_device_vn.h>
+
+#include <vector>
+#include <string>
+
+using std::string;
+
+ConfigManager::ConfigManager(Agent *agent) : agent_(agent), trigger_(NULL) {
+    int task_id = TaskScheduler::GetInstance()->GetTaskId("db::DBTable");
+    trigger_.reset
+        (new TaskTrigger(boost::bind(&ConfigManager::Run, this), task_id, 0));
+}
+
+ConfigManager::~ConfigManager() {
+}
+
+ConfigManager::Node::Node(IFMapDependencyManager::IFMapNodePtr state) :
+    state_(state) {
+}
+
+ConfigManager::Node::~Node() {
+}
+
+// Run the change-list
+bool ConfigManager::Run() {
+    uint32_t count = 0;
+    NodeListIterator it;
+
+    // Logical-interfaces can potentially trigger changes to VMInterfaces
+    // So, run thru the logical-interfaces first
+    it = logical_interface_list_.begin();
+    while ((count < kIterationCount) && (it != logical_interface_list_.end())) {
+        NodeListIterator prev = it++;
+        IFMapNodeState *state = prev->state_.get();
+        IFMapNode *node = state->node();
+        DBRequest req;
+        if (agent_->interface_table()->LogicalInterfaceProcessConfig(node, req)) {
+            agent_->interface_table()->Enqueue(&req);
+        }
+        logical_interface_list_.erase(prev);
+        count++;
+    }
+
+    // Run thru changelist for VMI
+    it = vmi_list_.begin();
+    while ((count < kIterationCount) && (it != vmi_list_.end())) {
+        NodeListIterator prev = it++;
+        IFMapNodeState *state = prev->state_.get();
+        IFMapNode *node = state->node();
+        DBRequest req;
+        if (agent_->interface_table()->VmiProcessConfig(node, req)) {
+            agent_->interface_table()->Enqueue(&req);
+        }
+        vmi_list_.erase(prev);
+        count++;
+    }
+
+    // changelist for physical-device-vn entries
+    PhysicalDeviceVnIterator it_dev_vn = physical_device_vn_list_.begin();
+    while ((count < kIterationCount) &&
+           (it_dev_vn != physical_device_vn_list_.end())) {
+        PhysicalDeviceVnIterator prev = it_dev_vn++;
+        PhysicalDeviceVnTable *table = agent_->physical_device_vn_table();
+        table->PhysicalDeviceVnAdd(prev->dev_, prev->vn_);
+        physical_device_vn_list_.erase(prev);
+        count++;
+    }
+
+    trigger_->Reset();
+    if (vmi_list_.size() != 0 || logical_interface_list_.size() != 0 ||
+        physical_device_vn_list_.size() != 0) {
+        trigger_->Set();
+        return false;
+    }
+
+    return true;
+}
+
+void ConfigManager::AddVmiNode(IFMapNode *node) {
+    IFMapDependencyManager *dep = agent_->oper_db()->dependency_manager();
+    Node n(dep->SetState(node));
+    vmi_list_.insert(n);
+    trigger_->Set();
+}
+
+void ConfigManager::DelVmiNode(IFMapNode *node) {
+    IFMapDependencyManager *dep = agent_->oper_db()->dependency_manager();
+    IFMapNodeState *state = dep->IFMapNodeGet(node);
+    if (state == NULL)
+        return;
+    Node n(state);
+    vmi_list_.erase(n);
+}
+
+uint32_t ConfigManager::VmiNodeCount() {
+    return vmi_list_.size();
+}
+
+void ConfigManager::AddLogicalInterfaceNode(IFMapNode *node) {
+    IFMapDependencyManager *dep = agent_->oper_db()->dependency_manager();
+    Node n(dep->SetState(node));
+    logical_interface_list_.insert(n);
+    trigger_->Set();
+}
+
+void ConfigManager::DelLogicalInterfaceNode(IFMapNode *node) {
+    IFMapDependencyManager *dep = agent_->oper_db()->dependency_manager();
+    IFMapNodeState *state = dep->IFMapNodeGet(node);
+    if (state == NULL)
+        return;
+    Node n(state);
+    logical_interface_list_.erase(n);
+}
+
+uint32_t ConfigManager::LogicalInterfaceNodeCount() {
+    return logical_interface_list_.size();
+}
+
+ConfigManager::PhysicalDeviceVnEntry::PhysicalDeviceVnEntry
+    (const boost::uuids::uuid &dev, const boost::uuids::uuid &vn) :
+        dev_(dev), vn_(vn) {
+}
+
+void ConfigManager::AddPhysicalDeviceVn(const boost::uuids::uuid &dev,
+                                        const boost::uuids::uuid &vn) {
+    physical_device_vn_list_.insert(PhysicalDeviceVnEntry(dev, vn));
+    trigger_->Set();
+}
+
+void ConfigManager::DelPhysicalDeviceVn(const boost::uuids::uuid &dev,
+                                        const boost::uuids::uuid &vn) {
+    physical_device_vn_list_.erase(PhysicalDeviceVnEntry(dev, vn));
+}
+
+uint32_t ConfigManager::PhysicalDeviceVnCount() {
+    return physical_device_vn_list_.size();
+}

src/vnsw/agent/oper/config_manager.h

@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
+ */
+#ifndef SRC_VNSW_AGENT_OPER_CONFIG_MANAGER_H_
+#define SRC_VNSW_AGENT_OPER_CONFIG_MANAGER_H_
+
+/*****************************************************************************
+ * Implements change list for the oper table. Normally, the oper objects
+ * process configuration thru IFNodeToReq or IFLinkToReq callback methods.
+ *
+ * In scaled environments there can be large number of invocations to
+ * IFNodeToReq and IFLinkToReq APIs. The config processing routines are 
+ * written to act on the latest version of configuration. So, an invocation
+ * of config routine will override all previous invocations. Ideally, we wnat
+ * to invoke the config callbacks only once after all configuraiton is
+ * received. However, we dont have any marker to identify end of configuration.
+ * 
+ * The next best design we follow is to add the objects changed into a 
+ * change list. The change-list is run from a task-trigger. The list can
+ * potentially compress multiple changes to a node
+ *
+ * The changelist should also take care of dependency between objects. For 
+ * example, VMInterface has reference to VirtualNetwork. So, the change list 
+ * for virtual-network should be invoked before VMInterface. The changelist
+ * should take of all dependencies.
+ *
+ * To simplify current design, the changelist is implemented only to objects
+ * virtual-machine-interface, logical-interfaces and physical-device-vn
+ *****************************************************************************/
+
+#include <cmn/agent_cmn.h>
+#include <cmn/agent_db.h>
+#include <operdb_init.h>
+#include <ifmap_dependency_manager.h>
+
+class ConfigManager {
+public:
+    // Number of changelist entries to pick in one run
+    const static uint32_t kIterationCount = 32;
+    // Set of changed IFMapNodes
+    struct Node {
+        Node(IFMapDependencyManager::IFMapNodePtr state);
+        ~Node();
+        IFMapDependencyManager::IFMapNodePtr state_;
+    };
+
+    struct NodeCmp {
+        bool operator() (const Node &lhs, const Node &rhs) {
+            return lhs.state_.get() < rhs.state_.get();
+        }
+    };
+    typedef std::set<Node, NodeCmp> NodeList;
+    typedef NodeList::iterator NodeListIterator;
+
+    // Set of changed PhysicalDeviceVn entries
+    struct PhysicalDeviceVnEntry {
+        PhysicalDeviceVnEntry(const boost::uuids::uuid &dev,
+                              const boost::uuids::uuid &vn);
+        boost::uuids::uuid dev_;
+        boost::uuids::uuid vn_;
+    };
+
+    struct PhysicalDeviceVnEntryCmp {
+        bool operator() (const PhysicalDeviceVnEntry &lhs,
+                         const PhysicalDeviceVnEntry &rhs) {
+            if (lhs.dev_ != rhs.dev_)
+                return lhs.dev_ < rhs.dev_;
+
+            return lhs.vn_ < rhs.vn_;
+        }
+    };
+
+    typedef std::set<PhysicalDeviceVnEntry, PhysicalDeviceVnEntryCmp>
+        PhysicalDeviceVnList;
+    typedef PhysicalDeviceVnList::iterator PhysicalDeviceVnIterator;
+
+    ConfigManager(Agent *agent);
+    virtual ~ConfigManager();
+
+    bool Run();
+
+    void AddVmiNode(IFMapNode *node);
+    void DelVmiNode(IFMapNode *node);
+    uint32_t VmiNodeCount();
+
+    void AddLogicalInterfaceNode(IFMapNode *node);
+    void DelLogicalInterfaceNode(IFMapNode *node);
+    uint32_t LogicalInterfaceNodeCount();
+
+    void AddPhysicalDeviceVn(const boost::uuids::uuid &dev,
+                             const boost::uuids::uuid &vn);
+    void DelPhysicalDeviceVn(const boost::uuids::uuid &dev,
+                             const boost::uuids::uuid &vn);
+    uint32_t PhysicalDeviceVnCount();
+
+private:
+    Agent *agent_;
+    std::auto_ptr<TaskTrigger> trigger_;
+    NodeList vmi_list_;
+    NodeList logical_interface_list_;
+    PhysicalDeviceVnList physical_device_vn_list_;
+
+    DISALLOW_COPY_AND_ASSIGN(ConfigManager);
+};
+
+#endif  // SRC_VNSW_AGENT_OPER_CONFIG_MANAGER_H_

src/vnsw/agent/oper/ifmap_dependency_manager.h

@@ -89,6 +89,7 @@ class IFMapDependencyManager {
      * Add DBState to an IFMapNode
      */
     IFMapNodePtr SetState(IFMapNode *node);
+    IFMapNodeState *IFMapNodeGet(IFMapNode *node);
     /*
      * Register a notification callback.
      */
@@ -120,7 +121,6 @@ class IFMapDependencyManager {
     void ChangeListAdd(IFMapNode *node);
 
     void IFMapNodeReset(IFMapNode *node);
-    IFMapNodeState *IFMapNodeGet(IFMapNode *node);
 
     DB *database_;
     DBGraph *graph_;

src/vnsw/agent/oper/interface.h

@@ -273,6 +273,8 @@ class InterfaceTable : public AgentOperDBTable {
     bool OperDBResync(DBEntry *entry, const DBRequest *req);
 
     // Config handlers
+    bool LogicalInterfaceProcessConfig(IFMapNode *node, DBRequest &req);
+    bool VmiProcessConfig(IFMapNode *node, DBRequest &req);
     bool VmiIFNodeToReq(IFMapNode *node, DBRequest &req);
     bool VmiIFNodeToUuid(IFMapNode *node, boost::uuids::uuid &u);
     bool LogicalInterfaceIFNodeToUuid(IFMapNode *node, boost::uuids::uuid &u);

src/vnsw/agent/oper/logical_interface.cc

@@ -15,6 +15,7 @@
 #include <oper/physical_device.h>
 #include <oper/logical_interface.h>
 #include <oper/vm_interface.h>
+#include <oper/config_manager.h>
 
 #include <vector>
 #include <string>
@@ -292,8 +293,8 @@ bool InterfaceTable::LogicalInterfaceIFNodeToUuid(IFMapNode *node,
     return true;
 }
 
-bool InterfaceTable::LogicalInterfaceIFNodeToReq(IFMapNode *node,
-                                                 DBRequest &req) {
+bool InterfaceTable::LogicalInterfaceProcessConfig(IFMapNode *node,
+                                                   DBRequest &req) {
     autogen::LogicalInterface *port =
         static_cast <autogen::LogicalInterface *>(node->GetObject());
     assert(port);
@@ -325,6 +326,27 @@ bool InterfaceTable::LogicalInterfaceIFNodeToReq(IFMapNode *node,
     return false;
 }
 
+bool InterfaceTable::LogicalInterfaceIFNodeToReq(IFMapNode *node,
+                                                 DBRequest &req) {
+    autogen::LogicalInterface *port =
+        static_cast <autogen::LogicalInterface *>(node->GetObject());
+    assert(port);
+
+    boost::uuids::uuid u;
+    if (agent()->cfg_listener()->GetCfgDBStateUuid(node, u) == false)
+        return false;
+
+    req.key.reset(BuildKey(node, u));
+    if (node->IsDeleted()) {
+        agent()->config_manager()->DelLogicalInterfaceNode(node);
+        req.oper = DBRequest::DB_ENTRY_DELETE;
+        return true;
+    }
+
+    agent()->config_manager()->AddLogicalInterfaceNode(node);
+    return false;
+}
+
 #if 0
 void LogicalInterface::SetSandeshData(SandeshLogicalInterface *data) const {
 }