Dummy task to awake scheduler

Change-Id: If15f50de49b3da71c9ee01ccfd88c1be2ddc48e2
Partial-Bug: #1538920

src/vnsw/agent/cmn/agent.cc

@@ -284,6 +284,7 @@ void Agent::CopyConfig(AgentParam *params) {
     test_mode_ = params_->test_mode();
     tsn_enabled_ = params_->isTsnAgent();
     tor_agent_enabled_ = params_->isTorAgent();
+    tbb_keepawake_timeout_ = params_->tbb_keepawake_timeout();
 }
 
 DiscoveryAgentClient *Agent::discovery_client() const {
@@ -338,6 +339,30 @@ void Agent::InitCollector() {
 
 }
 
+bool Agent::TbbKeepAwake() {
+    tbb_awake_count_++;
+    return true;
+}
+
+void Agent::InitDone() {
+    // Its observed that sometimes TBB doesnt scheduler misses spawn events
+    // and doesnt schedule a task till its triggered again. As a work around
+    // start a dummy timer that fires and awake TBB periodically
+    if (tbb_keepawake_timeout_) {
+        uint32_t task_id = task_scheduler_->GetTaskId("Agent::TbbKeepAwake");
+        tbb_awake_timer_ = TimerManager::CreateTimer(*event_mgr_->io_service(),
+                                                     "TBB Keep Awake",
+                                                     task_id, 0);
+        tbb_awake_timer_->Start(tbb_keepawake_timeout_,
+                                boost::bind(&Agent::TbbKeepAwake, this));
+    }
+}
+
+void Agent::Shutdown() {
+    if (tbb_awake_timer_)
+        TimerManager::DeleteTimer(tbb_awake_timer_);
+}
+
 static bool interface_exist(string &name) {
 	struct if_nameindex *ifs = NULL;
 	struct if_nameindex *head = NULL;
@@ -446,7 +471,9 @@ Agent::Agent() :
     vrouter_server_port_(0), vrouter_max_labels_(0), vrouter_max_nexthops_(0),
     vrouter_max_interfaces_(0), vrouter_max_vrfs_(0),
     vrouter_max_mirror_entries_(0), vrouter_max_bridge_entries_(0),
-    vrouter_max_oflow_bridge_entries_(0), flow_stats_req_handler_(NULL) {
+    vrouter_max_oflow_bridge_entries_(0), flow_stats_req_handler_(NULL),
+    tbb_keepawake_timeout_(kDefaultTbbKeepawakeTimeout), tbb_awake_timer_(NULL),
+    tbb_awake_count_(0) {
 
     assert(singleton_ == NULL);
     singleton_ = this;

src/vnsw/agent/cmn/agent.h

@@ -12,6 +12,7 @@
 #include <boost/intrusive_ptr.hpp>
 #include <cmn/agent_cmn.h>
 #include <base/connection_info.h>
+#include <base/timer.h>
 #include "net/mac_address.h"
 
 class Agent;
@@ -209,6 +210,7 @@ class Agent {
     static const uint32_t kDefaultFlowCacheTimeout = 0;
     // Max number of threads
     static const uint32_t kMaxTbbThreads = 8;
+    static const uint32_t kDefaultTbbKeepawakeTimeout = (20); //time-millisecs
 
     enum ForwardingMode {
         NONE,
@@ -238,7 +240,7 @@ class Agent {
 
     Agent();
     virtual ~Agent();
-    void Shutdown() { }
+    void Shutdown();
 
     static Agent *GetInstance() {return singleton_;}
     static const std::string &NullString() {return null_string_;};
@@ -912,6 +914,8 @@ class Agent {
 
     void TaskTrace(const char *file_name, uint32_t line_no, const Task *task,
                    const char *description, uint32_t delay);
+    bool TbbKeepAwake();
+
 private:
 
     AgentParam *params_;
@@ -1081,6 +1085,9 @@ class Agent {
     std::string vrouter_build_info_;
     FlowStatsReqHandler flow_stats_req_handler_;
 
+    uint32_t tbb_keepawake_timeout_;
+    Timer *tbb_awake_timer_;
+    uint64_t tbb_awake_count_;
     // Constants
     static const std::string config_file_;
     static const std::string log_file_;

src/vnsw/agent/contrail-vrouter-agent.conf

@@ -196,3 +196,6 @@ docker_command=/usr/bin/opencontrail-vrouter-docker
 #
 # Log message if time taken to schedule task exceeds a threshold (in msec)
 # log_schedule_threshold = 25
+#
+# TBB Keepawake timer interval
+# tbb_keepawake_timeout = 20

src/vnsw/agent/init/agent_init.cc

@@ -286,7 +286,9 @@ void AgentInit::InitDoneBase() {
     if (cfg_.get()) {
         cfg_->InitDone();
     }
+    agent_->InitDone();
     InitDone();
+
     // Enable task latency measurements once init is done
     agent_->task_scheduler()->EnableLatencyThresholds
         (agent_param_->tbb_exec_delay() * 1000,
@@ -438,6 +440,8 @@ void AgentInit::ModulesShutdownBase() {
     if (agent_->cfg()) {
         agent_->cfg()->Shutdown();
     }
+
+    agent_->Shutdown();
     return;
 }
 

src/vnsw/agent/init/agent_param.cc

@@ -466,6 +466,11 @@ void AgentParam::ParseTaskSection() {
                                     "TASK.log_schedule_threshold")) {
         tbb_schedule_delay_ = 0;
     }
+
+    if (!GetValueFromTree<uint32_t>(tbb_keepawake_timeout_,
+                                    "TASK.tbb_keepawake_timeout")) {
+        tbb_keepawake_timeout_ = Agent::kDefaultTbbKeepawakeTimeout;
+    }
 }
 
 void AgentParam::ParseMetadataProxy() {
@@ -676,6 +681,8 @@ void AgentParam::ParseTaskSectionArguments
                           "TASK.log_exec_threshold");
     GetOptValue<uint32_t>(var_map, tbb_schedule_delay_,
                           "TASK.log_schedule_threshold");
+    GetOptValue<uint32_t>(var_map, tbb_keepawake_timeout_,
+                          "TASK.tbb_keepawake_timeout");
 }
 
 void AgentParam::ParseMetadataProxyArguments
@@ -1150,7 +1157,8 @@ AgentParam::AgentParam(Agent *agent, bool enable_flow_options,
         agent_base_dir_(),
         subnet_hosts_resolvable_(true),
         tbb_exec_delay_(0),
-        tbb_schedule_delay_(0) {
+        tbb_schedule_delay_(0),
+        tbb_keepawake_timeout_(Agent::kDefaultTbbKeepawakeTimeout) {
     vgw_config_table_ = std::auto_ptr<VirtualGatewayConfigTable>
         (new VirtualGatewayConfigTable(agent));
 
@@ -1309,6 +1317,8 @@ AgentParam::AgentParam(Agent *agent, bool enable_flow_options,
          "Log message if task takes more than threshold (msec) to execute")
         ("TASK.log_schedule_threshold", opt::value<uint32_t>(),
          "Log message if task takes more than threshold (msec) to schedule")
+        ("TASK.tbb_keepawake_timeout", opt::value<uint32_t>(),
+         "Timeout for the TBB keepawake timer")
         ;
     options_.add(tbb);
 }

src/vnsw/agent/init/agent_param.h

@@ -226,6 +226,7 @@ class AgentParam  {
 
     uint32_t tbb_exec_delay() const { return tbb_exec_delay_; }
     uint32_t tbb_schedule_delay() const { return tbb_schedule_delay_; }
+    uint32_t tbb_keepawake_timeout() const { return tbb_keepawake_timeout_; }
 protected:
     void set_hypervisor_mode(HypervisorMode m) { hypervisor_mode_ = m; }
     virtual void InitFromSystem();
@@ -421,6 +422,7 @@ class AgentParam  {
     // TBB related
     uint32_t tbb_exec_delay_;
     uint32_t tbb_schedule_delay_;
+    uint32_t tbb_keepawake_timeout_;
     DISALLOW_COPY_AND_ASSIGN(AgentParam);
 };
 

src/vnsw/agent/init/test/tbb.ini

@@ -0,0 +1,146 @@
+#
+# Vnswad configuration options
+#
+
+[CONTROL-NODE]
+# IP address to be used to connect to control-node. Maximum of 2 IP addresses
+# (separated by a space) can be provided. If no IP is configured then the
+# value provided by discovery service will be used.
+server=127.0.0.1
+
+[DEFAULT]
+# IP address and port to be used to connect to collector. If these are not
+# configured, value provided by discovery service will be used. Multiple
+# IP:port strings separated by space can be provided
+# collectors=127.0.0.1:8086
+
+# Enable/disable debug logging. Possible values are 0 (disable) and 1 (enable)
+debug=1
+
+# Aging time for flow-records in seconds
+flow_cache_timeout=30
+
+# Hostname of compute-node. If this is not configured value from `hostname`
+# will be taken
+# hostname=
+
+# Http server port for inspecting vnswad state (useful for debugging)
+# http_server_port=8085
+
+# Category for logging. Default value is '*'
+# log_category=
+
+# Local log file name
+# log_file=/var/log/contrail/vrouter.log
+
+# Log severity levels. Possible values are SYS_EMERG, SYS_ALERT, SYS_CRIT,
+# SYS_ERR, SYS_WARN, SYS_NOTICE, SYS_INFO and SYS_DEBUG. Default is SYS_DEBUG
+# log_level=SYS_DEBUG
+
+# Enable/Disable local file logging. Possible values are 0 (disable) and 1 (enable)
+# log_local=0
+
+# Enable/Disable logging of flow messages. Possible values are 0 (disable) and 1 (enable)
+# log_flow=0
+
+# Encapsulation type for tunnel. Possible values are MPLSoGRE, MPLSoUDP, VXLAN
+tunnel_type=MPLSoGRE
+
+# DHCP relay mode (true or false) to determine if a DHCP request in fabric
+# interface with an unconfigured IP should be relayed or not
+dhcp_relay_mode=true
+
+# Agent base directory
+agent_base_directory=/var/lib/contrail
+
+[DISCOVERY]
+# IP address and port of discovery server
+# port=5998
+server=10.3.1.1
+
+# Number of control-nodes info to be provided by Discovery service. Possible
+# values are 1 and 2
+max_control_nodes=2
+
+[DNS]
+# IP address and port to be used to connect to dns-node. Maximum of 2 IP
+# addresses (separated by a space) can be provided. If no IP is configured then
+# the value provided by discovery service will be used.
+server=127.0.0.1:53
+
+[HYPERVISOR]
+# Hypervisor type. Possible values are kvm, xen and vmware
+type=xen
+
+# Link-local IP address and prefix in ip/prefix_len format (for xen)
+xen_ll_ip=169.254.0.1/24
+
+# Link-local interface name when hypervisor type is Xen
+xen_ll_interface=xenapi
+
+# Physical interface name when hypervisor type is vmware
+# vmware_physical_interface=
+
+[FLOWS]
+# Maximum flows allowed per VM (given as % of maximum system flows)
+max_vm_flows=50
+# Maximum number of link-local flows allowed across all VMs
+max_system_linklocal_flows=1024
+# Maximum number of link-local flows allowed per VM
+max_vm_linklocal_flows=512
+
+[METADATA]
+# Shared secret for metadata proxy service
+metadata_proxy_secret=contrail
+
+[NETWORKS]
+# control-channel IP address used by WEB-UI to connect to vnswad to fetch
+# required information
+# control_network_ip=
+
+[VIRTUAL-HOST-INTERFACE]
+# name of virtual host interface
+name=vhost0
+
+# IP address and prefix in ip/prefix_len format
+ip=10.1.1.1/24
+
+# Gateway IP address for virtual host
+gateway=10.1.1.254
+
+# Physical interface name to which virtual host interface maps to
+physical_interface=vnet0
+
+[GATEWAY-0]
+routing_instance=default-domain:admin:public:public
+interface=vgw
+
+# Virtual network ip blocks for which gateway service is required. Each IP
+# block is represented as ip/prefix. Multiple IP blocks are represented by
+# separating each with a space
+ip_blocks=1.1.1.1/24
+
+[GATEWAY-1]
+# Name of the routing_instance for which the gateway is being configured
+# routing_instance=default-domain:admin:public:public1
+
+# Gateway interface name
+# interface=vgw1
+
+# Virtual network ip blocks for which gateway service is required. Each IP
+# block is represented as ip/prefix. Multiple IP blocks are represented by
+# separating each with a space
+# ip_blocks=2.2.1.0/24 2.2.2.0/24
+
+# Routes to be exported in routing_instance. Each route is represented as
+# ip/prefix. Multiple routes are represented by separating each with a space
+# routes= 10.10.10.1/24 11.11.11.1/24
+
+[TASK]
+# Log message if time taken to execute task exceeds a threshold (in msec)
+log_exec_threshold = 10
+#
+# Log message if time taken to schedule task exceeds a threshold (in msec)
+log_schedule_threshold = 25
+# TBB Keepawake timer interval
+tbb_keepawake_timeout = 50

src/vnsw/agent/init/test/test_agent_init.cc

@@ -84,6 +84,39 @@ TEST_F(FlowTest, Agent_Conf_file_2) {
     EXPECT_EQ(param.subnet_hosts_resolvable(), false);
 }
 
+TEST_F(FlowTest, Agent_Tbb_Option_1) {
+    int argc = 1;
+    char *argv[] = {
+        (char *) "",
+    };
+
+    AgentParam param(Agent::GetInstance());
+    param.ParseArguments(argc, argv);
+    param.Init("controller/src/vnsw/agent/init/test/tbb.ini", "test-param");
+
+    EXPECT_EQ(param.tbb_keepawake_timeout(), 50);
+    EXPECT_EQ(param.tbb_exec_delay(), 10);
+    EXPECT_EQ(param.tbb_schedule_delay(), 25);
+}
+
+TEST_F(FlowTest, Agent_Tbb_Option_Arguments) {
+    int argc = 7;
+    char *argv[] = {
+        (char *) "",
+        (char *) "--TASK.log_exec_threshold",           (char *)"100",
+        (char *) "--TASK.log_schedule_threshold",      (char *)"200",
+        (char *) "--TASK.tbb_keepawake_timeout",      (char *)"300",
+    };
+
+    AgentParam param(Agent::GetInstance());
+    param.ParseArguments(argc, argv);
+    param.Init("controller/src/vnsw/agent/init/test/tbb.ini", "test-param");
+
+    EXPECT_EQ(param.tbb_exec_delay(), 100);
+    EXPECT_EQ(param.tbb_schedule_delay(), 200);
+    EXPECT_EQ(param.tbb_keepawake_timeout(), 300);
+}
+
 // Check that linklocal flows are updated when the system limits are lowered
 TEST_F(FlowTest, Agent_Conf_file_3) {
     struct rlimit rl;