From bcc9bbc62b93381e5b69ccb04ec2ce43c15d9bc7 Mon Sep 17 00:00:00 2001 From: arvindvis Date: Thu, 14 Apr 2016 16:51:02 -0700 Subject: [PATCH] This fix introduces throtlling of sandesh messages. The daemons which use sandesh generator can specify the throttling value in msgs/second and specify it either as command line argument or in their respective config file sandesh_send_rate_limit Partial-Bug: 1469414 Change-Id: Idda249149f905a8a89e861e048e39b3f93f05ffd --- src/analytics/contrail-collector.conf | 4 ++++ .../contrail_snmp_collector/snmpcfg.py | 5 +++++ .../contrail_snmp_collector/snmpuve.py | 3 +++ .../contrail-topology/contrail_topology/config.py | 5 +++++ .../contrail_topology/topology_uve.py | 3 +++ src/analytics/main.cc | 1 + src/analytics/options.cc | 7 ++++++- src/analytics/options.h | 3 ++- src/analytics/test/options_test.cc | 15 +++++++++++++-- src/config/api-server/utils.py | 5 ++++- src/config/api-server/vnc_cfg_api_server.py | 4 ++++ .../device_manager/device_manager.py | 7 +++++++ src/config/schema-transformer/to_bgp.py | 7 +++++++ src/config/svc-monitor/svc_monitor/logger.py | 6 +++++- src/config/svc-monitor/svc_monitor/svc_monitor.py | 6 +++++- src/control-node/contrail-control.conf | 3 +++ src/control-node/main.cc | 2 ++ src/control-node/options.cc | 6 ++++++ src/control-node/options.h | 3 ++- src/control-node/test/options_test.cc | 15 +++++++++++++-- src/discovery/disc_server.py | 7 +++++++ src/discovery/disc_server_zk.py | 7 +++++++ src/dns/cmn/dns_options.cc | 6 ++++++ src/dns/cmn/dns_options.h | 2 ++ src/dns/contrail-dns.conf | 4 ++++ src/dns/main.cc | 1 + src/dns/test/dns_options_test.cc | 15 +++++++++++++-- src/nodemgr/main.py | 8 +++++++- src/opserver/alarmgen.py | 6 +++++- src/opserver/alarmgen_cfg.py | 8 +++++++- src/opserver/contrail-analytics-api.conf | 3 +++ src/opserver/opserver.py | 11 +++++++++-- src/query_engine/contrail-query-engine.conf | 3 +++ src/query_engine/options.cc | 7 +++++++ src/query_engine/options.h | 2 ++ src/query_engine/qed.cc | 1 + src/query_engine/test/options_test.cc | 15 +++++++++++++-- src/storage/stats-daemon/stats-daemon.conf | 3 +++ .../stats-daemon/stats_daemon/storage_nodemgr.py | 8 +++++++- src/vnsw/agent/cfg/discovery_agent.cc | 1 + src/vnsw/agent/cmn/agent.cc | 1 + src/vnsw/agent/cmn/agent.h | 4 +++- src/vnsw/agent/contrail-vrouter-agent.conf | 4 ++++ src/vnsw/agent/init/agent_param.cc | 14 +++++++++++++- src/vnsw/agent/init/agent_param.h | 5 +++++ 45 files changed, 234 insertions(+), 22 deletions(-) diff --git a/src/analytics/contrail-collector.conf b/src/analytics/contrail-collector.conf index 3ee6a868dc0..e75ae2fdb0a 100644 --- a/src/analytics/contrail-collector.conf +++ b/src/analytics/contrail-collector.conf @@ -64,6 +64,10 @@ log_local=1 # UDP port to listen on for receiving ipfix messages. -1 to disable. # ipfix_port=4739 +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +sandesh_send_rate_limit=100 + [COLLECTOR] # Everything in this section is optional diff --git a/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py b/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py index 6356048e431..1e4a53a7d3a 100644 --- a/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py +++ b/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py @@ -90,6 +90,7 @@ def parse(self): 'fast_scan_frequency' : 60, 'http_server_port' : 5920, 'zookeeper' : '127.0.0.1:2181', + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } ksopts = { 'auth_host': '127.0.0.1', @@ -176,6 +177,8 @@ def parse(self): help="where to look for snmp credentials") group.add_argument("--api_server", help="ip:port of api-server for snmp credentials") + group.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec.") self._args = parser.parse_args(remaining_argv) if type(self._args.collectors) is str: self._args.collectors = self._args.collectors.split() @@ -258,3 +261,5 @@ def frequency(self): def http_port(self): return self._args.http_server_port + def sandesh_send_rate_limit(self): + return self._args.sandesh_send_rate_limit diff --git a/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpuve.py b/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpuve.py index 07dd0096126..bd11fa8c5fb 100644 --- a/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpuve.py +++ b/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpuve.py @@ -36,6 +36,9 @@ def __init__(self, conf, instance='0'): self._node_type_name = NodeTypeNames[node_type] self._hostname = socket.gethostname() self._instance_id = instance + if self._conf.sandesh_send_rate_limit() is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._conf.sandesh_send_rate_limit()); sandesh_global.init_generator(self._moduleid, self._hostname, self._node_type_name, self._instance_id, self._conf.collectors(), diff --git a/src/analytics/contrail-topology/contrail_topology/config.py b/src/analytics/contrail-topology/contrail_topology/config.py index f728c16552b..10615f50b35 100644 --- a/src/analytics/contrail-topology/contrail_topology/config.py +++ b/src/analytics/contrail-topology/contrail_topology/config.py @@ -71,6 +71,7 @@ def parse(self): 'scan_frequency' : 60, 'http_server_port': 5921, 'zookeeper' : '127.0.0.1:2181', + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } config = None @@ -119,6 +120,8 @@ def parse(self): help="introspect server port") parser.add_argument("--zookeeper", help="ip:port of zookeeper server") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec.") self._args = parser.parse_args(remaining_argv) if type(self._args.collectors) is str: self._args.collectors = self._args.collectors.split() @@ -168,3 +171,5 @@ def frequency(self): def http_port(self): return self._args.http_server_port + def sandesh_send_rate_limit(self): + return self._args.sandesh_send_rate_limit diff --git a/src/analytics/contrail-topology/contrail_topology/topology_uve.py b/src/analytics/contrail-topology/contrail_topology/topology_uve.py index 1253f5c5ab4..d932d302349 100644 --- a/src/analytics/contrail-topology/contrail_topology/topology_uve.py +++ b/src/analytics/contrail-topology/contrail_topology/topology_uve.py @@ -19,6 +19,9 @@ def __init__(self, conf): self._node_type_name = NodeTypeNames[node_type] self._hostname = socket.gethostname() self._instance_id = '0' + if self._conf.sandesh_send_rate_limit() is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._conf.sandesh_send_rate_limit()) sandesh_global.init_generator(self._moduleid, self._hostname, self._node_type_name, self._instance_id, self._conf.collectors(), diff --git a/src/analytics/main.cc b/src/analytics/main.cc index f636e0b44fd..2a8c8f9bddf 100644 --- a/src/analytics/main.cc +++ b/src/analytics/main.cc @@ -356,6 +356,7 @@ int main(int argc, char *argv[]) unsigned short coll_port = analytics.GetCollector()->GetPort(); VizSandeshContext vsc(&analytics); + Sandesh::set_send_rate_limit(options.sandesh_send_rate_limit()); Sandesh::InitCollector( module_id, analytics.name(), diff --git a/src/analytics/options.cc b/src/analytics/options.cc index 7da5a3a1749..806a583be2f 100644 --- a/src/analytics/options.cc +++ b/src/analytics/options.cc @@ -151,7 +151,10 @@ void Options::Initialize(EventManager &evm, ("DEFAULT.disable_flow_collection", opt::bool_switch(&disable_flow_collection_), "Disable flow message collection") - + ("DEFAULT.sandesh_send_rate_limit", + opt::value()->default_value( + Sandesh::get_send_rate_limit()), + "Sandesh send rate limit in messages/sec") ("DISCOVERY.port", opt::value()->default_value( default_discovery_port), "Port of Discovery Server") @@ -310,6 +313,8 @@ void Options::Process(int argc, char *argv[], GetOptValue(var_map, syslog_port_, "DEFAULT.syslog_port"); GetOptValue(var_map, sflow_port_, "DEFAULT.sflow_port"); GetOptValue(var_map, ipfix_port_, "DEFAULT.ipfix_port"); + GetOptValue(var_map, sandesh_ratelimit_, + "DEFAULT.sandesh_send_rate_limit"); GetOptValue(var_map, discovery_port_, "DISCOVERY.port"); GetOptValue(var_map, discovery_server_, "DISCOVERY.server"); diff --git a/src/analytics/options.h b/src/analytics/options.h index f4a8ace576c..6e313cea7d2 100644 --- a/src/analytics/options.h +++ b/src/analytics/options.h @@ -58,6 +58,7 @@ class Options { const int ipfix_port() const { return ipfix_port_; } const bool test_mode() const { return test_mode_; } const bool disable_flow_collection() const { return disable_flow_collection_; } + const uint32_t sandesh_send_rate_limit() const { return sandesh_ratelimit_; } private: template @@ -121,6 +122,6 @@ class Options { std::vector kafka_broker_list_; uint16_t partitions_; bool disable_flow_collection_; - + uint32_t sandesh_ratelimit_; boost::program_options::options_description config_file_options_; }; diff --git a/src/analytics/test/options_test.cc b/src/analytics/test/options_test.cc index caa4d4f7401..cb703b0936d 100644 --- a/src/analytics/test/options_test.cc +++ b/src/analytics/test/options_test.cc @@ -82,6 +82,7 @@ TEST_F(OptionsTest, NoArguments) { EXPECT_EQ(options_.dup(), false); EXPECT_EQ(options_.test_mode(), false); EXPECT_EQ(options_.disable_flow_collection(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 0); uint16_t protobuf_port(0); EXPECT_FALSE(options_.collector_protobuf_port(&protobuf_port)); } @@ -124,19 +125,22 @@ TEST_F(OptionsTest, DefaultConfFile) { EXPECT_EQ(options_.dup(), false); EXPECT_EQ(options_.test_mode(), false); EXPECT_EQ(options_.disable_flow_collection(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 100); uint16_t protobuf_port(0); EXPECT_FALSE(options_.collector_protobuf_port(&protobuf_port)); } TEST_F(OptionsTest, OverrideStringFromCommandLine) { - int argc = 3; + int argc = 4; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=controller/src/analytics/contrail-collector.conf"; char argv_2[] = "--DEFAULT.log_file=test.log"; + char argv_3[] = "--DEFAULT.sandesh_send_rate_limit=5"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; + argv[3] = argv_3; options_.Parse(evm_, argc, argv); @@ -168,6 +172,7 @@ TEST_F(OptionsTest, OverrideStringFromCommandLine) { EXPECT_EQ(options_.dup(), false); EXPECT_EQ(options_.test_mode(), false); EXPECT_EQ(options_.disable_flow_collection(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); uint16_t protobuf_port(0); EXPECT_FALSE(options_.collector_protobuf_port(&protobuf_port)); } @@ -238,6 +243,7 @@ TEST_F(OptionsTest, CustomConfigFile) { "test_mode=1\n" "syslog_port=101\n" "disable_flow_collection=1\n" + "sandesh_send_rate_limit=5\n" "\n" "[COLLECTOR]\n" "port=100\n" @@ -303,6 +309,7 @@ TEST_F(OptionsTest, CustomConfigFile) { uint16_t protobuf_port(0); EXPECT_TRUE(options_.collector_protobuf_port(&protobuf_port)); EXPECT_EQ(protobuf_port, 3333); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); } TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { @@ -325,6 +332,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { "log_local=0\n" "test_mode=1\n" "syslog_port=102\n" + "sandesh_send_rate_limit=5\n" "\n" "[COLLECTOR]\n" "port=100\n" @@ -345,7 +353,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { config_file << config; config_file.close(); - int argc = 9; + int argc = 10; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=./options_test_collector_config_file.conf"; @@ -356,6 +364,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { char argv_6[] = "--DEFAULT.cassandra_server_list=21.20.20.2:200"; char argv_7[] = "--DEFAULT.cassandra_server_list=31.30.30.3:300"; char argv_8[] = "--COLLECTOR.protobuf_port=3334"; + char argv_9[] = "--DEFAULT.sandesh_send_rate_limit=7"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; @@ -365,6 +374,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { argv[6] = argv_6; argv[7] = argv_7; argv[8] = argv_8; + argv[9] = argv_9; options_.Parse(evm_, argc, argv); @@ -402,6 +412,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { uint16_t protobuf_port(0); EXPECT_TRUE(options_.collector_protobuf_port(&protobuf_port)); EXPECT_EQ(protobuf_port, 3334); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 7); } TEST_F(OptionsTest, MultitokenVector) { diff --git a/src/config/api-server/utils.py b/src/config/api-server/utils.py index 9f5cec57e83..d94158e727c 100644 --- a/src/config/api-server/utils.py +++ b/src/config/api-server/utils.py @@ -9,7 +9,7 @@ import ConfigParser import gen.resource_xsd import vnc_quota -from pysandesh.sandesh_base import Sandesh +from pysandesh.sandesh_base import Sandesh, SandeshSystem from pysandesh.gen_py.sandesh.ttypes import SandeshLevel _WEB_HOST = '0.0.0.0' @@ -64,6 +64,7 @@ def parse_args(args_str): 'rabbit_max_pending_updates': '4096', 'cluster_id': '', 'max_requests': 1024, + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } # ssl options secopts = { @@ -250,6 +251,8 @@ def parse_args(args_str): parser.add_argument( "--max_requests", type=int, help="Maximum number of concurrent requests served by api server") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec.") args_obj, remaining_argv = parser.parse_known_args(remaining_argv) args_obj.config_sections = config if type(args_obj.cassandra_server_list) is str: diff --git a/src/config/api-server/vnc_cfg_api_server.py b/src/config/api-server/vnc_cfg_api_server.py index 6b63b07cd11..c15a2fb88d5 100644 --- a/src/config/api-server/vnc_cfg_api_server.py +++ b/src/config/api-server/vnc_cfg_api_server.py @@ -333,6 +333,10 @@ def __init__(self, args_str=None): # sandesh init self._sandesh = Sandesh() + # Reset the sandesh send rate limit value + if self._args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._args.sandesh_send_rate_limit) module = Module.API_SERVER module_name = ModuleNames[Module.API_SERVER] node_type = Module2NodeType[module] diff --git a/src/config/device-manager/device_manager/device_manager.py b/src/config/device-manager/device_manager/device_manager.py index e4a00ab8e5d..76a27499d73 100644 --- a/src/config/device-manager/device_manager/device_manager.py +++ b/src/config/device-manager/device_manager/device_manager.py @@ -116,6 +116,10 @@ def __init__(self, args=None): ModuleNames[Module.DEVICE_MANAGER]) self._sandesh = Sandesh() + # Reset the sandesh send rate limit value + if self._args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._args.sandesh_send_rate_limit) module = Module.DEVICE_MANAGER module_name = ModuleNames[module] node_type = Module2NodeType[module] @@ -413,6 +417,7 @@ def parse_args(args_str): 'use_syslog': False, 'syslog_facility': Sandesh._DEFAULT_SYSLOG_FACILITY, 'cluster_id': '', + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } secopts = { 'use_certs': False, @@ -498,6 +503,8 @@ def parse_args(args_str): help="Tenant name for keystone admin user") parser.add_argument("--cluster_id", help="Used for database keyspace separation") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") args = parser.parse_args(remaining_argv) if type(args.cassandra_server_list) is str: args.cassandra_server_list = args.cassandra_server_list.split() diff --git a/src/config/schema-transformer/to_bgp.py b/src/config/schema-transformer/to_bgp.py index 901b3f221e2..0113a5acc9f 100644 --- a/src/config/schema-transformer/to_bgp.py +++ b/src/config/schema-transformer/to_bgp.py @@ -2746,6 +2746,10 @@ def __init__(self, args=None): ModuleNames[Module.SCHEMA_TRANSFORMER]) _sandesh = Sandesh() + # Reset the sandesh send rate limit value + if args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + args.sandesh_send_rate_limit) sandesh.VnList.handle_request = self.sandesh_vn_handle_request sandesh.RoutintInstanceList.handle_request = \ self.sandesh_ri_handle_request @@ -3909,6 +3913,7 @@ def parse_args(args_str): 'use_syslog': False, 'syslog_facility': Sandesh._DEFAULT_SYSLOG_FACILITY, 'cluster_id': '', + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } secopts = { 'use_certs': False, @@ -4005,6 +4010,8 @@ def parse_args(args_str): help="Tenant name for keystone admin user") parser.add_argument("--cluster_id", help="Used for database keyspace separation") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") args = parser.parse_args(remaining_argv) if type(args.cassandra_server_list) is str: args.cassandra_server_list = args.cassandra_server_list.split() diff --git a/src/config/svc-monitor/svc_monitor/logger.py b/src/config/svc-monitor/svc_monitor/logger.py index f15d329a0b9..3754b8586f9 100644 --- a/src/config/svc-monitor/svc_monitor/logger.py +++ b/src/config/svc-monitor/svc_monitor/logger.py @@ -15,7 +15,7 @@ from cfgm_common import vnc_cpu_info from cfgm_common.uve.service_instance.ttypes import * -from pysandesh.sandesh_base import Sandesh +from pysandesh.sandesh_base import Sandesh, SandeshSystem from pysandesh.gen_py.sandesh.ttypes import SandeshLevel from sandesh_common.vns.ttypes import Module, NodeType @@ -202,6 +202,10 @@ def uve_svc_instance(self, si_fq_name_str, status=None, # init sandesh def _sandesh_init(self, discovery): sandesh_instance = Sandesh() + # Reset the sandesh send rate limit value + if self._args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._args.sandesh_send_rate_limit) sandesh.ServiceInstanceList.handle_request =\ self.sandesh_si_handle_request sandesh_instance.init_generator( diff --git a/src/config/svc-monitor/svc_monitor/svc_monitor.py b/src/config/svc-monitor/svc_monitor/svc_monitor.py index 7fa2e2a786f..6e5f1cb6cd3 100644 --- a/src/config/svc-monitor/svc_monitor/svc_monitor.py +++ b/src/config/svc-monitor/svc_monitor/svc_monitor.py @@ -40,7 +40,7 @@ from config_db import * from cfgm_common.dependency_tracker import DependencyTracker -from pysandesh.sandesh_base import Sandesh +from pysandesh.sandesh_base import Sandesh, SandeshSystem from pysandesh.gen_py.sandesh.ttypes import SandeshLevel from pysandesh.gen_py.process_info.ttypes import ConnectionType, \ ConnectionStatus @@ -1035,6 +1035,7 @@ def parse_args(args_str): 'region_name': None, 'cluster_id': '', 'check_service_interval': '60', + 'sandesh_send_rate_limit' : SandeshSystem.get_sandesh_send_rate_limit(), } secopts = { 'use_certs': False, @@ -1158,6 +1159,9 @@ def parse_args(args_str): help="Region name for openstack API") parser.add_argument("--cluster_id", help="Used for database keyspace separation") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec.") + args = parser.parse_args(remaining_argv) args.config_sections = config if type(args.cassandra_server_list) is str: diff --git a/src/control-node/contrail-control.conf b/src/control-node/contrail-control.conf index 51cd3720bc6..75f461cca1d 100644 --- a/src/control-node/contrail-control.conf +++ b/src/control-node/contrail-control.conf @@ -21,6 +21,9 @@ log_local=1 # test_mode=0 # xmpp_server_port=5269 +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +sandesh_send_rate_limit=100 [DISCOVERY] # port=5998 # server=127.0.0.1 # discovery_server IP address diff --git a/src/control-node/main.cc b/src/control-node/main.cc index 76bd538c44b..3f54013d22e 100644 --- a/src/control-node/main.cc +++ b/src/control-node/main.cc @@ -546,6 +546,8 @@ int main(int argc, char *argv[]) { BgpSandeshContext sandesh_context; RegisterSandeshShowXmppExtensions(&sandesh_context); + + Sandesh::set_send_rate_limit(options.sandesh_send_rate_limit()); if (sandesh_generator_init) { NodeType::type node_type = g_vns_constants.Module2NodeType.find(module)->second; diff --git a/src/control-node/options.cc b/src/control-node/options.cc index dd66d79d8a8..7edee318228 100644 --- a/src/control-node/options.cc +++ b/src/control-node/options.cc @@ -131,6 +131,10 @@ void Options::Initialize(EventManager &evm, ("DEFAULT.xmpp_server_key", opt::value()->default_value("/etc/contrail/ssl/private/server.key"), "XMPP Server ssl private key") + ("DEFAULT.sandesh_send_rate_limit", + opt::value()->default_value( + Sandesh::get_send_rate_limit()), + "Sandesh send rate limit in messages/sec") ("DISCOVERY.port", opt::value()->default_value( default_discovery_port), @@ -262,6 +266,8 @@ bool Options::Process(int argc, char *argv[], GetOptValue(var_map, xmpp_auth_enable_, "DEFAULT.xmpp_auth_enable"); GetOptValue(var_map, xmpp_server_cert_, "DEFAULT.xmpp_server_cert"); GetOptValue(var_map, xmpp_server_key_, "DEFAULT.xmpp_server_key"); + GetOptValue(var_map, sandesh_ratelimit_, + "DEFAULT.sandesh_send_rate_limit"); GetOptValue(var_map, discovery_port_, "DISCOVERY.port"); GetOptValue(var_map, discovery_server_, "DISCOVERY.server"); diff --git a/src/control-node/options.h b/src/control-node/options.h index 60a5ed1b9a9..f7e12793686 100644 --- a/src/control-node/options.h +++ b/src/control-node/options.h @@ -44,6 +44,7 @@ class Options { const bool collectors_configured() const { return collectors_configured_; } const int tcp_hold_time() const { return tcp_hold_time_; } const bool optimize_snat() const { return optimize_snat_; } + const uint32_t sandesh_send_rate_limit() const { return sandesh_ratelimit_; } private: @@ -94,7 +95,7 @@ class Options { bool collectors_configured_; int tcp_hold_time_; bool optimize_snat_; - + uint32_t sandesh_ratelimit_; std::vector default_collector_server_list_; boost::program_options::options_description config_file_options_; }; diff --git a/src/control-node/test/options_test.cc b/src/control-node/test/options_test.cc index dafd9f8eea9..cbbcf841560 100644 --- a/src/control-node/test/options_test.cc +++ b/src/control-node/test/options_test.cc @@ -77,6 +77,7 @@ TEST_F(OptionsTest, NoArguments) { EXPECT_EQ(options_.ifmap_certs_store(), ""); EXPECT_EQ(options_.xmpp_port(), default_xmpp_port); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 0); } TEST_F(OptionsTest, DefaultConfFile) { @@ -113,17 +114,20 @@ TEST_F(OptionsTest, DefaultConfFile) { EXPECT_EQ(options_.ifmap_certs_store(), ""); EXPECT_EQ(options_.xmpp_port(), default_xmpp_port); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 100); } TEST_F(OptionsTest, OverrideStringFromCommandLine) { - int argc = 3; + int argc = 4; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=controller/src/control-node/contrail-control.conf"; char argv_2[] = "--DEFAULT.log_file=test.log"; + char argv_3[] = "--DEFAULT.sandesh_send_rate_limit=5"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; + argv[3] = argv_3; options_.Parse(evm_, argc, argv); @@ -151,6 +155,7 @@ TEST_F(OptionsTest, OverrideStringFromCommandLine) { EXPECT_EQ(options_.ifmap_certs_store(), ""); EXPECT_EQ(options_.xmpp_port(), default_xmpp_port); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); } TEST_F(OptionsTest, OverrideBooleanFromCommandLine) { @@ -211,6 +216,7 @@ TEST_F(OptionsTest, CustomConfigFile) { "log_local=1\n" "test_mode=1\n" "xmpp_server_port=100\n" + "sandesh_send_rate_limit=5\n" "\n" "[DISCOVERY]\n" "port=100\n" @@ -265,6 +271,7 @@ TEST_F(OptionsTest, CustomConfigFile) { EXPECT_EQ(options_.ifmap_certs_store(), "test-store"); EXPECT_EQ(options_.xmpp_port(), 100); EXPECT_EQ(options_.test_mode(), true); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); } TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { @@ -287,6 +294,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { "log_local=0\n" "test_mode=1\n" "xmpp_server_port=100\n" + "sandesh_send_rate_limit=5\n" "\n" "[DISCOVERY]\n" "port=100\n" @@ -303,7 +311,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { config_file << config; config_file.close(); - int argc = 7; + int argc = 8; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=./options_test_config_file.conf"; @@ -312,6 +320,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { char argv_4[] = "--DEFAULT.collectors=11.10.10.1:100"; char argv_5[] = "--DEFAULT.collectors=21.20.20.2:200"; char argv_6[] = "--DEFAULT.collectors=31.30.30.3:300"; + char argv_7[] = "--DEFAULT.sandesh_send_rate_limit=7"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; @@ -319,6 +328,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { argv[4] = argv_4; argv[5] = argv_5; argv[6] = argv_6; + argv[7] = argv_7; options_.Parse(evm_, argc, argv); @@ -352,6 +362,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { EXPECT_EQ(options_.ifmap_certs_store(), "test-store"); EXPECT_EQ(options_.xmpp_port(), 100); EXPECT_EQ(options_.test_mode(), true); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 7); } TEST_F(OptionsTest, CustomConfigFileWithInvalidHostIp) { diff --git a/src/discovery/disc_server.py b/src/discovery/disc_server.py index e8a51abc3b9..7cec4222e60 100644 --- a/src/discovery/disc_server.py +++ b/src/discovery/disc_server.py @@ -176,6 +176,9 @@ def __init__(self, args): # sandesh init self._sandesh = Sandesh() + if self._args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._args.sandesh_send_rate_limit) module = Module.DISCOVERY_SERVICE module_name = ModuleNames[module] node_type = Module2NodeType[module] @@ -1084,6 +1087,7 @@ def parse_args(args_str): 'log_category': '', 'log_file': Sandesh._DEFAULT_LOG_FILE, 'worker_id': '0', + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } # per service options @@ -1170,6 +1174,9 @@ def parse_args(args_str): parser.add_argument( "--worker_id", help="Worker Id") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") + args = parser.parse_args(remaining_argv) args.conf_file = args.conf_file args.service_config = service_config diff --git a/src/discovery/disc_server_zk.py b/src/discovery/disc_server_zk.py index c97da564f0d..a4400bcac3f 100644 --- a/src/discovery/disc_server_zk.py +++ b/src/discovery/disc_server_zk.py @@ -175,6 +175,10 @@ def __init__(self, args): # sandesh init self._sandesh = Sandesh() + # Reset the sandesh send rate limit value + if self._args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._args.sandesh_send_rate_limit) module = Module.DISCOVERY_SERVICE module_name = ModuleNames[module] node_type = Module2NodeType[module] @@ -1014,6 +1018,7 @@ def parse_args(args_str): 'use_syslog': False, 'syslog_facility': Sandesh._DEFAULT_SYSLOG_FACILITY, 'worker_id': '0', + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } # per service options @@ -1094,6 +1099,8 @@ def parse_args(args_str): parser.add_argument( "--worker_id", help="Worker Id") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") args = parser.parse_args(remaining_argv) args.conf_file = args.conf_file args.service_config = service_config diff --git a/src/dns/cmn/dns_options.cc b/src/dns/cmn/dns_options.cc index 5410cb9ed0b..306ecf9c361 100644 --- a/src/dns/cmn/dns_options.cc +++ b/src/dns/cmn/dns_options.cc @@ -127,6 +127,10 @@ void Options::Initialize(EventManager &evm, "Syslog facility to receive log lines") ("DEFAULT.test_mode", opt::bool_switch(&test_mode_), "Enable dns to run in test-mode") + ("DEFAULT.sandesh_send_rate_limit", + opt::value()->default_value( + Sandesh::get_send_rate_limit()), + "Sandesh send rate limit in messages/sec") ("DISCOVERY.port", opt::value()->default_value( default_discovery_port), @@ -255,6 +259,8 @@ void Options::Process(int argc, char *argv[], GetOptValue(var_map, log_level_, "DEFAULT.log_level"); GetOptValue(var_map, use_syslog_, "DEFAULT.use_syslog"); GetOptValue(var_map, syslog_facility_, "DEFAULT.syslog_facility"); + GetOptValue(var_map, send_ratelimit_, + "DEFAULT.sandesh_send_rate_limit"); GetOptValue(var_map, discovery_port_, "DISCOVERY.port"); GetOptValue(var_map, discovery_server_, "DISCOVERY.server"); diff --git a/src/dns/cmn/dns_options.h b/src/dns/cmn/dns_options.h index 518cceaf531..4f9b740eed6 100644 --- a/src/dns/cmn/dns_options.h +++ b/src/dns/cmn/dns_options.h @@ -46,6 +46,7 @@ class Options { const std::string ifmap_certs_store() const { return ifmap_certs_store_; } const bool test_mode() const { return test_mode_; } const bool collectors_configured() const { return collectors_configured_; } + const uint32_t sandesh_send_rate_limit() const { return send_ratelimit_; } private: @@ -99,6 +100,7 @@ class Options { bool test_mode_; bool collectors_configured_; std::vector default_collector_server_list_; + uint32_t send_ratelimit_; boost::program_options::options_description config_file_options_; }; diff --git a/src/dns/contrail-dns.conf b/src/dns/contrail-dns.conf index 5b64385b7b4..03cb0a1b608 100644 --- a/src/dns/contrail-dns.conf +++ b/src/dns/contrail-dns.conf @@ -25,6 +25,10 @@ log_local=1 # test_mode=0 # log_property_file= # log4cplus property file +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +sandesh_send_rate_limit=100 + [DISCOVERY] # port=5998 # server=127.0.0.1 # discovery_server IP address diff --git a/src/dns/main.cc b/src/dns/main.cc index cde36e36849..769aaa10da4 100644 --- a/src/dns/main.cc +++ b/src/dns/main.cc @@ -130,6 +130,7 @@ int main(int argc, char *argv[]) { boost::system::error_code ec; string hostname = host_name(ec); Dns::SetHostName(hostname); + Sandesh::set_send_rate_limit(options.sandesh_send_rate_limit()); if (options.discovery_server().empty()) { NodeType::type node_type = g_vns_constants.Module2NodeType.find(module)->second; diff --git a/src/dns/test/dns_options_test.cc b/src/dns/test/dns_options_test.cc index af0ee30bf58..890f993e1a4 100644 --- a/src/dns/test/dns_options_test.cc +++ b/src/dns/test/dns_options_test.cc @@ -76,6 +76,7 @@ TEST_F(OptionsTest, NoArguments) { EXPECT_EQ(options_.ifmap_user(), "dns_user"); EXPECT_EQ(options_.ifmap_certs_store(), ""); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 0); } TEST_F(OptionsTest, DefaultConfFile) { @@ -118,10 +119,11 @@ TEST_F(OptionsTest, DefaultConfFile) { EXPECT_EQ(options_.ifmap_user(), "dns_user"); EXPECT_EQ(options_.ifmap_certs_store(), ""); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 100); } TEST_F(OptionsTest, OverrideStringFromCommandLine) { - int argc = 6; + int argc = 7; char *argv[argc]; char argv_0[] = "dns_options_test"; char argv_1[] = "--conf_file=controller/src/dns/contrail-dns.conf"; @@ -129,12 +131,14 @@ TEST_F(OptionsTest, OverrideStringFromCommandLine) { char argv_3[] = "--DEFAULT.rndc_config_file=test.rndc"; char argv_4[] = "--DEFAULT.rndc_secret=secret123"; char argv_5[] = "--DEFAULT.log_property_file=log4cplus.prop"; + char argv_6[] = "--DEFAULT.sandesh_send_rate_limit=5"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; argv[3] = argv_3; argv[4] = argv_4; argv[5] = argv_5; + argv[6] = argv_6; options_.Parse(evm_, argc, argv); @@ -167,6 +171,7 @@ TEST_F(OptionsTest, OverrideStringFromCommandLine) { EXPECT_EQ(options_.ifmap_user(), "dns_user"); EXPECT_EQ(options_.ifmap_certs_store(), ""); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); } TEST_F(OptionsTest, OverrideBooleanFromCommandLine) { @@ -231,6 +236,7 @@ TEST_F(OptionsTest, CustomConfigFile) { "log_local=1\n" "test_mode=1\n" "log_property_file=log4cplus.prop\n" + "sandesh_send_rate_limit=5\n" "\n" "[DISCOVERY]\n" "port=100\n" @@ -289,6 +295,7 @@ TEST_F(OptionsTest, CustomConfigFile) { EXPECT_EQ(options_.ifmap_user(), "test-user"); EXPECT_EQ(options_.ifmap_certs_store(), "test-store"); EXPECT_EQ(options_.test_mode(), true); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); std::remove("./dns_options_test_config_file.conf"); } @@ -316,6 +323,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { "log_level=SYS_DEBUG\n" "log_local=0\n" "test_mode=1\n" + "sandesh_send_rate_limit=5\n" "\n" "[DISCOVERY]\n" "port=100\n" @@ -332,7 +340,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { config_file << config; config_file.close(); - int argc = 10; + int argc = 11; char *argv[argc]; char argv_0[] = "dns_options_test"; char argv_1[] = "--conf_file=./dns_options_test_config_file.conf"; @@ -344,6 +352,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { char argv_7[] = "--DEFAULT.named_config_directory=/etc/contrail/dns/test"; char argv_8[] = "--DEFAULT.rndc_config_file=new.rndc"; char argv_9[] = "--DEFAULT.rndc_secret=new-secret-123"; + char argv_10[] = "--DEFAULT.sandesh_send_rate_limit=7"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; @@ -354,6 +363,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { argv[7] = argv_7; argv[8] = argv_8; argv[9] = argv_9; + argv[10] = argv_10; options_.Parse(evm_, argc, argv); @@ -389,6 +399,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { EXPECT_EQ(options_.ifmap_user(), "test-user"); EXPECT_EQ(options_.ifmap_certs_store(), "test-store"); EXPECT_EQ(options_.test_mode(), true); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 7); std::remove("./dns_options_test_config_file.conf"); } diff --git a/src/nodemgr/main.py b/src/nodemgr/main.py index bda730585b2..e824a813cc5 100755 --- a/src/nodemgr/main.py +++ b/src/nodemgr/main.py @@ -46,7 +46,7 @@ from nodemgr.config_event_manager import ConfigEventManager from nodemgr.vrouter_event_manager import VrouterEventManager from nodemgr.database_event_manager import DatabaseEventManager - +from pysandesh.sandesh_base import SandeshSystem def usage(): print doc @@ -69,6 +69,8 @@ def main(args_str=' '.join(sys.argv[1:])): 'hostip': '127.0.0.1', 'minimum_diskgb': 256, 'cassandra_repair_interval': 24, + 'sandesh_send_rate_limit': \ + SandeshSystem.get_sandesh_send_rate_limit(), } node_type = args.nodetype if (node_type == 'contrail-analytics'): @@ -114,6 +116,8 @@ def main(args_str=' '.join(sys.argv[1:])): nargs='+', help='Collector addresses in format' + 'ip1:port1 ip2:port2') + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") if (node_type == 'contrail-database'): parser.add_argument("--minimum_diskgb", type=int, @@ -134,6 +138,8 @@ def main(args_str=' '.join(sys.argv[1:])): sys.stderr.write("Discovery port: " + str(discovery_port) + "\n") collector_addr = _args.collectors sys.stderr.write("Collector address: " + str(collector_addr) + "\n") + if _args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit(_args.sandesh_send_rate_limit) # done parsing arguments if not 'SUPERVISOR_SERVER_URL' in os.environ: diff --git a/src/opserver/alarmgen.py b/src/opserver/alarmgen.py index ae17e093c4d..cb3b3e2b066 100644 --- a/src/opserver/alarmgen.py +++ b/src/opserver/alarmgen.py @@ -61,9 +61,13 @@ def __init__(self, conf): self._node_type_name = NodeTypeNames[node_type] self._hostname = socket.gethostname() self._instance_id = self._conf.worker_id() + # Reset the sandesh send rate limit value + if self._conf.sandesh_send_rate_limit() is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._conf.sandesh_send_rate_limit()) sandesh_global.init_generator(self._moduleid, self._hostname, self._node_type_name, self._instance_id, - self._conf.collectors(), + self._conf.collectors(), self._node_type_name, self._conf.http_port(), ['opserver.sandesh', 'sandesh']) diff --git a/src/opserver/alarmgen_cfg.py b/src/opserver/alarmgen_cfg.py index cab4d78fced..b282d15e160 100644 --- a/src/opserver/alarmgen_cfg.py +++ b/src/opserver/alarmgen_cfg.py @@ -63,7 +63,8 @@ def parse(self): 'partitions' : 5, 'zk_list' : None, 'redis_uve_list' : ['127.0.0.1:6379'], - 'alarmgen_list' : ['127.0.0.1:0'] + 'alarmgen_list' : ['127.0.0.1:0'], + 'sandesh_send_rate_limit' : SandeshSystem.get_sandesh_send_rate_limit(), } redis_opts = { @@ -147,6 +148,8 @@ def parse(self): parser.add_argument("--alarmgen_list", help="List of alarmgens in ip:inst format. For internal use only", nargs="+") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") self._args = parser.parse_args(remaining_argv) if type(self._args.collectors) is str: self._args.collectors = self._args.collectors.split() @@ -218,3 +221,6 @@ def redis_password(self): def redis_server_port(self): return self._args.redis_server_port + + def sandesh_send_rate_limit(self): + return self._args.sandesh_send_rate_limit diff --git a/src/opserver/contrail-analytics-api.conf b/src/opserver/contrail-analytics-api.conf index efb7b68658b..1d94b62499b 100644 --- a/src/opserver/contrail-analytics-api.conf +++ b/src/opserver/contrail-analytics-api.conf @@ -9,6 +9,9 @@ log_local = 1 log_level = SYS_NOTICE #log_category = log_file = /var/log/contrail/contrail-analytics-api.log +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +#sandesh_send_rate_limit = 100 [DISCOVERY] #disc_server_ip = 127.0.0.1 diff --git a/src/opserver/opserver.py b/src/opserver/opserver.py index 07ca9f1740e..37e40ebd5a8 100644 --- a/src/opserver/opserver.py +++ b/src/opserver/opserver.py @@ -475,6 +475,10 @@ def __init__(self): if self._args.dup: self._hostname += 'dup' opserver_sandesh_req_impl = OpserverSandeshReqImpl(self) + # Reset the sandesh send rate limit value + if self._args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + self._args.sandesh_send_rate_limit) sandesh_global.init_generator(self._moduleid, self._hostname, self._node_type_name, self._instance_id, self._args.collectors, 'opserver_context', @@ -759,7 +763,9 @@ def _parse_args(self, args_str=' '.join(sys.argv[1:])): 'analytics_data_ttl' : 48, 'analytics_config_audit_ttl' : -1, 'analytics_statistics_ttl' : -1, - 'analytics_flow_ttl' : -1 + 'analytics_flow_ttl' : -1, + 'sandesh_send_rate_limit': SandeshSystem. \ + get_sandesh_send_rate_limit(), } redis_opts = { 'redis_server_port' : 6379, @@ -851,7 +857,8 @@ def _parse_args(self, args_str=' '.join(sys.argv[1:])): nargs="+") parser.add_argument("--auto_db_purge", action="store_true", help="Automatically purge database if disk usage cross threshold") - + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") self._args = parser.parse_args(remaining_argv) if type(self._args.collectors) is str: self._args.collectors = self._args.collectors.split() diff --git a/src/query_engine/contrail-query-engine.conf b/src/query_engine/contrail-query-engine.conf index 95aa4072d55..ad7c4d788c2 100644 --- a/src/query_engine/contrail-query-engine.conf +++ b/src/query_engine/contrail-query-engine.conf @@ -22,6 +22,9 @@ log_local=1 # max_tasks=16 # start_time=0 # test_mode=0 +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +sandesh_send_rate_limit=100 [DISCOVERY] # port=5998 diff --git a/src/query_engine/options.cc b/src/query_engine/options.cc index cc41e045837..be06c62478a 100644 --- a/src/query_engine/options.cc +++ b/src/query_engine/options.cc @@ -120,6 +120,11 @@ void Options::Initialize(EventManager &evm, ("DEFAULT.test_mode", opt::bool_switch(&test_mode_), "Enable query-engine to run in test-mode") + ("DEFAULT.sandesh_send_rate_limit", + opt::value()->default_value( + Sandesh::get_send_rate_limit()), + "Sandesh send rate limit in messages/sec") + ("DISCOVERY.port", opt::value()->default_value( default_discovery_port), "Port of Discovery Server") @@ -231,6 +236,8 @@ void Options::Process(int argc, char *argv[], GetOptValue(var_map, start_time_, "DEFAULT.start_time"); GetOptValue(var_map, max_tasks_, "DEFAULT.max_tasks"); GetOptValue(var_map, max_slice_, "DEFAULT.max_slice"); + GetOptValue(var_map, send_ratelimit_, + "DEFAULT.sandesh_send_rate_limit"); GetOptValue(var_map, discovery_port_, "DISCOVERY.port"); GetOptValue(var_map, discovery_server_, "DISCOVERY.server"); diff --git a/src/query_engine/options.h b/src/query_engine/options.h index b1060b22ddf..4493f1a37cd 100644 --- a/src/query_engine/options.h +++ b/src/query_engine/options.h @@ -43,6 +43,7 @@ class Options { const std::string syslog_facility() const { return syslog_facility_; } const int analytics_data_ttl() const { return analytics_data_ttl_; } const bool test_mode() const { return test_mode_; } + const uint32_t sandesh_send_rate_limit() const { return send_ratelimit_; } private: @@ -86,6 +87,7 @@ class Options { int max_slice_; bool test_mode_; int analytics_data_ttl_; + uint32_t send_ratelimit_; std::vector cassandra_server_list_; std::vector collector_server_list_; diff --git a/src/query_engine/qed.cc b/src/query_engine/qed.cc index 842bf1bbf36..59159daf99f 100644 --- a/src/query_engine/qed.cc +++ b/src/query_engine/qed.cc @@ -209,6 +209,7 @@ main(int argc, char *argv[]) { options.hostname(), module_name, instance_id, boost::bind(&GetProcessStateCb, _1, _2, _3, 3)); + Sandesh::set_send_rate_limit(options.sandesh_send_rate_limit()); Sandesh::InitGenerator( module_name, options.hostname(), diff --git a/src/query_engine/test/options_test.cc b/src/query_engine/test/options_test.cc index e015a346d6d..8871648059f 100644 --- a/src/query_engine/test/options_test.cc +++ b/src/query_engine/test/options_test.cc @@ -80,6 +80,7 @@ TEST_F(OptionsTest, NoArguments) { EXPECT_EQ(options_.max_tasks(), 0); EXPECT_EQ(options_.max_slice(), 100); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 0); } TEST_F(OptionsTest, DefaultConfFile) { @@ -117,17 +118,20 @@ TEST_F(OptionsTest, DefaultConfFile) { EXPECT_EQ(options_.max_tasks(), 0); EXPECT_EQ(options_.max_slice(), 100); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 100); } TEST_F(OptionsTest, OverrideStringFromCommandLine) { - int argc = 3; + int argc = 4; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=controller/src/query_engine/contrail-query-engine.conf"; char argv_2[] = "--DEFAULT.log_file=test.log"; + char argv_3[] = "--DEFAULT.sandesh_send_rate_limit=5"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; + argv[3] = argv_3; options_.Parse(evm_, argc, argv); @@ -156,6 +160,7 @@ TEST_F(OptionsTest, OverrideStringFromCommandLine) { EXPECT_EQ(options_.max_tasks(), 0); EXPECT_EQ(options_.max_slice(), 100); EXPECT_EQ(options_.test_mode(), false); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); } TEST_F(OptionsTest, OverrideBooleanFromCommandLine) { @@ -220,6 +225,7 @@ TEST_F(OptionsTest, CustomConfigFile) { "start_time=123456\n" "max_tasks=200\n" "max_slice=500\n" + "sandesh_send_rate_limit=5\n" "\n" "[DISCOVERY]\n" "port=100\n" @@ -279,6 +285,7 @@ TEST_F(OptionsTest, CustomConfigFile) { EXPECT_EQ(options_.max_tasks(), 200); EXPECT_EQ(options_.max_slice(), 500); EXPECT_EQ(options_.test_mode(), true); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 5); } TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { @@ -305,6 +312,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { "start_time=543457\n" "max_tasks=900\n" "max_slice=800\n" + "sandesh_send_rate_limit=5\n" "\n" "[DISCOVERY]\n" "port=100\n" @@ -320,7 +328,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { config_file << config; config_file.close(); - int argc = 15; + int argc = 16; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=./options_test_query_engine.conf"; @@ -336,6 +344,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { char argv_11[] = "--DEFAULT.collectors=11.10.10.1:100"; char argv_12[] = "--DEFAULT.collectors=21.20.20.2:200"; char argv_13[] = "--DEFAULT.collectors=31.30.30.3:300"; + char argv_14[] = "--DEFAULT.sandesh_send_rate_limit=7"; argv[0] = argv_0; argv[1] = argv_1; argv[2] = argv_2; @@ -350,6 +359,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { argv[11] = argv_11; argv[12] = argv_12; argv[14] = argv_13; + argv[15] = argv_14; options_.Parse(evm_, argc, argv); @@ -388,6 +398,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { EXPECT_EQ(options_.max_tasks(), 900); EXPECT_EQ(options_.max_slice(), 800); EXPECT_EQ(options_.test_mode(), true); + EXPECT_EQ(options_.sandesh_send_rate_limit(), 7); } TEST_F(OptionsTest, MultitokenVector) { diff --git a/src/storage/stats-daemon/stats-daemon.conf b/src/storage/stats-daemon/stats-daemon.conf index dff13ea7dbd..b5e6171947c 100644 --- a/src/storage/stats-daemon/stats-daemon.conf +++ b/src/storage/stats-daemon/stats-daemon.conf @@ -5,3 +5,6 @@ log_file=/var/log/contrail/contrail-storage-stats.log log_level=SYS_NOTICE log_local=1 disc_server_ip=127.0.0.1 +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +# sandesh_send_rate_limit=100 diff --git a/src/storage/stats-daemon/stats_daemon/storage_nodemgr.py b/src/storage/stats-daemon/stats_daemon/storage_nodemgr.py index 23a5d11a815..5453c277646 100644 --- a/src/storage/stats-daemon/stats_daemon/storage_nodemgr.py +++ b/src/storage/stats-daemon/stats_daemon/storage_nodemgr.py @@ -678,7 +678,8 @@ def parse_args(args_str): 'log_local': True, 'log_level': 'SYS_NOTICE', 'log_category': '', - 'log_file': Sandesh._DEFAULT_LOG_FILE + 'log_file': Sandesh._DEFAULT_LOG_FILE, + 'sandesh_send_rate_limit': SandeshSystem.get_sandesh_send_rate_limit(), } if args.conf_file: @@ -713,6 +714,8 @@ def parse_args(args_str): help="Category filter for local logging of sandesh messages") parser.add_argument("--log_file", help="Filename for the logs to be written to") + parser.add_argument("--sandesh_send_rate_limit", type=int, + help="Sandesh send rate limit in messages/sec") args = parser.parse_args(remaining_argv) return args @@ -742,6 +745,9 @@ def main(args_str=None): _disc = client.DiscoveryClient(args.disc_server_ip, args.disc_server_port, module_name) + if args.sandesh_send_rate_limit is not None: + SandeshSystem.set_sandesh_send_rate_limit( \ + args.sandesh_send_rate_limit) sandesh_global.init_generator( module_name, socket.gethostname(), diff --git a/src/vnsw/agent/cfg/discovery_agent.cc b/src/vnsw/agent/cfg/discovery_agent.cc index ac847cfb666..b3808f2bbcb 100644 --- a/src/vnsw/agent/cfg/discovery_agent.cc +++ b/src/vnsw/agent/cfg/discovery_agent.cc @@ -147,6 +147,7 @@ void DiscoveryAgentClient::DiscoverServices() { ds_client, _1, _2, _3); std::vector list; list.clear(); + Sandesh::set_send_rate_limit(agent->sandesh_send_rate_limit()); Sandesh::InitGenerator(agent->module_name(), agent->host_name(), node_type_name, diff --git a/src/vnsw/agent/cmn/agent.cc b/src/vnsw/agent/cmn/agent.cc index 01f827bee53..46cf909e65d 100644 --- a/src/vnsw/agent/cmn/agent.cc +++ b/src/vnsw/agent/cmn/agent.cc @@ -316,6 +316,7 @@ void Agent::InitCollector() { Module::type module = static_cast(module_type_); NodeType::type node_type = g_vns_constants.Module2NodeType.find(module)->second; + Sandesh::set_send_rate_limit(params_->sandesh_send_rate_limit()); if (params_->collector_server_list().size() != 0) { Sandesh::InitGenerator(module_name(), host_name(), diff --git a/src/vnsw/agent/cmn/agent.h b/src/vnsw/agent/cmn/agent.h index 01bea35e526..bc3cfdbb70a 100644 --- a/src/vnsw/agent/cmn/agent.h +++ b/src/vnsw/agent/cmn/agent.h @@ -774,6 +774,8 @@ class Agent { int introspect_port() const { return introspect_port_;} + uint32_t sandesh_send_rate_limit() { return send_ratelimit_; } + DB *db() const {return db_;} TaskScheduler *task_scheduler() const { return task_scheduler_; } @@ -976,7 +978,7 @@ class Agent { std::string instance_id_; int module_type_; std::string module_name_; - + uint32_t send_ratelimit_; // DB handles DB *db_; TaskScheduler *task_scheduler_; diff --git a/src/vnsw/agent/contrail-vrouter-agent.conf b/src/vnsw/agent/contrail-vrouter-agent.conf index af9b905cc8a..78ce63d0a7d 100644 --- a/src/vnsw/agent/contrail-vrouter-agent.conf +++ b/src/vnsw/agent/contrail-vrouter-agent.conf @@ -63,6 +63,10 @@ log_local=1 # interface with an unconfigured IP should be relayed or not # dhcp_relay_mode= +# Sandesh send rate limit can be used to throttle system logs transmitted per +# second. System logs are dropped if the sending rate is exceeded +# sandesh_send_rate_limit=100 + [DISCOVERY] #If DEFAULT.collectors and/or CONTROL-NODE and/or DNS is not specified this #section is mandatory. Else this section is optional diff --git a/src/vnsw/agent/init/agent_param.cc b/src/vnsw/agent/init/agent_param.cc index 9c429e33327..a8639c44f5b 100644 --- a/src/vnsw/agent/init/agent_param.cc +++ b/src/vnsw/agent/init/agent_param.cc @@ -453,6 +453,11 @@ void AgentParam::ParseDefaultSection() { if (!GetValueFromTree(tcp_hold_time_, "DEFAULT.tcp_hold_time")) { tcp_hold_time_ = 30; } + + if (!GetValueFromTree(send_ratelimit_, + "DEFAULT.sandesh_send_rate_limit")) { + send_ratelimit_ = Sandesh::get_send_rate_limit(); + } } void AgentParam::ParseTaskSection() { @@ -668,6 +673,8 @@ void AgentParam::ParseDefaultSectionArguments GetOptValue(var_map, xmpp_auth_enable_, "DEFAULT.xmpp_auth_enable"); GetOptValue(var_map, xmpp_server_cert_, "DEFAULT.xmpp_server_cert"); GetValueFromTree(tcp_hold_time_, "DEFAULT.tcp_hold_time"); + GetOptValue(var_map, send_ratelimit_, + "DEFAULT.sandesh_send_rate_limit"); } void AgentParam::ParseTaskSectionArguments @@ -1153,7 +1160,8 @@ AgentParam::AgentParam(Agent *agent, bool enable_flow_options, agent_base_dir_(), tbb_exec_delay_(0), tbb_schedule_delay_(0), - tbb_keepawake_timeout_(Agent::kDefaultTbbKeepawakeTimeout) { + tbb_keepawake_timeout_(Agent::kDefaultTbbKeepawakeTimeout), + send_ratelimit_(sandesh_send_rate_limit()) { vgw_config_table_ = std::auto_ptr (new VirtualGatewayConfigTable(agent)); @@ -1211,6 +1219,10 @@ AgentParam::AgentParam(Agent *agent, bool enable_flow_options, "control-channel IP address used by WEB-UI to connect to vnswad") ("DEFAULT.platform", opt::value(), "Mode in which vrouter is running, option are dpdk or vnic") + ("DEFAULT.sandesh_send_rate_limit", + opt::value()->default_value( + Sandesh::get_send_rate_limit()), + "Sandesh send rate limit in messages/sec") ; options_.add(generic); diff --git a/src/vnsw/agent/init/agent_param.h b/src/vnsw/agent/init/agent_param.h index facf3751502..90eccf86342 100644 --- a/src/vnsw/agent/init/agent_param.h +++ b/src/vnsw/agent/init/agent_param.h @@ -219,6 +219,8 @@ class AgentParam { uint32_t tbb_exec_delay() const { return tbb_exec_delay_; } uint32_t tbb_schedule_delay() const { return tbb_schedule_delay_; } uint32_t tbb_keepawake_timeout() const { return tbb_keepawake_timeout_; } + uint32_t sandesh_send_rate_limit() { return send_ratelimit_; } + protected: void set_hypervisor_mode(HypervisorMode m) { hypervisor_mode_ = m; } virtual void InitFromSystem(); @@ -414,6 +416,9 @@ class AgentParam { uint32_t tbb_exec_delay_; uint32_t tbb_schedule_delay_; uint32_t tbb_keepawake_timeout_; + + uint32_t send_ratelimit_; + DISALLOW_COPY_AND_ASSIGN(AgentParam); };