From a169eca59da9841a18f48268784ae8479d855afb Mon Sep 17 00:00:00 2001 From: Megh Bhatt Date: Wed, 22 Feb 2017 16:38:50 -0800 Subject: [PATCH] Fix issues related to default configuration file for analytics daemons 1. Currently the analytics daemons do not pick up their default configuration file and so the supervisord ini file or the systemd service file have to launch them with the --config_file option on command line. This causes issue when either new init system is used or new provisioning method is used or when upgrading and the daemon needs an additional default configuration file 2. Change the analytics daemons to have default configuration files so that they can be launched without specifying --config_file option in the supervisord ini file or the systemd service file 3. Cleanup to display the default values when --help option is used 4. Cleanup of contrail-collector and contrail-query-engine --help display to club options based on sections Change-Id: Ibd054e4ec753f7ee292f65cffd472e88b1fb62fd Partial-Bug: #1667082 --- .../contrail-snmp-collector.ini | 2 +- .../contrail_snmp_collector/snmpcfg.py | 12 +- .../contrail_topology/config.py | 12 +- src/analytics/options.cc | 134 ++++++++++++------ src/analytics/test/options_test.cc | 8 +- src/config/utils/contrail-status.py | 33 +++-- src/opserver/alarmgen_cfg.py | 12 +- src/opserver/contrail-alarm-gen.ini | 8 +- src/opserver/contrail-analytics-api.ini | 2 +- src/opserver/opserver.py | 7 +- .../systemd/contrail-alarm-gen.service | 2 +- src/query_engine/options.cc | 40 +++++- src/query_engine/test/SConscript | 2 +- src/query_engine/test/options_test.cc | 45 +++--- src/sandesh/common/vns.sandesh | 52 ++++--- 15 files changed, 239 insertions(+), 132 deletions(-) diff --git a/src/analytics/contrail-snmp-collector/contrail-snmp-collector.ini b/src/analytics/contrail-snmp-collector/contrail-snmp-collector.ini index 53227d049a7..c7a1b5f225a 100644 --- a/src/analytics/contrail-snmp-collector/contrail-snmp-collector.ini +++ b/src/analytics/contrail-snmp-collector/contrail-snmp-collector.ini @@ -1,5 +1,5 @@ [program:contrail-snmp-collector] -command=/usr/bin/contrail-snmp-collector --device-config-file /etc/contrail/device.ini +command=/usr/bin/contrail-snmp-collector priority=340 autostart=true killasgroup=true diff --git a/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py b/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py index 02d80140794..e7f569f9823 100644 --- a/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py +++ b/src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py @@ -8,10 +8,12 @@ import discoveryclient.client as client from sandesh_common.vns.ttypes import Module from sandesh_common.vns.constants import ModuleNames, \ - API_SERVER_DISCOVERY_SERVICE_NAME, HttpPortSnmpCollector + API_SERVER_DISCOVERY_SERVICE_NAME, HttpPortSnmpCollector, \ + ServicesDefaultConfigurationFiles, SERVICE_SNMP_COLLECTOR class CfgParser(object): - CONF_DEFAULT_PATH = '/etc/contrail/contrail-snmp-collector.conf' + CONF_DEFAULT_PATHS = ServicesDefaultConfigurationFiles.get( + SERVICE_SNMP_COLLECTOR, None) def __init__(self, argv): self._devices = [] self._args = None @@ -72,9 +74,7 @@ def parse(self): conf_parser = argparse.ArgumentParser(add_help=False) kwargs = {'help': "Specify config file", 'metavar':"FILE", - 'action':'append'} - if os.path.exists(self.CONF_DEFAULT_PATH): - kwargs['default'] = [self.CONF_DEFAULT_PATH] + 'action':'append', 'default':self.CONF_DEFAULT_PATHS} conf_parser.add_argument("-c", "--conf_file", **kwargs) args, remaining_argv = conf_parser.parse_known_args(self._argv.split()) @@ -134,7 +134,7 @@ def parse(self): # print script description with -h/--help description=__doc__, # Don't mess with format of description - formatter_class=argparse.RawDescriptionHelpFormatter, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) defaults.update(ksopts) defaults.update(disc_opts) diff --git a/src/analytics/contrail-topology/contrail_topology/config.py b/src/analytics/contrail-topology/contrail_topology/config.py index 51d987d4d69..e29544304e0 100644 --- a/src/analytics/contrail-topology/contrail_topology/config.py +++ b/src/analytics/contrail-topology/contrail_topology/config.py @@ -5,14 +5,16 @@ from pysandesh.sandesh_base import * from pysandesh.gen_py.sandesh.ttypes import SandeshLevel from sandesh_common.vns.constants import ModuleNames, HttpPortTopology, \ - API_SERVER_DISCOVERY_SERVICE_NAME, OpServerAdminPort + API_SERVER_DISCOVERY_SERVICE_NAME, OpServerAdminPort, \ + ServicesDefaultConfigurationFiles, SERVICE_TOPOLOGY from sandesh_common.vns.ttypes import Module import discoveryclient.client as discovery_client import traceback from vnc_api.vnc_api import VncApi class CfgParser(object): - CONF_DEFAULT_PATH = '/etc/contrail/contrail-topology.conf' + CONF_DEFAULT_PATHS = ServicesDefaultConfigurationFiles.get( + SERVICE_TOPOLOGY, None) def __init__(self, argv): self._args = None self.__pat = None @@ -61,10 +63,8 @@ def parse(self): conf_parser = argparse.ArgumentParser(add_help=False) kwargs = {'help': "Specify config file", 'metavar':"FILE", - 'action':'append' + 'action':'append', 'default': self.CONF_DEFAULT_PATHS, } - if os.path.exists(self.CONF_DEFAULT_PATH): - kwargs['default'] = [self.CONF_DEFAULT_PATH] conf_parser.add_argument("-c", "--conf_file", **kwargs) args, remaining_argv = conf_parser.parse_known_args(self._argv.split()) @@ -124,7 +124,7 @@ def parse(self): # print script description with -h/--help description=__doc__, # Don't mess with format of description - formatter_class=argparse.RawDescriptionHelpFormatter, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) defaults.update(disc_opts) defaults.update(ksopts) diff --git a/src/analytics/options.cc b/src/analytics/options.cc index dc12ad51759..d39294bf63e 100644 --- a/src/analytics/options.cc +++ b/src/analytics/options.cc @@ -5,6 +5,7 @@ #include #include +#include #include #include "analytics/buildinfo.h" @@ -47,17 +48,29 @@ void Options::Initialize(EventManager &evm, exit(-1); } - vector conf_files; - conf_files.push_back("/etc/contrail/contrail-collector.conf"); - conf_files.push_back("/etc/contrail/contrail-keystone-auth.conf"); + map >::const_iterator it = + g_vns_constants.ServicesDefaultConfigurationFiles.find( + g_vns_constants.SERVICE_COLLECTOR); + assert(it != g_vns_constants.ServicesDefaultConfigurationFiles.end()); + const vector &conf_files(it->second); opt::options_description generic("Generic options"); // Command line only options. + ostringstream conf_files_oss; + bool first = true; + BOOST_FOREACH(const string &cfile, conf_files) { + if (first) { + conf_files_oss << cfile; + first = false; + } else { + conf_files_oss << ", " << cfile; + } + } generic.add_options() ("conf_file", opt::value >()->default_value( - conf_files, - "Configuration file")) + conf_files, conf_files_oss.str()), + "Configuration file") ("help", "help message") ("version", "Display version information") ; @@ -117,18 +130,8 @@ void Options::Initialize(EventManager &evm, "Cassandra compaction strategy for flow tables"); // Command line and config file options. - opt::options_description config("Configuration options"); - config.add_options() - ("COLLECTOR.port", opt::value()->default_value( - default_collector_port), - "Listener port of sandesh collector server") - ("COLLECTOR.server", - opt::value()->default_value("0.0.0.0"), - "IP address of sandesh collector server") - ("COLLECTOR.protobuf_port", - opt::value()->default_value( - default_collector_protobuf_port), - "Listener port of Google Protocol Buffer collector server") + opt::options_description database_config("Database Configuration options"); + database_config.add_options() ("DATABASE.disk_usage_percentage.high_watermark0", opt::value()->default_value( default_disk_usage_percentage_high_watermark0), @@ -153,7 +156,6 @@ void Options::Initialize(EventManager &evm, opt::value()->default_value( default_disk_usage_percentage_low_watermark2), "Disk usage percentage low watermark 2") - ("DATABASE.pending_compaction_tasks.high_watermark0", opt::value()->default_value( default_pending_compaction_tasks_high_watermark0), @@ -178,7 +180,6 @@ void Options::Initialize(EventManager &evm, opt::value()->default_value( default_pending_compaction_tasks_low_watermark2), "Cassandra pending compaction tasks low watermark 2") - ("DATABASE.high_watermark0.message_severity_level", opt::value()->default_value( default_high_watermark0_message_severity_level), @@ -204,23 +205,65 @@ void Options::Initialize(EventManager &evm, default_low_watermark2_message_severity_level), "Low Watermark 2 Message severity level") + ("DATABASE.cluster_id", opt::value()->default_value(""), + "Analytics Cluster Id") + ("DATABASE.disable_all_writes", + opt::bool_switch(&cassandra_options_.disable_all_db_writes_), + "Disable all writes to the database") + ("DATABASE.disable_statistics_writes", + opt::bool_switch(&cassandra_options_.disable_db_stats_writes_), + "Disable statistics writes to the database") + ("DATABASE.disable_message_writes", + opt::bool_switch(&cassandra_options_.disable_db_messages_writes_), + "Disable message writes to the database") + ("DATABASE.enable_message_keyword_writes", + opt::bool_switch(&enable_db_messages_keyword_writes_)-> + default_value(false), + "Enable message keyword writes to the database") + ; + + // Command line and config file options. + opt::options_description collector_config("Collector Configuration options"); + collector_config.add_options() + ("COLLECTOR.port", opt::value()->default_value( + default_collector_port), + "Listener port of sandesh collector server") + ("COLLECTOR.server", + opt::value()->default_value("0.0.0.0"), + "IP address of sandesh collector server") + ("COLLECTOR.protobuf_port", + opt::value()->default_value( + default_collector_protobuf_port), + "Listener port of Google Protocol Buffer collector server") ("COLLECTOR.structured_syslog_port", opt::value()->default_value( default_collector_structured_syslog_port), "Listener port of Structured Syslog collector server") + ; + // Command line and config file options. + opt::options_description config("Configuration options"); + config.add_options() ("DEFAULT.analytics_data_ttl", - opt::value()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::GLOBAL_TTL)->second), - "global TTL(hours) for analytics data") + opt::value()->default_value( + g_viz_constants.TtlValuesDefault.find( + TtlType::GLOBAL_TTL)->second), + "TTL in hours for analytics data") ("DEFAULT.analytics_config_audit_ttl", - opt::value()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::CONFIGAUDIT_TTL)->second), - "global TTL(hours) for analytics config audit data") + opt::value()->default_value( + g_viz_constants.TtlValuesDefault.find( + TtlType::CONFIGAUDIT_TTL)->second), + "TTL in hours for analytics configuration audit data") ("DEFAULT.analytics_statistics_ttl", - opt::value()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::STATSDATA_TTL)->second), - "global TTL(hours) for analytics stats data") + opt::value()->default_value( + g_viz_constants.TtlValuesDefault.find( + TtlType::STATSDATA_TTL)->second), + "TTL in hours for analytics statistics data") ("DEFAULT.analytics_flow_ttl", - opt::value()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::FLOWDATA_TTL)->second), - "global TTL(hours) for analytics flow data") + opt::value()->default_value( + g_viz_constants.TtlValuesDefault.find( + TtlType::FLOWDATA_TTL)->second), + "TTL in hours for analytics flow data") ("DEFAULT.cassandra_server_list", opt::value >()->default_value( default_cassandra_server_list, default_cassandra_server), @@ -282,28 +325,16 @@ void Options::Initialize(EventManager &evm, ("DEFAULT.disable_flow_collection", opt::bool_switch(&disable_flow_collection_), "Disable flow message collection") - ("DATABASE.cluster_id", opt::value()->default_value(""), - "Analytics Cluster Id") - ("DATABASE.disable_all_writes", - opt::bool_switch(&cassandra_options_.disable_all_db_writes_), - "Disable all writes to the database") - ("DATABASE.disable_statistics_writes", - opt::bool_switch(&cassandra_options_.disable_db_stats_writes_), - "Disable statistics writes to the database") - ("DATABASE.disable_message_writes", - opt::bool_switch(&cassandra_options_.disable_db_messages_writes_), - "Disable message writes to the database") - ("DATABASE.enable_message_keyword_writes", - opt::bool_switch(&enable_db_messages_keyword_writes_)-> - default_value(false), - "Enable message keyword writes to the database") - ("DISCOVERY.port", opt::value()->default_value( default_discovery_port), "Port of Discovery Server") ("DISCOVERY.server", opt::value()->default_value(""), "IP address of Discovery Server") + ; + // Command line and config file options. + opt::options_description redis_config("Redis Configuration options"); + redis_config.add_options() ("REDIS.port", opt::value()->default_value(default_redis_port), "Port of Redis-uve server") @@ -311,6 +342,11 @@ void Options::Initialize(EventManager &evm, "IP address of Redis Server") ("REDIS.password", opt::value()->default_value(""), "password for Redis Server") + ; + + // Command line and config file options. + opt::options_description keystone_config("Keystone Configuration options"); + keystone_config.add_options() ("KEYSTONE.auth_host", opt::value()->default_value("127.0.0.1"), "IP address of keystone Server") ("KEYSTONE.auth_port", @@ -336,7 +372,11 @@ void Options::Initialize(EventManager &evm, "/etc/contrail/ks-key"), "Keystone private key") ("KEYSTONE.cafile", opt::value()->default_value( "/etc/contrail/ks-ca"), "Keystone CA chain") + ; + // Command line and config file options. + opt::options_description sandesh_config("Sandesh Configuration options"); + sandesh_config.add_options() ("SANDESH.sandesh_keyfile", opt::value()->default_value( "/etc/contrail/ssl/private/server-privkey.pem"), "Sandesh ssl private key") @@ -354,8 +394,12 @@ void Options::Initialize(EventManager &evm, "Enable ssl for introspect connection") ; - config_file_options_.add(config).add(cassandra_config); - cmdline_options.add(generic).add(config).add(cassandra_config); + config_file_options_.add(config).add(cassandra_config) + .add(database_config).add(sandesh_config).add(keystone_config) + .add(collector_config).add(redis_config); + cmdline_options.add(generic).add(config).add(cassandra_config) + .add(database_config).add(sandesh_config).add(keystone_config) + .add(collector_config).add(redis_config); } template diff --git a/src/analytics/test/options_test.cc b/src/analytics/test/options_test.cc index a1ef8dec8fd..39ffa39b595 100644 --- a/src/analytics/test/options_test.cc +++ b/src/analytics/test/options_test.cc @@ -55,9 +55,11 @@ TEST_F(OptionsTest, NoArguments) { argv[0] = argv_0; options_.Parse(evm_, argc, argv); - vector expected_conf_files_; - expected_conf_files_.push_back("/etc/contrail/contrail-collector.conf"); - expected_conf_files_.push_back("/etc/contrail/contrail-keystone-auth.conf"); + map >::const_iterator it = + g_vns_constants.ServicesDefaultConfigurationFiles.find( + g_vns_constants.SERVICE_COLLECTOR); + assert(it != g_vns_constants.ServicesDefaultConfigurationFiles.end()); + const vector &expected_conf_files_(it->second); TASK_UTIL_EXPECT_VECTOR_EQ(default_cassandra_server_list_, options_.cassandra_server_list()); EXPECT_EQ(options_.redis_server(), "127.0.0.1"); diff --git a/src/config/utils/contrail-status.py b/src/config/utils/contrail-status.py index da240c4bf72..fd1720237b7 100755 --- a/src/config/utils/contrail-status.py +++ b/src/config/utils/contrail-status.py @@ -17,7 +17,7 @@ from StringIO import StringIO from lxml import etree from sandesh_common.vns.constants import ServiceHttpPortMap, \ - NodeUVEImplementedServices, ServicesDefaultConfigurationFile, \ + NodeUVEImplementedServices, ServicesDefaultConfigurationFiles, \ BackupImplementedServices DPDK_NETLINK_TCP_PORT = 20914 @@ -268,22 +268,13 @@ def check_svc(svc, initd_svc=False): status='inactive' print '%-30s%s%s' %(psvc, status, bootstatus) -_DEFAULT_CONF_FILE_DIR = '/etc/contrail/' -_DEFAULT_CONF_FILE_EXTENSION = '.conf' - -def get_http_server_port_from_conf(svc_name, debug): - # Open and extract conf file - if svc_name in ServicesDefaultConfigurationFile: - default_conf_file = ServicesDefaultConfigurationFile[svc_name] - else: - default_conf_file = _DEFAULT_CONF_FILE_DIR + svc_name + \ - _DEFAULT_CONF_FILE_EXTENSION +def _get_http_server_port_from_conf(svc_name, conf_file, debug): try: - fp = open(default_conf_file) + fp = open(conf_file) except IOError as e: if debug: print '{0}: Could not read filename {1}'.format(\ - svc_name, default_conf_file) + svc_name, conf_file) return -1 else: data = StringIO('\n'.join(line.strip() for line in fp)) @@ -318,6 +309,22 @@ def get_http_server_port_from_conf(svc_name, debug): fp.close() return http_server_port +_DEFAULT_CONF_FILE_DIR = '/etc/contrail/' +_DEFAULT_CONF_FILE_EXTENSION = '.conf' + +def get_http_server_port_from_conf(svc_name, debug): + # Open and extract conf file + if svc_name in ServicesDefaultConfigurationFiles: + default_conf_files = ServicesDefaultConfigurationFiles[svc_name] + else: + default_conf_files = [_DEFAULT_CONF_FILE_DIR + svc_name + \ + _DEFAULT_CONF_FILE_EXTENSION] + for conf_file in default_conf_files: + http_server_port = _get_http_server_port_from_conf(svc_name, conf_file, + debug) + if http_server_port != -1: + return http_server_port + def get_default_http_server_port(svc_name, debug): if svc_name in ServiceHttpPortMap: return ServiceHttpPortMap[svc_name] diff --git a/src/opserver/alarmgen_cfg.py b/src/opserver/alarmgen_cfg.py index 05310168afc..dcbade1e34e 100644 --- a/src/opserver/alarmgen_cfg.py +++ b/src/opserver/alarmgen_cfg.py @@ -1,6 +1,12 @@ +# +# Copyright (c) 2017 Juniper Networks, Inc. All rights reserved. +# + import argparse, os, ConfigParser, sys, re from pysandesh.sandesh_base import * from pysandesh.gen_py.sandesh.ttypes import SandeshLevel +from sandesh_common.vns.constants import SERVICE_ALARM_GENERATOR, \ + ServicesDefaultConfigurationFiles class CfgParser(object): @@ -37,7 +43,9 @@ def parse(self): # Turn off help, so we print all options in response to -h conf_parser = argparse.ArgumentParser(add_help=False) conf_parser.add_argument("-c", "--conf_file", action="append", - help="Specify config file", metavar="FILE") + help="Specify config file", metavar="FILE", + default=ServicesDefaultConfigurationFiles.get( + SERVICE_ALARM_GENERATOR, None)) args, remaining_argv = conf_parser.parse_known_args(self._argv.split()) defaults = { @@ -121,7 +129,7 @@ def parse(self): # print script description with -h/--help description=__doc__, # Don't mess with format of description - formatter_class=argparse.RawDescriptionHelpFormatter, + formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) defaults.update(redis_opts) diff --git a/src/opserver/contrail-alarm-gen.ini b/src/opserver/contrail-alarm-gen.ini index e0fb4d2f7ce..f0ddf3a818e 100644 --- a/src/opserver/contrail-alarm-gen.ini +++ b/src/opserver/contrail-alarm-gen.ini @@ -1,15 +1,13 @@ [program:contrail-alarm-gen] -command=/usr/bin/contrail-alarm-gen --conf_file /etc/contrail/contrail-alarm-gen.conf --conf_file /etc/contrail/contrail-keystone-auth.conf --http_server_port 920%(process_num)01d --worker_id %(process_num)s --log_file /var/log/contrail/contrail-alarm-gen-%(process_num)s.log -numprocs=1 -process_name=%(process_num)s +command=/usr/bin/contrail-alarm-gen priority=440 autostart=true killasgroup=true stopsignal=KILL stdout_capture_maxbytes=1MB redirect_stderr=true -stdout_logfile=/var/log/contrail/contrail-alarm-gen-%(process_num)s-stdout.log -stderr_logfile=/var/log/contrail/contrail-alarm-gen-%(process_num)s-stderr.log +stdout_logfile=/var/log/contrail/contrail-alarm-gen-stdout.log +stderr_logfile=/var/log/contrail/contrail-alarm-gen-stderr.log startsecs=5 exitcodes=0 ; 'expected' exit codes for process (default 0,2) user=contrail diff --git a/src/opserver/contrail-analytics-api.ini b/src/opserver/contrail-analytics-api.ini index bcbf50ebe30..561ec857530 100644 --- a/src/opserver/contrail-analytics-api.ini +++ b/src/opserver/contrail-analytics-api.ini @@ -1,5 +1,5 @@ [program:contrail-analytics-api] -command=/usr/bin/contrail-analytics-api -c /etc/contrail/contrail-analytics-api.conf +command=/usr/bin/contrail-analytics-api priority=440 autostart=true killasgroup=true diff --git a/src/opserver/opserver.py b/src/opserver/opserver.py index cea50193789..26ad8ad3f40 100644 --- a/src/opserver/opserver.py +++ b/src/opserver/opserver.py @@ -53,7 +53,8 @@ INSTANCE_ID_DEFAULT, COLLECTOR_DISCOVERY_SERVICE_NAME,\ ANALYTICS_API_SERVER_DISCOVERY_SERVICE_NAME, ALARM_GENERATOR_SERVICE_NAME, \ OpServerAdminPort, CLOUD_ADMIN_ROLE, APIAAAModes, \ - AAA_MODE_CLOUD_ADMIN, AAA_MODE_NO_AUTH + AAA_MODE_CLOUD_ADMIN, AAA_MODE_NO_AUTH, \ + ServicesDefaultConfigurationFiles, SERVICE_OPSERVER from sandesh.viz.constants import _TABLES, _OBJECT_TABLES,\ _OBJECT_TABLE_SCHEMA, _OBJECT_TABLE_COLUMN_VALUES, \ _STAT_TABLES, STAT_OBJECTID_FIELD, STAT_VT_PREFIX, \ @@ -815,7 +816,9 @@ def _parse_args(self, args_str=' '.join(sys.argv[1:])): conf_parser = argparse.ArgumentParser(add_help=False) conf_parser.add_argument("-c", "--conf_file", action='append', - help="Specify config file", metavar="FILE") + help="Specify config file", metavar="FILE", + default=ServicesDefaultConfigurationFiles.get( + SERVICE_OPSERVER, None)) args, remaining_argv = conf_parser.parse_known_args(args_str.split()) defaults = { diff --git a/src/opserver/systemd/contrail-alarm-gen.service b/src/opserver/systemd/contrail-alarm-gen.service index f4114d18570..37d422ba3b2 100755 --- a/src/opserver/systemd/contrail-alarm-gen.service +++ b/src/opserver/systemd/contrail-alarm-gen.service @@ -4,7 +4,7 @@ After=network.target [Service] Type=simple -ExecStart=/usr/bin/contrail-alarm-gen -c /etc/contrail/contrail-alarm-gen.conf +ExecStart=/usr/bin/contrail-alarm-gen PIDFile=/var/run/contrail/contrail-alarm-gen.pid TimeoutStopSec=0 Restart=always diff --git a/src/query_engine/options.cc b/src/query_engine/options.cc index 41fecb62ac4..bb567647bbb 100644 --- a/src/query_engine/options.cc +++ b/src/query_engine/options.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include "base/contrail_ports.h" #include "base/logging.h" @@ -57,16 +58,29 @@ void Options::Initialize(EventManager &evm, exit(-1); } - vector conf_files; - conf_files.push_back("/etc/contrail/contrail-query-engine.conf"); + map >::const_iterator it = + g_vns_constants.ServicesDefaultConfigurationFiles.find( + g_vns_constants.SERVICE_QUERY_ENGINE); + assert(it != g_vns_constants.ServicesDefaultConfigurationFiles.end()); + const vector &conf_files(it->second); opt::options_description generic("Generic options"); // Command line only options. + ostringstream conf_files_oss; + bool first = true; + BOOST_FOREACH(const string &cfile, conf_files) { + if (first) { + conf_files_oss << cfile; + first = false; + } else { + conf_files_oss << ", " << cfile; + } + } generic.add_options() ("conf_file", opt::value >()->default_value( - conf_files, - "Configuration file")) + conf_files, conf_files_oss.str()), + "Configuration file") ("help", "help message") ("version", "Display version information") ; @@ -149,7 +163,11 @@ void Options::Initialize(EventManager &evm, "Port of Discovery Server") ("DISCOVERY.server", opt::value(), "IP address of Discovery Server") + ; + // Command line and config file options. + opt::options_description redis_config("Redis Configuration options"); + redis_config.add_options() ("REDIS.port", opt::value()->default_value(default_redis_port), "Port of Redis-uve server") @@ -157,7 +175,11 @@ void Options::Initialize(EventManager &evm, "IP address of Redis Server") ("REDIS.password", opt::value()->default_value(""), "password for Redis Server") + ; + // Command line and config file options. + opt::options_description sandesh_config("Sandesh Configuration options"); + sandesh_config.add_options() ("SANDESH.sandesh_keyfile", opt::value()->default_value( "/etc/contrail/ssl/private/server-privkey.pem"), "Sandesh ssl private key") @@ -173,13 +195,19 @@ void Options::Initialize(EventManager &evm, ("SANDESH.introspect_ssl_enable", opt::bool_switch(&sandesh_config_.introspect_ssl_enable), "Enable ssl for introspect connection") + ; + // Command line and config file options. + opt::options_description database_config("Database Configuration options"); + database_config.add_options() ("DATABASE.cluster_id", opt::value()->default_value(""), "Analytics Cluster Id") ; - config_file_options_.add(config).add(cassandra_config); - cmdline_options.add(generic).add(config).add(cassandra_config); + config_file_options_.add(config).add(cassandra_config) + .add(redis_config).add(sandesh_config).add(database_config); + cmdline_options.add(generic).add(config).add(cassandra_config) + .add(redis_config).add(sandesh_config).add(database_config); } template diff --git a/src/query_engine/test/SConscript b/src/query_engine/test/SConscript index 3f4778c9ab7..c8557976757 100644 --- a/src/query_engine/test/SConscript +++ b/src/query_engine/test/SConscript @@ -53,7 +53,7 @@ query_test = env.UnitTest('query_test', options_test = env.UnitTest('options_test', ['../buildinfo.o', '../options.o', '../../analytics/viz_constants.o', 'options_test.cc'], - platform_exclude=[('Ubuntu','14.04')]) + ) env.Alias('src/query_engine:options_test', options_test) utils_test = env.UnitTest('utils_test', ['../utils.o', diff --git a/src/query_engine/test/options_test.cc b/src/query_engine/test/options_test.cc index 0b69d28a0fb..d545230d7f7 100644 --- a/src/query_engine/test/options_test.cc +++ b/src/query_engine/test/options_test.cc @@ -26,19 +26,25 @@ static uint16_t default_discovery_port = ContrailPorts::DiscoveryServerPort(); class OptionsTest : public ::testing::Test { protected: - OptionsTest() { } - - virtual void SetUp() { + OptionsTest() : + default_cassandra_server_list_(1, "127.0.0.1:9160"), + default_collector_server_list_() { + map >::const_iterator it = + g_vns_constants.ServicesDefaultConfigurationFiles.find( + g_vns_constants.SERVICE_QUERY_ENGINE); + assert(it != g_vns_constants.ServicesDefaultConfigurationFiles.end()); + default_conf_files_ = it->second; boost::system::error_code error; hostname_ = host_name(error); host_ip_ = GetHostIp(evm_.io_service(), hostname_); - default_cassandra_server_list_.push_back("127.0.0.1:9160"); - default_collector_server_list_.push_back("127.0.0.1:8086"); - default_conf_files_.push_back("/etc/contrail/contrail-query-engine.conf"); + } + + virtual void SetUp() { } virtual void TearDown() { remove("./options_test_query_engine.conf"); + remove("./options_test_cassandra_config_file.conf"); } EventManager evm_; @@ -58,16 +64,13 @@ TEST_F(OptionsTest, NoArguments) { options_.Parse(evm_, argc, argv); - vector expected_conf_files_; - expected_conf_files_.push_back("/etc/contrail/contrail-query-engine.conf"); - TASK_UTIL_EXPECT_VECTOR_EQ(default_cassandra_server_list_, options_.cassandra_server_list()); TASK_UTIL_EXPECT_VECTOR_EQ(default_collector_server_list_, options_.collector_server_list()); EXPECT_EQ(options_.redis_server(), "127.0.0.1"); EXPECT_EQ(options_.redis_port(), default_redis_port); - TASK_UTIL_EXPECT_VECTOR_EQ(expected_conf_files_, + TASK_UTIL_EXPECT_VECTOR_EQ(default_conf_files_, options_.config_file()); EXPECT_EQ(options_.discovery_server(), ""); EXPECT_EQ(options_.discovery_port(), default_discovery_port); @@ -86,7 +89,8 @@ TEST_F(OptionsTest, NoArguments) { EXPECT_EQ(options_.max_tasks(), 0); EXPECT_EQ(options_.max_slice(), 100); EXPECT_EQ(options_.test_mode(), false); - EXPECT_EQ(options_.sandesh_send_rate_limit(), 0); + EXPECT_EQ(options_.sandesh_send_rate_limit(), + g_sandesh_constants.DEFAULT_SANDESH_SEND_RATELIMIT); } TEST_F(OptionsTest, DefaultConfFile) { @@ -127,7 +131,8 @@ TEST_F(OptionsTest, DefaultConfFile) { EXPECT_EQ(options_.max_tasks(), 0); EXPECT_EQ(options_.max_slice(), 100); EXPECT_EQ(options_.test_mode(), false); - EXPECT_EQ(options_.sandesh_send_rate_limit(), 100); + EXPECT_EQ(options_.sandesh_send_rate_limit(), + g_sandesh_constants.DEFAULT_SANDESH_SEND_RATELIMIT); } TEST_F(OptionsTest, OverrideStringFromCommandLine) { @@ -367,7 +372,7 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { config_file << cassandra_config; config_file.close(); - int argc = 19; + int argc = 18; char *argv[argc]; char argv_0[] = "options_test"; char argv_1[] = "--conf_file=./options_test_query_engine.conf"; @@ -400,11 +405,11 @@ TEST_F(OptionsTest, CustomConfigFileAndOverrideFromCommandLine) { argv[10] = argv_10; argv[11] = argv_11; argv[12] = argv_12; - argv[14] = argv_13; - argv[15] = argv_14; - argv[16] = argv_15; - argv[17] = argv_16; - argv[18] = argv_17; + argv[13] = argv_13; + argv[14] = argv_14; + argv[15] = argv_15; + argv[16] = argv_16; + argv[17] = argv_17; options_.Parse(evm_, argc, argv); @@ -457,7 +462,7 @@ TEST_F(OptionsTest, MultitokenVector) { char argv_0[] = "options_test"; char argv_1[] = "--DEFAULT.collectors=10.10.10.1:100 20.20.20.2:200"; char argv_2[] = "--DEFAULT.cassandra_server_list=30.30.30.3:300"; - char argv_3[] = "--conf_file=controller/src/analytics/contrail-collector.conf" + char argv_3[] = "--conf_file=controller/src/query_engine/contrail-query-engine.conf" " controller/src/analytics/contrail-database.conf"; char argv_4[] = "--conf_file=controller/src/analytics/test-conf.conf"; argv[0] = argv_0; @@ -474,7 +479,7 @@ TEST_F(OptionsTest, MultitokenVector) { vector cassandra_server_list; cassandra_server_list.push_back("30.30.30.3:300"); vector option_file_list; - option_file_list.push_back("controller/src/analytics/contrail-collector.conf"); + option_file_list.push_back("controller/src/query_engine/contrail-query-engine.conf"); option_file_list.push_back("controller/src/analytics/contrail-database.conf"); option_file_list.push_back("controller/src/analytics/test-conf.conf"); TASK_UTIL_EXPECT_VECTOR_EQ(options_.config_file(), diff --git a/src/sandesh/common/vns.sandesh b/src/sandesh/common/vns.sandesh index 83e1dfd8a56..6623b1856fe 100644 --- a/src/sandesh/common/vns.sandesh +++ b/src/sandesh/common/vns.sandesh @@ -179,26 +179,38 @@ const list BackupImplementedServices = [ SERVICE_DEVICE_MANAGER, ] -const map ServicesDefaultConfigurationFile = { - SERVICE_VROUTER_AGENT : '/etc/contrail/contrail-vrouter-agent.conf', - SERVICE_CONTROL_NODE : '/etc/contrail/contrail-control.conf', - SERVICE_COLLECTOR : '/etc/contrail/contrail-collector.conf', - SERVICE_QUERY_ENGINE : '/etc/contrail/contrail-query-engine.conf', - SERVICE_OPSERVER : '/etc/contrail/contrail-analytics-api.conf', - SERVICE_ALARM_GENERATOR : '/etc/contrail/contrail-alarm-gen.conf', - SERVICE_ALARM_GENERATOR_0 : '/etc/contrail/contrail-alarm-gen.conf', - SERVICE_SNMP_COLLECTOR : '/etc/contrail/contrail-snmp-collector.conf', - SERVICE_TOPOLOGY : '/etc/contrail/contrail-topology.conf', - SERVICE_API_SERVER : '/etc/contrail/contrail-api.conf', - SERVICE_API_SERVER_0 : '/etc/contrail/contrail-api.conf', - SERVICE_DISCOVERY : '/etc/contrail/contrail-discovery.conf', - SERVICE_DISCOVERY_0 : '/etc/contrail/contrail-discovery.conf', - SERVICE_SCHEMA_TRANSFORMER : '/etc/contrail/contrail-schema.conf', - SERVICE_SVC_MONITOR : '/etc/contrail/contrail-svc-monitor.conf', - SERVICE_DEVICE_MANAGER : '/etc/contrail/contrail-device-manager.conf', - SERVICE_DNSD : '/etc/contrail/dns.conf', - SERVICE_KUBE_MANAGER : '/etc/contrail/contrail-kubernetes.conf', - SERVICE_MESOS_MANAGER : '/etc/contrail/contrail-mesos.conf', +const string ContrailKeystoneAuthConfigurationFile = '/etc/contrail/contrail-keystone-auth.conf' +const string ContrailDatabaseConfigurationFile = '/etc/contrail/contrail-database.conf' + +const map > ServicesDefaultConfigurationFiles = { + SERVICE_VROUTER_AGENT : ['/etc/contrail/contrail-vrouter-agent.conf'], + SERVICE_CONTROL_NODE : ['/etc/contrail/contrail-control.conf'], + SERVICE_COLLECTOR : ['/etc/contrail/contrail-collector.conf', + ContrailKeystoneAuthConfigurationFile, + ContrailDatabaseConfigurationFile] + SERVICE_QUERY_ENGINE : ['/etc/contrail/contrail-query-engine.conf', + ContrailDatabaseConfigurationFile], + SERVICE_OPSERVER : ['/etc/contrail/contrail-analytics-api.conf', + ContrailKeystoneAuthConfigurationFile, + ContrailDatabaseConfigurationFile] + SERVICE_ALARM_GENERATOR : ['/etc/contrail/contrail-alarm-gen.conf', + ContrailKeystoneAuthConfigurationFile] + SERVICE_ALARM_GENERATOR_0 : ['/etc/contrail/contrail-alarm-gen.conf', + ContrailKeystoneAuthConfigurationFile] + SERVICE_SNMP_COLLECTOR : ['/etc/contrail/contrail-snmp-collector.conf', + ContrailKeystoneAuthConfigurationFile] + SERVICE_TOPOLOGY : ['/etc/contrail/contrail-topology.conf', + ContrailKeystoneAuthConfigurationFile], + SERVICE_API_SERVER : ['/etc/contrail/contrail-api.conf'], + SERVICE_API_SERVER_0 : ['/etc/contrail/contrail-api.conf'], + SERVICE_DISCOVERY : ['/etc/contrail/contrail-discovery.conf'], + SERVICE_DISCOVERY_0 : ['/etc/contrail/contrail-discovery.conf'], + SERVICE_SCHEMA_TRANSFORMER : ['/etc/contrail/contrail-schema.conf'], + SERVICE_SVC_MONITOR : ['/etc/contrail/contrail-svc-monitor.conf'], + SERVICE_DEVICE_MANAGER : ['/etc/contrail/contrail-device-manager.conf'], + SERVICE_DNSD : ['/etc/contrail/dns.conf'], + SERVICE_KUBE_MANAGER : ['/etc/contrail/contrail-kubernetes.conf'], + SERVICE_MESOS_MANAGER : ['/etc/contrail/contrail-mesos.conf'], } enum Module {