Fix issues related to default configuration file for analytics daemons

1. Currently the analytics daemons do not pick up their default configuration
   file and so the supervisord ini file or the systemd service file have to
   launch them with the --config_file option on command line. This causes
   issue when either new init system is used or new provisioning method is
   used or when upgrading and the daemon needs an additional default
   configuration file
2. Change the analytics daemons to have default configuration files so that
   they can be launched without specifying --config_file option in the
   supervisord ini file or the systemd service file
3. Cleanup to display the default values when --help option is used
4. Cleanup of contrail-collector and contrail-query-engine --help display to
   club options based on sections

Change-Id: Ibd054e4ec753f7ee292f65cffd472e88b1fb62fd
Partial-Bug: #1667082

src/analytics/contrail-snmp-collector/contrail-snmp-collector.ini

@@ -1,5 +1,5 @@
 [program:contrail-snmp-collector]
-command=/usr/bin/contrail-snmp-collector --device-config-file /etc/contrail/device.ini
+command=/usr/bin/contrail-snmp-collector
 priority=340
 autostart=true
 killasgroup=true

src/analytics/contrail-snmp-collector/contrail_snmp_collector/snmpcfg.py

@@ -8,10 +8,12 @@
 import discoveryclient.client as client
 from sandesh_common.vns.ttypes import Module
 from sandesh_common.vns.constants import ModuleNames, \
-         API_SERVER_DISCOVERY_SERVICE_NAME, HttpPortSnmpCollector
+    API_SERVER_DISCOVERY_SERVICE_NAME, HttpPortSnmpCollector, \
+    ServicesDefaultConfigurationFiles, SERVICE_SNMP_COLLECTOR
 
 class CfgParser(object):
-    CONF_DEFAULT_PATH = '/etc/contrail/contrail-snmp-collector.conf'
+    CONF_DEFAULT_PATHS = ServicesDefaultConfigurationFiles.get(
+        SERVICE_SNMP_COLLECTOR, None)
     def __init__(self, argv):
         self._devices = []
         self._args = None
@@ -72,9 +74,7 @@ def parse(self):
         conf_parser = argparse.ArgumentParser(add_help=False)
 
         kwargs = {'help': "Specify config file", 'metavar':"FILE",
-                'action':'append'}
-        if os.path.exists(self.CONF_DEFAULT_PATH):
-            kwargs['default'] = [self.CONF_DEFAULT_PATH]
+                'action':'append', 'default':self.CONF_DEFAULT_PATHS}
         conf_parser.add_argument("-c", "--conf_file", **kwargs)
         args, remaining_argv = conf_parser.parse_known_args(self._argv.split())
 
@@ -134,7 +134,7 @@ def parse(self):
             # print script description with -h/--help
             description=__doc__,
             # Don't mess with format of description
-            formatter_class=argparse.RawDescriptionHelpFormatter,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         )
         defaults.update(ksopts)
         defaults.update(disc_opts)

src/analytics/contrail-topology/contrail_topology/config.py

@@ -5,14 +5,16 @@
 from pysandesh.sandesh_base import *
 from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
 from sandesh_common.vns.constants import ModuleNames, HttpPortTopology, \
-    API_SERVER_DISCOVERY_SERVICE_NAME, OpServerAdminPort
+    API_SERVER_DISCOVERY_SERVICE_NAME, OpServerAdminPort, \
+    ServicesDefaultConfigurationFiles, SERVICE_TOPOLOGY
 from sandesh_common.vns.ttypes import Module
 import discoveryclient.client as discovery_client
 import traceback
 from vnc_api.vnc_api import VncApi
 
 class CfgParser(object):
-    CONF_DEFAULT_PATH = '/etc/contrail/contrail-topology.conf'
+    CONF_DEFAULT_PATHS = ServicesDefaultConfigurationFiles.get(
+        SERVICE_TOPOLOGY, None)
     def __init__(self, argv):
         self._args = None
         self.__pat = None
@@ -61,10 +63,8 @@ def parse(self):
         conf_parser = argparse.ArgumentParser(add_help=False)
 
         kwargs = {'help': "Specify config file", 'metavar':"FILE",
-                  'action':'append'
+                  'action':'append', 'default': self.CONF_DEFAULT_PATHS,
                  }
-        if os.path.exists(self.CONF_DEFAULT_PATH):
-            kwargs['default'] = [self.CONF_DEFAULT_PATH]
         conf_parser.add_argument("-c", "--conf_file", **kwargs)
         args, remaining_argv = conf_parser.parse_known_args(self._argv.split())
 
@@ -124,7 +124,7 @@ def parse(self):
             # print script description with -h/--help
             description=__doc__,
             # Don't mess with format of description
-            formatter_class=argparse.RawDescriptionHelpFormatter,
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         )
         defaults.update(disc_opts)
         defaults.update(ksopts)

src/analytics/options.cc

@@ -5,6 +5,7 @@
 
 #include <fstream>
 #include <iostream>
+#include <boost/foreach.hpp>
 #include <boost/asio/ip/host_name.hpp>
 
 #include "analytics/buildinfo.h"
@@ -47,17 +48,29 @@ void Options::Initialize(EventManager &evm,
         exit(-1);
     }
 
-    vector<string> conf_files;
-    conf_files.push_back("/etc/contrail/contrail-collector.conf");
-    conf_files.push_back("/etc/contrail/contrail-keystone-auth.conf");
+    map<string, vector<string> >::const_iterator it =
+        g_vns_constants.ServicesDefaultConfigurationFiles.find(
+            g_vns_constants.SERVICE_COLLECTOR);
+    assert(it != g_vns_constants.ServicesDefaultConfigurationFiles.end());
+    const vector<string> &conf_files(it->second);
 
     opt::options_description generic("Generic options");
 
     // Command line only options.
+    ostringstream conf_files_oss;
+    bool first = true;
+    BOOST_FOREACH(const string &cfile, conf_files) {
+        if (first) {
+            conf_files_oss << cfile;
+            first = false;
+        } else {
+            conf_files_oss << ", " << cfile;
+        }
+    }
     generic.add_options()
         ("conf_file", opt::value<vector<string> >()->default_value(
-                                               conf_files,
-             "Configuration file"))
+             conf_files, conf_files_oss.str()),
+             "Configuration file")
          ("help", "help message")
         ("version", "Display version information")
     ;
@@ -117,18 +130,8 @@ void Options::Initialize(EventManager &evm,
             "Cassandra compaction strategy for flow tables");
 
     // Command line and config file options.
-    opt::options_description config("Configuration options");
-    config.add_options()
-        ("COLLECTOR.port", opt::value<uint16_t>()->default_value(
-                                                default_collector_port),
-             "Listener port of sandesh collector server")
-        ("COLLECTOR.server",
-             opt::value<string>()->default_value("0.0.0.0"),
-             "IP address of sandesh collector server")
-        ("COLLECTOR.protobuf_port",
-            opt::value<uint16_t>()->default_value(
-                default_collector_protobuf_port),
-         "Listener port of Google Protocol Buffer collector server")
+    opt::options_description database_config("Database Configuration options");
+    database_config.add_options()
         ("DATABASE.disk_usage_percentage.high_watermark0",
             opt::value<uint32_t>()->default_value(
                 default_disk_usage_percentage_high_watermark0),
@@ -153,7 +156,6 @@ void Options::Initialize(EventManager &evm,
             opt::value<uint32_t>()->default_value(
                 default_disk_usage_percentage_low_watermark2),
             "Disk usage percentage low watermark 2")
-
         ("DATABASE.pending_compaction_tasks.high_watermark0",
             opt::value<uint32_t>()->default_value(
                 default_pending_compaction_tasks_high_watermark0),
@@ -178,7 +180,6 @@ void Options::Initialize(EventManager &evm,
             opt::value<uint32_t>()->default_value(
                 default_pending_compaction_tasks_low_watermark2),
             "Cassandra pending compaction tasks low watermark 2")
-
         ("DATABASE.high_watermark0.message_severity_level",
             opt::value<string>()->default_value(
                 default_high_watermark0_message_severity_level),
@@ -204,23 +205,65 @@ void Options::Initialize(EventManager &evm,
                 default_low_watermark2_message_severity_level),
             "Low Watermark 2 Message severity level")
 
+        ("DATABASE.cluster_id", opt::value<string>()->default_value(""),
+             "Analytics Cluster Id")
+        ("DATABASE.disable_all_writes",
+            opt::bool_switch(&cassandra_options_.disable_all_db_writes_),
+            "Disable all writes to the database")
+        ("DATABASE.disable_statistics_writes",
+            opt::bool_switch(&cassandra_options_.disable_db_stats_writes_),
+            "Disable statistics writes to the database")
+        ("DATABASE.disable_message_writes",
+            opt::bool_switch(&cassandra_options_.disable_db_messages_writes_),
+            "Disable message writes to the database")
+        ("DATABASE.enable_message_keyword_writes",
+            opt::bool_switch(&enable_db_messages_keyword_writes_)->
+                default_value(false),
+            "Enable message keyword writes to the database")
+        ;
+
+    // Command line and config file options.
+    opt::options_description collector_config("Collector Configuration options");
+    collector_config.add_options()
+        ("COLLECTOR.port", opt::value<uint16_t>()->default_value(
+                                                default_collector_port),
+             "Listener port of sandesh collector server")
+        ("COLLECTOR.server",
+             opt::value<string>()->default_value("0.0.0.0"),
+             "IP address of sandesh collector server")
+        ("COLLECTOR.protobuf_port",
+            opt::value<uint16_t>()->default_value(
+                default_collector_protobuf_port),
+         "Listener port of Google Protocol Buffer collector server")
         ("COLLECTOR.structured_syslog_port",
             opt::value<uint16_t>()->default_value(
                 default_collector_structured_syslog_port),
          "Listener port of Structured Syslog collector server")
+        ;
 
+    // Command line and config file options.
+    opt::options_description config("Configuration options");
+    config.add_options()
         ("DEFAULT.analytics_data_ttl",
-             opt::value<uint64_t>()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::GLOBAL_TTL)->second),
-             "global TTL(hours) for analytics data")
+             opt::value<uint64_t>()->default_value(
+                 g_viz_constants.TtlValuesDefault.find(
+                 TtlType::GLOBAL_TTL)->second),
+             "TTL in hours for analytics data")
         ("DEFAULT.analytics_config_audit_ttl",
-             opt::value<uint64_t>()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::CONFIGAUDIT_TTL)->second),
-             "global TTL(hours) for analytics config audit data")
+             opt::value<uint64_t>()->default_value(
+                 g_viz_constants.TtlValuesDefault.find(
+                 TtlType::CONFIGAUDIT_TTL)->second),
+             "TTL in hours for analytics configuration audit data")
         ("DEFAULT.analytics_statistics_ttl",
-             opt::value<uint64_t>()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::STATSDATA_TTL)->second),
-             "global TTL(hours) for analytics stats data")
+             opt::value<uint64_t>()->default_value(
+                 g_viz_constants.TtlValuesDefault.find(
+                 TtlType::STATSDATA_TTL)->second),
+             "TTL in hours for analytics statistics data")
         ("DEFAULT.analytics_flow_ttl",
-             opt::value<uint64_t>()->default_value(g_viz_constants.TtlValuesDefault.find(TtlType::FLOWDATA_TTL)->second),
-             "global TTL(hours) for analytics flow data")
+             opt::value<uint64_t>()->default_value(
+                 g_viz_constants.TtlValuesDefault.find(
+                 TtlType::FLOWDATA_TTL)->second),
+             "TTL in hours for analytics flow data")
         ("DEFAULT.cassandra_server_list",
            opt::value<vector<string> >()->default_value(
                default_cassandra_server_list, default_cassandra_server),
@@ -282,35 +325,28 @@ void Options::Initialize(EventManager &evm,
         ("DEFAULT.disable_flow_collection",
             opt::bool_switch(&disable_flow_collection_),
             "Disable flow message collection")
-        ("DATABASE.cluster_id", opt::value<string>()->default_value(""),
-             "Analytics Cluster Id")
-        ("DATABASE.disable_all_writes",
-            opt::bool_switch(&cassandra_options_.disable_all_db_writes_),
-            "Disable all writes to the database")
-        ("DATABASE.disable_statistics_writes",
-            opt::bool_switch(&cassandra_options_.disable_db_stats_writes_),
-            "Disable statistics writes to the database")
-        ("DATABASE.disable_message_writes",
-            opt::bool_switch(&cassandra_options_.disable_db_messages_writes_),
-            "Disable message writes to the database")
-        ("DATABASE.enable_message_keyword_writes",
-            opt::bool_switch(&enable_db_messages_keyword_writes_)->
-                default_value(false),
-            "Enable message keyword writes to the database")
-
         ("DISCOVERY.port", opt::value<uint16_t>()->default_value(
                                                        default_discovery_port),
              "Port of Discovery Server")
         ("DISCOVERY.server", opt::value<string>()->default_value(""),
              "IP address of Discovery Server")
+        ;
 
+    // Command line and config file options.
+    opt::options_description redis_config("Redis Configuration options");
+    redis_config.add_options()
         ("REDIS.port",
              opt::value<uint16_t>()->default_value(default_redis_port),
              "Port of Redis-uve server")
         ("REDIS.server", opt::value<string>()->default_value("127.0.0.1"),
              "IP address of Redis Server")
         ("REDIS.password", opt::value<string>()->default_value(""),
              "password for Redis Server")
+        ;
+
+    // Command line and config file options.
+    opt::options_description keystone_config("Keystone Configuration options");
+    keystone_config.add_options()
         ("KEYSTONE.auth_host", opt::value<string>()->default_value("127.0.0.1"),
              "IP address of keystone Server")
         ("KEYSTONE.auth_port",
@@ -336,7 +372,11 @@ void Options::Initialize(EventManager &evm,
                     "/etc/contrail/ks-key"), "Keystone private key")
         ("KEYSTONE.cafile", opt::value<string>()->default_value(
                     "/etc/contrail/ks-ca"), "Keystone CA chain")
+        ;
 
+    // Command line and config file options.
+    opt::options_description sandesh_config("Sandesh Configuration options");
+    sandesh_config.add_options()
         ("SANDESH.sandesh_keyfile", opt::value<string>()->default_value(
             "/etc/contrail/ssl/private/server-privkey.pem"),
             "Sandesh ssl private key")
@@ -354,8 +394,12 @@ void Options::Initialize(EventManager &evm,
              "Enable ssl for introspect connection")
         ;
 
-    config_file_options_.add(config).add(cassandra_config);
-    cmdline_options.add(generic).add(config).add(cassandra_config);
+    config_file_options_.add(config).add(cassandra_config)
+        .add(database_config).add(sandesh_config).add(keystone_config)
+        .add(collector_config).add(redis_config);
+    cmdline_options.add(generic).add(config).add(cassandra_config)
+        .add(database_config).add(sandesh_config).add(keystone_config)
+        .add(collector_config).add(redis_config);
 }
 
 template <typename ValueType>

src/analytics/test/options_test.cc

@@ -55,9 +55,11 @@ TEST_F(OptionsTest, NoArguments) {
     argv[0] = argv_0;
 
     options_.Parse(evm_, argc, argv);
-    vector<string> expected_conf_files_;
-    expected_conf_files_.push_back("/etc/contrail/contrail-collector.conf");
-    expected_conf_files_.push_back("/etc/contrail/contrail-keystone-auth.conf");
+    map<string, vector<string> >::const_iterator it =
+        g_vns_constants.ServicesDefaultConfigurationFiles.find(
+            g_vns_constants.SERVICE_COLLECTOR);
+    assert(it != g_vns_constants.ServicesDefaultConfigurationFiles.end());
+    const vector<string> &expected_conf_files_(it->second);
     TASK_UTIL_EXPECT_VECTOR_EQ(default_cassandra_server_list_,
                      options_.cassandra_server_list());
     EXPECT_EQ(options_.redis_server(), "127.0.0.1");

src/config/utils/contrail-status.py

@@ -17,7 +17,7 @@
 from StringIO import StringIO
 from lxml import etree
 from sandesh_common.vns.constants import ServiceHttpPortMap, \
-    NodeUVEImplementedServices, ServicesDefaultConfigurationFile, \
+    NodeUVEImplementedServices, ServicesDefaultConfigurationFiles, \
     BackupImplementedServices
 
 DPDK_NETLINK_TCP_PORT = 20914
@@ -268,22 +268,13 @@ def check_svc(svc, initd_svc=False):
         status='inactive'
     print '%-30s%s%s' %(psvc, status, bootstatus)
 
-_DEFAULT_CONF_FILE_DIR = '/etc/contrail/'
-_DEFAULT_CONF_FILE_EXTENSION = '.conf'
-
-def get_http_server_port_from_conf(svc_name, debug):
-    # Open and extract conf file
-    if svc_name in ServicesDefaultConfigurationFile:
-        default_conf_file = ServicesDefaultConfigurationFile[svc_name]
-    else:
-        default_conf_file = _DEFAULT_CONF_FILE_DIR + svc_name + \
-            _DEFAULT_CONF_FILE_EXTENSION
+def _get_http_server_port_from_conf(svc_name, conf_file, debug):
     try:
-        fp = open(default_conf_file)
+        fp = open(conf_file)
     except IOError as e:
         if debug:
             print '{0}: Could not read filename {1}'.format(\
-                svc_name, default_conf_file)
+                svc_name, conf_file)
         return -1
     else:
         data = StringIO('\n'.join(line.strip() for line in fp))
@@ -318,6 +309,22 @@ def get_http_server_port_from_conf(svc_name, debug):
         fp.close()
         return http_server_port
 
+_DEFAULT_CONF_FILE_DIR = '/etc/contrail/'
+_DEFAULT_CONF_FILE_EXTENSION = '.conf'
+
+def get_http_server_port_from_conf(svc_name, debug):
+    # Open and extract conf file
+    if svc_name in ServicesDefaultConfigurationFiles:
+        default_conf_files = ServicesDefaultConfigurationFiles[svc_name]
+    else:
+        default_conf_files = [_DEFAULT_CONF_FILE_DIR + svc_name + \
+            _DEFAULT_CONF_FILE_EXTENSION]
+    for conf_file in default_conf_files:
+        http_server_port = _get_http_server_port_from_conf(svc_name, conf_file,
+                                                           debug)
+        if http_server_port != -1:
+            return http_server_port
+
 def get_default_http_server_port(svc_name, debug):
     if svc_name in ServiceHttpPortMap:
         return ServiceHttpPortMap[svc_name]