Split config into 2 files, added option to load extra files

As a first step towards a lower-overhead config management system
(requested in #242), I changed the Conf initialization routine to expect
two files: One for the "group" config that must be the same on all
nodes, and one for a "node" config that contains only the options that
are unique per node. The "group" config is loaded first, then the "node"
config, so any options that appear in both files will take the values
specified in the "node" config file.

I also created a new top-level function, loadExtraFile, that allows a
client to repeat the config-file-loading process for an additional file
besides the two required ones. This should enable Cascade to use a
separate file for its options (i.e. "cascade.cfg") and ask Conf to load
and parse this file at startup.

include/derecho/conf/conf.hpp

@@ -7,7 +7,9 @@
 #include <inttypes.h>
 #include <map>
 #include <memory>
+#include <optional>
 #include <stdio.h>
+#include <string>
 #include <unistd.h>
 
 namespace derecho {
@@ -135,43 +137,37 @@ class Conf {
      * The name of the default configuration file that will be loaded if none is specified
      */
     static constexpr const char* default_conf_file = "derecho.cfg";
+    /**
+     * The name of the default node-local configuration file that will be loaded
+     * if none is specified
+     */
+    static constexpr const char* default_node_conf_file = "derecho_node.cfg";
 
-    /** Constructor:
-   *  Conf can read configure from multiple sources
-   *  - the command line argument has the highest priority, then,
-   *  - the configuration files
-   *  - the default values.
-   **/
-    Conf(int argc, char* argv[], getpot::GetPot* getpotcfg = nullptr) noexcept(true) {
-        // 1 - load configuration from configuration file
-        if(getpotcfg != nullptr) {
-            for(const std::string& key : getpotcfg->get_variable_names()) {
-                this->config[key] = (*getpotcfg)(key, "");
-            }
-        }
-        // 2 - load configuration from the command line
-        int c;
-        while(1) {
-            int option_index = 0;
-
-            c = getopt_long(argc, argv, "", long_options, &option_index);
-            if(c == -1) {
-                break;
-            }
-
-            switch(c) {
-                case 0:
-                    this->config[long_options[option_index].name] = optarg;
-                    break;
+    /**
+     * Constructor:
+     * Conf can read configuration from multiple sources
+     *  - the command line argument has the highest priority, then,
+     *  - the "group" configuration file
+     *  - the "node" (local) configuration file
+     *  - the default values.
+     **/
+    Conf(int argc, char* argv[], std::optional<std::string> group_conf_file,
+         std::optional<std::string> node_conf_file) noexcept(true);
 
-                case '?':
-                    break;
+    /**
+     * Loads configuration options from a file and adds them to the
+     * configuration table. If the file contains options that were already
+     * present in the in-memory configuration table, they will overwrite the
+     * values in memory. This method can be called after initialization, but
+     * it is not thread-safe, so the caller must ensure it is only called once.
+     * Also, this method does not check if the file exists before attempting to
+     * parse it with GetPot, so it may fail with an exception from GetPot if
+     * called with a nonexistant file.
+     *
+     * @param file_name The name/path of the file to read
+     */
+    void loadFromFile(const std::string& file_name);
 
-                default:
-                    std::cerr << "ignore unknown commandline code:" << c << std::endl;
-            }
-        }
-    }
     /** get configuration **/
     const std::string& getString(const std::string& key) const {
         return this->config.at(key);
@@ -208,18 +204,45 @@ class Conf {
     const bool hasCustomizedKey(const std::string& key) const {
         return (this->config.find(key) != this->config.end());
     }
-    // Initialize the singleton from the command line and the configuration file.
-    // The command line has higher priority than the configuration file
-    // The process we find the configuration file:
-    // 1) if conf_file is not null, use it, otherwise,
-    // 2) try DERECHO_CONF_FILE environment, otherwise,
-    // 3) use "derecho.cfg" at local, otherwise,
-    // 4) use all default settings.
-    // Note: initialize will be called on the first 'get()' if it is not called
-    // before.
+    /**
+     * Initialize the singleton from the command line and the configuration files.
+     * The command line has higher priority than the configuration files, and the
+     * node-local configuration file has higher priority than the group
+     * configuration file. The process for finding the group configuration file
+     * and node-local configuration file is:
+     * 1) if conf_file/node_conf_file is not null, use it, otherwise,
+     * 2) try DERECHO_CONF_FILE environment variable, otherwise,
+     * 3) use default_conf_file if it exists, otherwise,
+     * 4) use all default settings.
+     * Note: initialize will be called on the first 'get()' if it is not called
+     * before.
+     */
     static void initialize(int argc, char* argv[],
-                           const char* conf_file = nullptr);
+                           const char* conf_file = nullptr,
+                           const char* node_conf_file = nullptr);
+    /** Gets a const pointer to the singleton instance, initializing it if necessary. */
     static const Conf* get() noexcept(true);
+    /**
+     * Loads an additional configuration file into the Conf object, besides the
+     * two standard Derecho configuration files. Options set in the new config
+     * file have higher priority than (will overwrite) any options already set
+     * in the first initialization. If the optional env_var_name parameter is
+     * provided (non-null), it will be used as the name of an environment
+     * variable to check for the name of the file, and this filename will be
+     * used in preference to the file_name parameter if it exists. This function
+     * should be called after initialization, but unlike initialize() it is not
+     * thread-safe, so the caller must ensure it is only called once.
+     *
+     * @param default_file_name The "default" name of the configuration file to
+     * load. This is the file that will be loaded if env_var_name is not
+     * provided, is not set, or is set but points to a nonexistant file.
+     * @param env_var_name The name of an environment variable to check for a
+     * non-default file name
+     * @throw A std::runtime_error if the file could not be found after checking
+     * both the environment variable and the default file name.
+     */
+    static void loadExtraFile(const std::string& default_file_name,
+                              const char* env_var_name = nullptr);
 
     // Defines fields used for loading subgroup profiles in multicast_group.h
     static const std::vector<std::string> subgroupProfileFields;

src/conf/conf.cpp

@@ -82,13 +82,13 @@ struct option Conf::long_options[] = {
         MAKE_LONG_OPT_ENTRY(LOGGER_PERSISTENCE_LOG_LEVEL),
         {0, 0, 0, 0}};
 
-void Conf::initialize(int argc, char* argv[], const char* conf_file) {
+void Conf::initialize(int argc, char* argv[], const char* conf_file, const char* node_conf_file) {
     uint32_t expected = CONF_UNINITIALIZED;
     // if not initialized(0), set the flag to under initialization ...
     if(Conf::singleton_initialized_flag.compare_exchange_strong(
                expected, CONF_INITIALIZING, std::memory_order_acq_rel)) {
-        // 1 - get configuration file path
-        std::string real_conf_file;
+        // 1 - determine configuration file path, if possible, or leave real_conf_file "empty"
+        std::optional<std::string> real_conf_file;
         struct stat buffer;
         if(conf_file)
             real_conf_file = conf_file;
@@ -99,16 +99,21 @@ void Conf::initialize(int argc, char* argv[], const char* conf_file) {
             if(S_ISREG(buffer.st_mode) && (S_IRUSR | buffer.st_mode)) {
                 real_conf_file = default_conf_file;
             }
-        } else
-            real_conf_file.clear();
+        }
+        // 1.5 - same path detection but for node_conf_file
+        std::optional<std::string> real_node_conf_file;
+        if(node_conf_file) {
+            real_node_conf_file = node_conf_file;
+        } else if(std::getenv("DERECHO_NODE_CONF_FILE")) {
+            real_node_conf_file = std::getenv("DERECHO_NODE_CONF_FILE");
+        } else if(stat(default_node_conf_file, &buffer) == 0) {
+            if(S_ISREG(buffer.st_mode) && (S_IRUSR | buffer.st_mode)) {
+                real_node_conf_file = default_node_conf_file;
+            }
+        }
 
         // 2 - load configuration
-        getpot::GetPot* cfg = nullptr;
-        if(!real_conf_file.empty()) {
-            cfg = new getpot::GetPot(real_conf_file);
-        }
-        Conf::singleton = std::make_unique<Conf>(argc, argv, cfg);
-        delete cfg;
+        Conf::singleton = std::make_unique<Conf>(argc, argv, real_conf_file, real_node_conf_file);
 
         // 3 - set optional log-level keys to equal the default log level if they are not present
         const std::string& default_log_level = Conf::singleton->getString(LOGGER_DEFAULT_LOG_LEVEL);
@@ -160,6 +165,46 @@ void Conf::initialize(int argc, char* argv[], const char* conf_file) {
     }
 }
 
+void Conf::loadFromFile(const std::string& file_name) {
+    getpot::GetPot file_parser(file_name);
+    for(const std::string& key : file_parser.get_variable_names()) {
+        this->config[key] = file_parser(key, "");
+    }
+}
+
+Conf::Conf(int argc, char* argv[], std::optional<std::string> group_conf_file,
+         std::optional<std::string> node_conf_file) noexcept {
+    // 1 - load configuration from configuration file
+    if(group_conf_file) {
+        loadFromFile(group_conf_file.value());
+    }
+    if(node_conf_file) {
+        loadFromFile(node_conf_file.value());
+    }
+    // 2 - load configuration from the command line
+    int c;
+    while(1) {
+        int option_index = 0;
+
+        c = getopt_long(argc, argv, "", long_options, &option_index);
+        if(c == -1) {
+            break;
+        }
+
+        switch(c) {
+            case 0:
+                this->config[long_options[option_index].name] = optarg;
+                break;
+
+            case '?':
+                break;
+
+            default:
+                std::cerr << "ignore unknown commandline code:" << c << std::endl;
+        }
+    }
+}
+
 // should we force the user to call Conf::initialize() by throw an expcetion
 // for uninitialized configuration?
 const Conf* Conf::get() noexcept(true) {
@@ -170,6 +215,39 @@ const Conf* Conf::get() noexcept(true) {
     return Conf::singleton.get();
 }
 
+void Conf::loadExtraFile(const std::string& default_file_name, const char* env_var_name) {
+    std::string real_file_name;
+    // Use the file named in the environment variable if it exists and is readable
+    if(env_var_name) {
+        const char* env_filename = std::getenv(env_var_name);
+        struct stat stat_buffer;
+        if(env_filename && stat(env_filename, &stat_buffer) == 0) {
+            if(S_ISREG(stat_buffer.st_mode) && (S_IRUSR | stat_buffer.st_mode)) {
+                real_file_name = env_filename;
+            }
+        }
+    }
+    // If that didn't work, see if the default file name works
+    if(real_file_name.empty()) {
+        struct stat stat_buffer;
+        if(stat(default_file_name.c_str(), &stat_buffer) == 0) {
+            if(S_ISREG(stat_buffer.st_mode) && (S_IRUSR | stat_buffer.st_mode)) {
+                real_file_name = default_file_name;
+            }
+        }
+    }
+    // If both the environment variable and the default don't exist, throw an error
+    if(real_file_name.empty()) {
+        throw std::runtime_error("Could not open configuration file " + default_file_name);
+    }
+    while(Conf::singleton_initialized_flag.load(std::memory_order_acquire) != CONF_INITIALIZED) {
+        char* empty_arg[1] = {nullptr};
+        Conf::initialize(0, empty_arg, nullptr);
+    }
+    singleton->loadFromFile(real_file_name);
+}
+
+
 const std::string& getConfString(const std::string& key) {
     return Conf::get()->getString(key);
 }

src/conf/derecho-sample.cfg

@@ -9,10 +9,6 @@ leader_external_port = 32645
 restart_leaders = 127.0.0.1,127.0.0.1
 # list of GMS ports of the restart leaders, in the same order
 restart_leader_ports = 23580,23581
-# my local id - each node should have a different id
-local_id = 0
-# my local ip address
-local_ip = 127.0.0.1
 # derecho gms port
 gms_port = 23580
 # derecho state-transfer port
@@ -116,45 +112,6 @@ block_size = 1024
 window_size = 50
 rdmc_send_algorithm = binomial_send
 
-# RDMA section contains configurations of the following
-# - which RDMA device to use
-# - device configurations
-[RDMA]
-# 1. provider = bgq|gni|efa|hook|netdir|psm|psm2|psm3|rxd|rxm|shm|udp|usnic|verbs
-# possible options(only 'sockets' and 'verbs' providers are tested so far):
-# bgq     - The Blue Gene/Q Fabric Provider
-# efa     - The Amazon Elastic Fabric Adapter
-# gni     - The GNI Fabric Provider (Cray XC (TM) systems)
-# hook    - The Hook Fabric Provider Utility
-# netdir  - The Network Direct Fabric Provider (Microsoft Network Direct SPI)
-# psm     - The PSM Fabric Provider
-# psm2    - The PSM2 Fabric Provider
-# psm3    - The PSM3 Fabric Provider
-# rxd     - The RxD (RDM over DGRAM) Utility Provider
-# rxm     - The RxM (RDM over MSG) Utility Provider
-# shm     - The SHM Fabric Provider
-# tcp     - The TCP Fabric Provider
-# udp     - The UDP Fabric Provider
-# usnic   - The usNIC Fabric Provider (Cisco VIC)
-# verbs   - The Verbs Fabric Provider
-# Please note that only "tcp" and "verbs" are tested this moment.
-provider = tcp
-
-# 2. domain
-# For sockets provider, domain is the NIC name (ifconfig | grep -v -e "^ ")
-# For verbs provider, domain is the device name (ibv_devices)
-domain = eth0
-
-# 3. tx_depth
-# tx_depth applies to hints->tx_attr->size, where hint is a struct fi_info object.
-# see https://ofiwg.github.io/libfabric/master/man/fi_getinfo.3.html
-tx_depth = 256
-
-# 4. rx_depth:
-# rx_depth applies to hints->rx_attr->size, where hint is a struct fi_info object.
-# see https://ofiwg.github.io/libfabric/master/man/fi_getinfo.3.html
-rx_depth = 256
-
 # Persistent configurations
 [PERS]
 # persistent directory for file system-based logfile.

src/conf/derecho_node-sample.cfg

@@ -0,0 +1,57 @@
+[DERECHO]
+# my local id - each node should have a different id
+local_id = 0
+# my local ip address
+local_ip = 127.0.0.1
+# These ports are optional: nodes will use the values from the group derecho.cfg by default,
+# but if the port options are specified here they will override the defaults.
+# derecho gms port
+gms_port = 23580
+# derecho state-transfer port
+state_transfer_port = 28366
+# sst tcp port
+sst_port = 37683
+# rdmc tcp port
+rdmc_port = 31675
+# externel tcp port listening to external clients
+external_port = 32645
+
+
+# RDMA section contains configurations of the following
+# - which RDMA device to use
+# - device configurations
+[RDMA]
+# 1. provider = bgq|gni|efa|hook|netdir|psm|psm2|psm3|rxd|rxm|shm|udp|usnic|verbs
+# possible options(only 'sockets' and 'verbs' providers are tested so far):
+# bgq     - The Blue Gene/Q Fabric Provider
+# efa     - The Amazon Elastic Fabric Adapter
+# gni     - The GNI Fabric Provider (Cray XC (TM) systems)
+# hook    - The Hook Fabric Provider Utility
+# netdir  - The Network Direct Fabric Provider (Microsoft Network Direct SPI)
+# psm     - The PSM Fabric Provider
+# psm2    - The PSM2 Fabric Provider
+# psm3    - The PSM3 Fabric Provider
+# rxd     - The RxD (RDM over DGRAM) Utility Provider
+# rxm     - The RxM (RDM over MSG) Utility Provider
+# shm     - The SHM Fabric Provider
+# tcp     - The TCP Fabric Provider
+# udp     - The UDP Fabric Provider
+# usnic   - The usNIC Fabric Provider (Cisco VIC)
+# verbs   - The Verbs Fabric Provider
+# Please note that only "tcp" and "verbs" are tested this moment.
+provider = tcp
+
+# 2. domain
+# For sockets provider, domain is the NIC name (ifconfig | grep -v -e "^ ")
+# For verbs provider, domain is the device name (ibv_devices)
+domain = eth0
+
+# 3. tx_depth
+# tx_depth applies to hints->tx_attr->size, where hint is a struct fi_info object.
+# see https://ofiwg.github.io/libfabric/master/man/fi_getinfo.3.html
+tx_depth = 256
+
+# 4. rx_depth:
+# rx_depth applies to hints->rx_attr->size, where hint is a struct fi_info object.
+# see https://ofiwg.github.io/libfabric/master/man/fi_getinfo.3.html
+rx_depth = 256

src/core/git_version.cpp

@@ -13,8 +13,8 @@ namespace derecho {
 const int MAJOR_VERSION = 2;
 const int MINOR_VERSION = 3;
 const int PATCH_VERSION = 0;
-const int COMMITS_AHEAD_OF_VERSION = 91;
+const int COMMITS_AHEAD_OF_VERSION = 92;
 const char* VERSION_STRING = "2.3.0";
-const char* VERSION_STRING_PLUS_COMMITS = "2.3.0+91";
+const char* VERSION_STRING_PLUS_COMMITS = "2.3.0+92";
 
 }