From 77e734f009d9103992360be7043119182d2b2513 Mon Sep 17 00:00:00 2001 From: Megh Bhatt Date: Wed, 7 Dec 2016 15:23:45 -0800 Subject: [PATCH] Change default compaction strategy and add option for flow tables Change default compaction strategy to SizeTieredCompactionStrategy Add CASSANDRA.flow_tables.compaction_strategy and default to DateTieredCompactionStrategy Closes-Bug: #1648252 Conflicts: src/analytics/db_handler.cc src/analytics/db_handler.h src/analytics/main.cc src/analytics/test/db_handler_mock.h src/analytics/viz_collector.cc src/analytics/viz_collector.h Change-Id: I4742dceaece999b31e065e004188e30919b461c3 --- src/analytics/db_handler.cc | 6 +++- src/analytics/db_handler.h | 3 ++ src/analytics/generator.cc | 2 +- src/analytics/main.cc | 1 + src/analytics/options.cc | 48 ++++++++++++++++++++-------- src/analytics/options.h | 4 +++ src/analytics/protobuf_collector.cc | 4 +-- src/analytics/test/db_handler_mock.h | 4 +-- src/analytics/viz_collector.cc | 4 ++- src/analytics/viz_collector.h | 1 + 10 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/analytics/db_handler.cc b/src/analytics/db_handler.cc index 9c78aa04b63..a372bc5e25d 100644 --- a/src/analytics/db_handler.cc +++ b/src/analytics/db_handler.cc @@ -72,6 +72,7 @@ DbHandler::DbHandler(EventManager *evm, const std::string& cassandra_user, const std::string& cassandra_password, const std::string &cassandra_compaction_strategy, + const std::string &cassandra_flow_tables_compaction_strategy, bool use_cql, const std::string &zookeeper_server_list, bool use_zookeeper, bool disable_all_writes, bool disable_statistics_writes, bool disable_messages_writes, @@ -82,6 +83,7 @@ DbHandler::DbHandler(EventManager *evm, use_cql_(use_cql), tablespace_(), compaction_strategy_(cassandra_compaction_strategy), + flow_tables_compaction_strategy_(cassandra_flow_tables_compaction_strategy), gen_partition_no_((uint8_t)g_viz_constants.PARTITION_MIN, (uint8_t)g_viz_constants.PARTITION_MAX), zookeeper_server_list_(zookeeper_server_list), @@ -187,7 +189,7 @@ bool DbHandler::CreateTables() { for (std::vector::const_iterator it = vizd_flow_tables.begin(); it != vizd_flow_tables.end(); it++) { - if (!dbif_->Db_AddColumnfamily(*it, compaction_strategy_)) { + if (!dbif_->Db_AddColumnfamily(*it, flow_tables_compaction_strategy_)) { DB_LOG(ERROR, it->cfname_ << " FAILED"); return false; } @@ -1794,6 +1796,7 @@ DbHandlerInitializer::DbHandlerInitializer(EventManager *evm, const std::vector &cassandra_ports, const TtlMap& ttl_map, const std::string &cassandra_user, const std::string &cassandra_password, const std::string &cassandra_compaction_strategy, + const std::string &cassandra_flow_tables_compaction_strategy, bool use_cql, const std::string &zookeeper_server_list, bool use_zookeeper, bool disable_all_db_writes, bool disable_db_stats_writes, bool disable_db_messages_writes, @@ -1804,6 +1807,7 @@ DbHandlerInitializer::DbHandlerInitializer(EventManager *evm, boost::bind(&DbHandlerInitializer::ScheduleInit, this), cassandra_ips, cassandra_ports, db_name, ttl_map, cassandra_user, cassandra_password, cassandra_compaction_strategy, + cassandra_flow_tables_compaction_strategy, use_cql, zookeeper_server_list, use_zookeeper, disable_all_db_writes, disable_db_stats_writes, disable_db_messages_writes, disable_db_messages_keyword_writes)), diff --git a/src/analytics/db_handler.h b/src/analytics/db_handler.h index 49f88e86c65..0edd2f46eac 100644 --- a/src/analytics/db_handler.h +++ b/src/analytics/db_handler.h @@ -89,6 +89,7 @@ class DbHandler { const std::string& cassandra_user, const std::string& cassandra_password, const std::string& cassandra_compaction_strategy, + const std::string& cassandra_flow_tables_compaction_strategy, bool use_cql, const std::string &zookeeper_server_list, bool use_zookeeper, bool disable_all_writes, bool disable_stats_writes, bool disable_messages_writes, bool disable_messages_keyword_writes); @@ -208,6 +209,7 @@ class DbHandler { bool use_cql_; std::string tablespace_; std::string compaction_strategy_; + std::string flow_tables_compaction_strategy_; UniformInt8RandomGenerator gen_partition_no_; std::string zookeeper_server_list_; bool use_zookeeper_; @@ -255,6 +257,7 @@ class DbHandlerInitializer { const std::string &cassandra_user, const std::string &cassandra_password, const std::string &cassandra_compaction_strategy, + const std::string &cassandra_flow_tables_compaction_strategy, bool use_cql, const std::string &zookeeper_server_list, bool use_zookeeper, bool disable_all_db_writes, diff --git a/src/analytics/generator.cc b/src/analytics/generator.cc index d3169550063..423cbeff32f 100644 --- a/src/analytics/generator.cc +++ b/src/analytics/generator.cc @@ -111,7 +111,7 @@ SandeshGenerator::SandeshGenerator(Collector * const collector, VizSession *sess source + ":" + node_type + ":" + module + ":" + instance_id, collector->analytics_ttl_map(), collector->cassandra_user(), collector->cassandra_password(), - std::string(), false, std::string(), + std::string(), std::string(), false, std::string(), false, false, false, false, false)); } else { //Use collector db_handler diff --git a/src/analytics/main.cc b/src/analytics/main.cc index 9745597c12a..c001a6af14d 100644 --- a/src/analytics/main.cc +++ b/src/analytics/main.cc @@ -371,6 +371,7 @@ int main(int argc, char *argv[]) ttl_map, options.cassandra_user(), options.cassandra_password(), options.cassandra_compaction_strategy(), + options.cassandra_flow_tables_compaction_strategy(), use_cql, zookeeper_server_list, use_zookeeper, options.disable_all_db_writes(), diff --git a/src/analytics/options.cc b/src/analytics/options.cc index 16ea91aae99..efc569b25e9 100644 --- a/src/analytics/options.cc +++ b/src/analytics/options.cc @@ -91,8 +91,12 @@ void Options::Initialize(EventManager &evm, "Cassandra password") ("CASSANDRA.compaction_strategy", opt::value()->default_value( - GenDb::g_gendb_constants.LEVELED_COMPACTION_STRATEGY), - "Cassandra compaction strategy");; + GenDb::g_gendb_constants.SIZE_TIERED_COMPACTION_STRATEGY), + "Cassandra compaction strategy") + ("CASSANDRA.flow_tables.compaction_strategy", + opt::value()->default_value( + GenDb::g_gendb_constants.DATE_TIERED_COMPACTION_STRATEGY), + "Cassandra compaction strategy for flow tables"); // Command line and config file options. opt::options_description config("Configuration options"); @@ -272,6 +276,25 @@ void Options::GetOptValueImpl( } } +static bool ValidateCompactionStrategyOption( + const std::string &compaction_strategy, + const std::string &option) { + if (!((compaction_strategy == + GenDb::g_gendb_constants.DATE_TIERED_COMPACTION_STRATEGY) || + (compaction_strategy == + GenDb::g_gendb_constants.LEVELED_COMPACTION_STRATEGY) || + (compaction_strategy == + GenDb::g_gendb_constants.SIZE_TIERED_COMPACTION_STRATEGY))) { + cout << "Invalid " << option << ", please select one of [" << + GenDb::g_gendb_constants.DATE_TIERED_COMPACTION_STRATEGY << ", " << + GenDb::g_gendb_constants.LEVELED_COMPACTION_STRATEGY << ", " << + GenDb::g_gendb_constants.SIZE_TIERED_COMPACTION_STRATEGY << "]" << + endl; + return false; + } + return true; +} + // Process command line options. They can come from a conf file as well. Options // from command line always overrides those that come from the config file. void Options::Process(int argc, char *argv[], @@ -374,18 +397,15 @@ void Options::Process(int argc, char *argv[], GetOptValue(var_map, cassandra_password_, "CASSANDRA.cassandra_password"); GetOptValue(var_map, cassandra_compaction_strategy_, "CASSANDRA.compaction_strategy"); - if (!((cassandra_compaction_strategy_ == - GenDb::g_gendb_constants.DATE_TIERED_COMPACTION_STRATEGY) || - (cassandra_compaction_strategy_ == - GenDb::g_gendb_constants.LEVELED_COMPACTION_STRATEGY) || - (cassandra_compaction_strategy_ == - GenDb::g_gendb_constants.SIZE_TIERED_COMPACTION_STRATEGY))) { - cout << "Invalid CASSANDRA.compaction_strategy," << - " please select one of [" << - GenDb::g_gendb_constants.DATE_TIERED_COMPACTION_STRATEGY << ", " << - GenDb::g_gendb_constants.LEVELED_COMPACTION_STRATEGY << ", " << - GenDb::g_gendb_constants.SIZE_TIERED_COMPACTION_STRATEGY << "]" << - endl; + if (!ValidateCompactionStrategyOption(cassandra_compaction_strategy_, + "CASSANDRA.compaction_strategy")) { + exit(-1); + } + GetOptValue(var_map, cassandra_flow_tables_compaction_strategy_, + "CASSANDRA.flow_tables.compaction_strategy"); + if (!ValidateCompactionStrategyOption( + cassandra_flow_tables_compaction_strategy_, + "CASSANDRA.flow_tables.compaction_strategy")) { exit(-1); } } diff --git a/src/analytics/options.h b/src/analytics/options.h index a566d09e92a..5516efe41c5 100644 --- a/src/analytics/options.h +++ b/src/analytics/options.h @@ -45,6 +45,9 @@ class Options { const std::string cassandra_compaction_strategy() const { return cassandra_compaction_strategy_; } + const std::string cassandra_flow_tables_compaction_strategy() const { + return cassandra_flow_tables_compaction_strategy_; + } const std::string hostname() const { return hostname_; } const std::string host_ip() const { return host_ip_; } const uint16_t http_server_port() const { return http_server_port_; } @@ -116,6 +119,7 @@ class Options { std::string cassandra_user_; std::string cassandra_password_; std::string cassandra_compaction_strategy_; + std::string cassandra_flow_tables_compaction_strategy_; std::string hostname_; std::string host_ip_; uint16_t http_server_port_; diff --git a/src/analytics/protobuf_collector.cc b/src/analytics/protobuf_collector.cc index b5840eb22c0..3fddbe24a2e 100644 --- a/src/analytics/protobuf_collector.cc +++ b/src/analytics/protobuf_collector.cc @@ -24,8 +24,8 @@ ProtobufCollector::ProtobufCollector(EventManager *evm, db_initializer_.reset(new DbHandlerInitializer(evm, kDbName, kDbTaskInstance, kDbTaskName, boost::bind(&ProtobufCollector::DbInitializeCb, this), cassandra_ips, cassandra_ports, ttl_map, cassandra_user, - cassandra_password, std::string(), false, std::string(), false, - false, false, false, false)); + cassandra_password, std::string(), std::string(), false, + std::string(), false, false, false, false, false)); db_handler_ = db_initializer_->GetDbHandler(); } else { db_handler_ = global_dbhandler; diff --git a/src/analytics/test/db_handler_mock.h b/src/analytics/test/db_handler_mock.h index a656fff3a1a..8b652387dd4 100644 --- a/src/analytics/test/db_handler_mock.h +++ b/src/analytics/test/db_handler_mock.h @@ -14,8 +14,8 @@ class DbHandlerMock : public DbHandler { DbHandlerMock(EventManager *evm, const TtlMap& ttl_map) : DbHandler(evm, boost::bind(&DbHandlerMock::StartDbifReinit, this), std::vector(1, "127.0.0.1"), - std::vector(1, 9160), "localhost", ttl_map, "", "", "", false, - "", false, false, false, false, false) + std::vector(1, 9160), "localhost", ttl_map, "", "", "", "", + false, "", false, false, false, false, false) { } void StartDbifReinit() { diff --git a/src/analytics/viz_collector.cc b/src/analytics/viz_collector.cc index 5494aeeaa0d..25d7b0a2f76 100644 --- a/src/analytics/viz_collector.cc +++ b/src/analytics/viz_collector.cc @@ -44,6 +44,7 @@ VizCollector::VizCollector(EventManager *evm, unsigned short listen_port, const std::string &cassandra_user, const std::string &cassandra_password, const std::string &cassandra_compaction_strategy, + const std::string &cassandra_flow_tables_compaction_strategy, bool use_cql, const std::string &zookeeper_server_list, bool use_zookeeper, bool disable_all_db_writes, bool disable_db_stats_writes, bool disable_db_messages_writes, @@ -52,7 +53,8 @@ VizCollector::VizCollector(EventManager *evm, unsigned short listen_port, std::string("collector:DbIf"), boost::bind(&VizCollector::DbInitializeCb, this), cassandra_ips, cassandra_ports, ttl_map, cassandra_user, - cassandra_password, cassandra_compaction_strategy, use_cql, + cassandra_password, cassandra_compaction_strategy, + cassandra_flow_tables_compaction_strategy, use_cql, zookeeper_server_list, use_zookeeper, disable_all_db_writes, disable_db_stats_writes, disable_db_messages_writes, disable_db_messages_keyword_writes)), diff --git a/src/analytics/viz_collector.h b/src/analytics/viz_collector.h index 15e5bd6c1f4..70950b67319 100644 --- a/src/analytics/viz_collector.h +++ b/src/analytics/viz_collector.h @@ -41,6 +41,7 @@ class VizCollector { const std::string& cassandra_user, const std::string& cassandra_password, const std::string &cassandra_compaction_strategy, + const std::string &cassandra_flow_tables_compaction_strategy, bool use_cql, const std::string &zookeeper_server_list, bool use_zookeeper, bool disable_all_db_writes,