From 6a68d65473a2bc6baff5b52cd58d0ae1ed8f2714 Mon Sep 17 00:00:00 2001 From: Megh Bhatt Date: Wed, 23 Sep 2015 15:49:03 -0700 Subject: [PATCH] Check the return value of Sandesh::InitGenerator and exit if sandesh initialization fails. Added kafka timer shutdown in contrail-query-engine shutdown. Increased the startsecs time for contrail-dns since it was taking around 7 secs for it to exit and hence supervisor needs to wait for that much time to declare it up. Tested by returning false from sandesh HTTP initialization. Closes-Bug: #1474258 Conflicts: src/control-node/main.cc src/query_engine/qed.cc Change-Id: Ie1a9d62dc53d8fe9f28b537130d9e03b5fba2e89 (cherry picked from commit 7d1192720f6a69deee552c3ce518ec22bdf4f4a7) --- src/analytics/OpServerProxy.cc | 13 ++++++++- src/analytics/OpServerProxy.h | 1 + src/analytics/main.cc | 17 +++++++++--- src/analytics/viz_collector.cc | 1 + src/control-node/main.cc | 30 +++++++++++++++----- src/dns/contrail-dns.ini | 2 +- src/dns/main.cc | 19 ++++++++++--- src/query_engine/qed.cc | 51 +++++++++++++++++++++------------- 8 files changed, 97 insertions(+), 37 deletions(-) diff --git a/src/analytics/OpServerProxy.cc b/src/analytics/OpServerProxy.cc index eb74daabb9d..10b77647ef2 100644 --- a/src/analytics/OpServerProxy.cc +++ b/src/analytics/OpServerProxy.cc @@ -546,12 +546,16 @@ class OpServerProxy::OpServerImpl { return true; } - ~OpServerImpl() { + void Shutdown() { TimerManager::DeleteTimer(kafka_timer_); kafka_timer_ = NULL; StopKafka(); } + ~OpServerImpl() { + assert(kafka_timer_ == NULL); + } + RedisInfo redis_uve_; private: /* these are made public, so they are accessed by OpServerProxy */ @@ -591,6 +595,13 @@ OpServerProxy::~OpServerProxy() { delete impl_; } +void +OpServerProxy::Shutdown() { + if (impl_) { + impl_->Shutdown(); + } +} + bool OpServerProxy::UVENotif(const std::string &type, const std::string &source, const std::string &node_type, diff --git a/src/analytics/OpServerProxy.h b/src/analytics/OpServerProxy.h index 068a428ee04..5ec30a94338 100644 --- a/src/analytics/OpServerProxy.h +++ b/src/analytics/OpServerProxy.h @@ -32,6 +32,7 @@ class OpServerProxy { uint16_t partitions); OpServerProxy() : impl_(NULL) { } virtual ~OpServerProxy(); + virtual void Shutdown(); virtual bool UVEUpdate(const std::string &type, const std::string &attr, const std::string &source, const std::string &node_type, diff --git a/src/analytics/main.cc b/src/analytics/main.cc index 3cb0a869b21..48baa7fe10b 100644 --- a/src/analytics/main.cc +++ b/src/analytics/main.cc @@ -186,8 +186,11 @@ static void ShutdownServers(VizCollector *viz_collector, viz_collector->Shutdown(); - TimerManager::DeleteTimer(collector_info_log_timer); - delete collector_info_trigger; + if (collector_info_log_timer) { + TimerManager::DeleteTimer(collector_info_log_timer); + delete collector_info_trigger; + collector_info_log_timer = NULL; + } ConnectionStateManager:: GetInstance()->Shutdown(); @@ -356,13 +359,19 @@ int main(int argc, char *argv[]) unsigned short coll_port = analytics.GetCollector()->GetPort(); VizSandeshContext vsc(&analytics); Sandesh::set_send_rate_limit(options.sandesh_send_rate_limit()); - Sandesh::InitCollector( + bool success(Sandesh::InitCollector( module_id, analytics.name(), g_vns_constants.NodeTypeNames.find(node_type)->second, instance_id, a_evm, "127.0.0.1", coll_port, - options.http_server_port(), &vsc); + options.http_server_port(), &vsc)); + if (!success) { + LOG(ERROR, "SANDESH: Initialization FAILED ... exiting"); + ShutdownServers(&analytics, NULL); + delete a_evm; + exit(1); + } Sandesh::SetLoggingParams(options.log_local(), options.log_category(), options.log_level()); diff --git a/src/analytics/viz_collector.cc b/src/analytics/viz_collector.cc index 9b96e7c8743..4ccf92441f3 100644 --- a/src/analytics/viz_collector.cc +++ b/src/analytics/viz_collector.cc @@ -123,6 +123,7 @@ void VizCollector::Shutdown() { } TcpServerManager::DeleteServer(collector_); + osp_->Shutdown(); syslog_listener_->Shutdown(); WaitForIdle(); diff --git a/src/control-node/main.cc b/src/control-node/main.cc index 248b671c78f..4eed15ed6d0 100644 --- a/src/control-node/main.cc +++ b/src/control-node/main.cc @@ -152,8 +152,9 @@ static void ShutdownServers( channel_manager->reset(); TcpServerManager::DeleteServer(xmpp_server); - TimerManager::DeleteTimer(node_info_log_timer); - + if (node_info_log_timer) { + TimerManager::DeleteTimer(node_info_log_timer); + } // Shutdown Discovery Service Client ShutdownDiscoveryClient(dsclient); @@ -537,8 +538,9 @@ int main(int argc, char *argv[]) { if (sandesh_generator_init) { NodeType::type node_type = g_vns_constants.Module2NodeType.find(module)->second; + bool success; if (options.collectors_configured()) { - Sandesh::InitGenerator( + success = Sandesh::InitGenerator( module_name, options.hostname(), g_vns_constants.NodeTypeNames.find(node_type)->second, @@ -548,7 +550,7 @@ int main(int argc, char *argv[]) { options.collector_server_list(), &sandesh_context); } else { - Sandesh::InitGenerator( + success = Sandesh::InitGenerator( g_vns_constants.ModuleNames.find(module)->second, options.hostname(), g_vns_constants.NodeTypeNames.find(node_type)->second, @@ -557,6 +559,11 @@ int main(int argc, char *argv[]) { options.http_server_port(), &sandesh_context); } + if (!success) { + LOG(ERROR, "SANDESH: Initialization FAILED ... exiting"); + Sandesh::Uninit(); + exit(1); + } } Sandesh::SetLoggingParams(options.log_local(), options.log_category(), @@ -678,11 +685,20 @@ int main(int argc, char *argv[]) { csf = boost::bind(&DiscoveryServiceClient::Subscribe, ds_client, _1, _2, _3); vector list; - Sandesh::InitGenerator(subscriber_name, options.hostname(), + bool success(Sandesh::InitGenerator(subscriber_name, + options.hostname(), node_type_name, g_vns_constants.INSTANCE_ID_DEFAULT, - &evm, options.http_server_port(), - csf, list, &sandesh_context); + &evm, + options.http_server_port(), + csf, + list, + &sandesh_context)); + if (!success) { + LOG(ERROR, "SANDESH: Initialization FAILED ... exiting"); + ShutdownServers(&bgp_peer_manager, ds_client, NULL); + exit(1); + } } } else { LOG(ERROR, "Invalid Discovery Server hostname or address " << diff --git a/src/dns/contrail-dns.ini b/src/dns/contrail-dns.ini index ca5a3d35287..78b7a30feb6 100644 --- a/src/dns/contrail-dns.ini +++ b/src/dns/contrail-dns.ini @@ -8,6 +8,6 @@ stdout_capture_maxbytes=1MB redirect_stderr=true stdout_logfile=/var/log/contrail/contrail-dns-stdout.log stderr_logfile=/dev/null -startsecs=5 +startsecs=10 exitcodes=0 ; 'expected' exit codes for process (default 0,2) user=contrail diff --git a/src/dns/main.cc b/src/dns/main.cc index b1725ad087b..5980d0429c2 100644 --- a/src/dns/main.cc +++ b/src/dns/main.cc @@ -134,7 +134,7 @@ int main(int argc, char *argv[]) { if (options.discovery_server().empty()) { NodeType::type node_type = g_vns_constants.Module2NodeType.find(module)->second; - Sandesh::InitGenerator( + bool success(Sandesh::InitGenerator( module_name, options.hostname(), g_vns_constants.NodeTypeNames.find(node_type)->second, @@ -142,7 +142,12 @@ int main(int argc, char *argv[]) { Dns::GetEventManager(), options.http_server_port(), 0, options.collector_server_list(), - NULL); + NULL)); + if (!success) { + LOG(ERROR, "SANDESH: Initialization FAILED ... exiting"); + Sandesh::Uninit(); + exit(1); + } } Sandesh::SetLoggingParams(options.log_local(), options.log_category(), options.log_level()); @@ -234,7 +239,7 @@ int main(int argc, char *argv[]) { _1, _2, _3); vector list; list.clear(); - Sandesh::InitGenerator(subscriber_name, + bool success(Sandesh::InitGenerator(subscriber_name, options.hostname(), node_type_name, g_vns_constants.INSTANCE_ID_DEFAULT, @@ -242,7 +247,13 @@ int main(int argc, char *argv[]) { options.http_server_port(), csf, list, - NULL); + NULL)); + if (!success) { + LOG(ERROR, "SANDESH: Initialization FAILED ... exiting"); + Sandesh::Uninit(); + Dns::ShutdownDiscoveryClient(ds_client); + exit(1); + } } } else { LOG(ERROR, "Invalid Discovery Server hostname or ip " << diff --git a/src/query_engine/qed.cc b/src/query_engine/qed.cc index 03169d5dcf6..9cad4556d96 100644 --- a/src/query_engine/qed.cc +++ b/src/query_engine/qed.cc @@ -120,6 +120,23 @@ static bool OptionsParse(Options &options, EventManager &evm, return false; } +static void ShutdownQe(DiscoveryServiceClient *ds_client, + Timer *qe_info_log_timer) { + if (ds_client) { + ds_client->Shutdown(); + delete ds_client; + } + if (qe_info_log_timer) { + qe_info_log_timer->Cancel(); + TimerManager::DeleteTimer(qe_info_log_timer); + } + WaitForIdle(); + Sandesh::Uninit(); + ConnectionStateManager:: + GetInstance()->Shutdown(); + WaitForIdle(); +} + int main(int argc, char *argv[]) { EventManager evm; @@ -208,7 +225,7 @@ main(int argc, char *argv[]) { instance_id, boost::bind(&GetProcessStateCb, _1, _2, _3, 3)); Sandesh::set_send_rate_limit(options.sandesh_send_rate_limit()); - Sandesh::InitGenerator( + bool success(Sandesh::InitGenerator( module_name, options.hostname(), g_vns_constants.NodeTypeNames.find(node_type)->second, @@ -217,7 +234,13 @@ main(int argc, char *argv[]) { options.http_server_port(), csf, options.collector_server_list(), - NULL); + NULL)); + if (!success) { + LOG(ERROR, "SANDESH: Initialization FAILED ... exiting"); + ShutdownQe(ds_client, NULL); + exit(1); + } + Sandesh::SetLoggingParams(options.log_local(), options.log_category(), options.log_level()); @@ -249,9 +272,9 @@ main(int argc, char *argv[]) { ostream_iterator(css, " ")); LOG(INFO, "Cassandra Servers: " << css.str()); - QueryEngine *qe; + boost::scoped_ptr qe; if (cassandra_ports.size() == 1 && cassandra_ports[0] == 0) { - qe = new QueryEngine(&evm, + qe.reset(new QueryEngine(&evm, options.redis_server(), options.redis_port(), options.redis_password(), @@ -259,9 +282,9 @@ main(int argc, char *argv[]) { options.max_slice(), options.analytics_data_ttl(), options.cassandra_user(), - options.cassandra_password()); + options.cassandra_password())); } else { - qe = new QueryEngine(&evm, + qe.reset(new QueryEngine(&evm, cassandra_ips, cassandra_ports, options.redis_server(), @@ -272,9 +295,8 @@ main(int argc, char *argv[]) { options.analytics_data_ttl(), options.cassandra_user(), options.cassandra_password(), - options.start_time()); + options.start_time())); } - (void) qe; CpuLoadData::Init(); qe_info_trigger = @@ -286,17 +308,6 @@ main(int argc, char *argv[]) { signal(SIGTERM, terminate_qe); evm.Run(); - if (ds_client) { - ds_client->Shutdown(); - delete ds_client; - } - qe_info_log_timer->Cancel(); - TimerManager::DeleteTimer(qe_info_log_timer); - WaitForIdle(); - delete qe; - Sandesh::Uninit(); - ConnectionStateManager:: - GetInstance()->Shutdown(); - WaitForIdle(); + ShutdownQe(ds_client, qe_info_log_timer); return 0; }