Skip to content

Commit

Permalink
Merge "Tracebuffer Support for Instance Manager" into R3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@opencontrail-ci-admin
Zuul authored and opencontrail-ci-admin committed Feb 24, 2016
2 parents 296106c + f7403a2 commit 37d5ed1
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 28 deletions.
9 changes: 9 additions & 0 deletions src/vnsw/agent/oper/agent.sandesh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2044,3 +2044,12 @@ response sandesh BgpAsAServiceSandeshResp {
trace sandesh BgpAsAServiceTrace {
1: string message;
}

/**
* @description: Trace message for InstanceManager
* @type: Trace
* @severity: DEBUG
*/
trace sandesh InstanceManagerTrace {
1: string message;
}
111 changes: 84 additions & 27 deletions src/vnsw/agent/oper/instance_manager.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
#include "base/util.h"

using boost::uuids::uuid;
SandeshTraceBufferPtr InstanceManagerTraceBuf(
SandeshTraceBufferCreate("InstanceManager", 1000));

static const char loadbalancer_config_path_default[] =
"/var/lib/contrail/loadbalancer/";
Expand Down Expand Up @@ -108,8 +110,10 @@ class InstanceManager::NamespaceStaleCleaner {
if (fs::exists(cfg_dir_path.str())) {
fs::remove_all(cfg_dir_path.str(), error);
if (error) {
LOG(ERROR, "Stale loadbalancer cfg fle delete error"
<< error.message());
std::stringstream ss;
ss << "Stale loadbalancer cfg fle delete error ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}
}
Expand Down Expand Up @@ -234,6 +238,10 @@ void InstanceManager::OnTaskTimeout(InstanceTaskQueue *task_queue) {
}

void InstanceManager::OnTaskTimeoutEventHandler(InstanceManagerChildEvent event) {
std::stringstream ss;
ss << "TaskTimeOut for the TaskQ " << event.task_queue;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

ScheduleNextTask(event.task_queue);
}

Expand All @@ -243,6 +251,10 @@ void InstanceManager::OnErrorEventHandler(InstanceManagerChildEvent event) {
return;
}

std::stringstream ss;
ss << "Error for the Task " << event.task << " " << event.errors;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

InstanceState *state = GetState(svc_instance);
if (state != NULL) {
state->set_errors(event.errors);
Expand All @@ -255,6 +267,10 @@ void InstanceManager::OnExitEventHandler(InstanceManagerChildEvent event) {
return;
}

std::stringstream ss;
ss << "Exit event for the Task " << event.task;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

UpdateStateStatusType(event);
for (std::vector<InstanceTaskQueue *>::iterator iter =
task_queues_.begin();
Expand Down Expand Up @@ -315,9 +331,6 @@ void InstanceManager::UpdateStateStatusType(InstanceManagerChildEvent event) {
}

state->set_status(error_status);
LOG(DEBUG, "NetNS update status for uuid: "
<< svc_instance->ToString()
<< " " << error_status);

if (error_status != 0) {
if (state->status_type() != InstanceState::Timeout) {
Expand All @@ -328,6 +341,11 @@ void InstanceManager::UpdateStateStatusType(InstanceManagerChildEvent event) {
} else if (state->status_type() == InstanceState::Stopping) {
state->set_status_type(InstanceState::Stopped);
}

std::stringstream ss;
ss << "For the task " << event.task << " error status " <<
error_status << " status type " << state->status_type();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}
}
Expand Down Expand Up @@ -451,6 +469,12 @@ bool InstanceManager::StartTask(InstanceTaskQueue *task_queue,

pid_t pid;
bool status = task->Run();

std::stringstream ss;
ss << "Run status for the task " << task << " " << status;
ss << " With running status " << task->is_running();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (status || task->is_running()) {
pid = task->pid();
if (state != NULL) {
Expand All @@ -463,13 +487,19 @@ bool InstanceManager::StartTask(InstanceTaskQueue *task_queue,
}
}
} else {
LOG(ERROR, "Instance task " << task << " attempt " << task->reattempts());

ss.str(std::string());
ss << "Run failure for the task " << task << " attempt " << task->reattempts();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (state) {
state->set_status_type(InstanceState::Reattempt);
state->set_cmd(task->cmd());
}
if (task->incr_reattempts() > netns_reattempts_) {
LOG(ERROR, "Instance task " << task << " reattempts exceeded");
ss.str(std::string());
ss << "Run failure for the task " << task << " attempts exceeded";
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
return false;
}
}
Expand Down Expand Up @@ -501,11 +531,11 @@ void InstanceManager::ScheduleNextTask(InstanceTaskQueue *task_queue) {
state->set_status_type(InstanceState::Timeout);
}

LOG(ERROR, "NetNS error timeout " << delay << " > " <<
netns_timeout_ << ", " << task->cmd());

LOG(ERROR, " Delay " << delay << "Timeout " <<
(netns_timeout_ * 2));
std::stringstream ss;
ss << "Timeout for the Task " << task << " delay " << delay;
ss << " netns timeout " << netns_timeout_ << " ";
ss << task->cmd();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (delay >= (netns_timeout_ * 2)) {
task->Terminate();
Expand All @@ -524,7 +554,10 @@ void InstanceManager::ScheduleNextTask(InstanceTaskQueue *task_queue) {

task_svc_instances_.erase(task);

LOG(ERROR, "Delete task " << task);
std::stringstream ss;
ss << "Delete of the Task " << task;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

DeleteState(svc_instance);

delete task;
Expand Down Expand Up @@ -600,24 +633,29 @@ void InstanceManager::StartServiceInstance(ServiceInstance *svc_instance,
InstanceState *state, bool update) {
const ServiceInstance::Properties &props = svc_instance->properties();
InstanceManagerAdapter *adapter = this->FindApplicableAdapter(props);
std::stringstream ss;
if (adapter != NULL) {
InstanceTask *task = adapter->CreateStartTask(props, update);
if (task != NULL) {
ss << "Starting the Task " << task << " " << task->cmd();
ss << " for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
task->set_on_data_cb(boost::bind(&InstanceManager::OnError,
this, _1, _2));
task->set_on_exit_cb(boost::bind(&InstanceManager::OnExit,
this, _1, _2));
state->set_properties(props);
RegisterSvcInstance(task, svc_instance);
std::stringstream info;
info << "Service run command queued: " << task->cmd();
Enqueue(task, props.instance_id);
LOG(DEBUG, info.str().c_str());
} else {
LOG(ERROR, "Error creating task!");
ss << "Error Starting the Task for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
} else {
LOG(ERROR, "Unknown virtualization type: " << props.virtualization_type);
ss << "Unknown virtualization type " << props.virtualization_type;
ss << " for " << svc_instance->ToString();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

Expand All @@ -626,23 +664,29 @@ void InstanceManager::StopServiceInstance(ServiceInstance *svc_instance,
InstanceState *state) {
const ServiceInstance::Properties &props = state->properties();
InstanceManagerAdapter *adapter = this->FindApplicableAdapter(props);
std::stringstream ss;
if (adapter != NULL) {
InstanceTask *task = adapter->CreateStopTask(props);
if (task != NULL) {
ss << "Stopping the Task " << task << " " << task->cmd();
ss << " for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
task->set_on_data_cb(boost::bind(&InstanceManager::OnError,
this, _1, _2));
task->set_on_exit_cb(boost::bind(&InstanceManager::OnExit,
this, _1, _2));
RegisterSvcInstance(task, svc_instance);
std::stringstream info;
info << "Service stop command queued: " << task->cmd();
Enqueue(task, props.instance_id);
LOG(DEBUG, info.str().c_str());
} else {
LOG(ERROR, "Error creating task!");
std::stringstream ss;
ss << "Error Stopping the Task for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
} else {
LOG(ERROR, "Unknown virtualization type: " << props.virtualization_type);
ss << "Unknown virtualization type " << props.virtualization_type;
ss << " for " << svc_instance->ToString();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

Expand Down Expand Up @@ -695,7 +739,9 @@ void InstanceManager::StopStaleNetNS(ServiceInstance::Properties &props) {
c_argv[i] = argv[i].c_str();
}

LOG(ERROR, "Stale NetNS " << cmd);
std::stringstream ss;
ss << "StaleNetNS " << cmd;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

pid_t pid = vfork();
if (pid == 0) {
Expand Down Expand Up @@ -762,10 +808,13 @@ void InstanceManager::EventObserver(
}

bool usable = svc_instance->IsUsable();
LOG(DEBUG, "NetNS event notification for uuid: " << svc_instance->ToString()
<< (usable ? " usable" : " not usable"));

std::stringstream ss;
ss << "NetNS event notification for uuid: " << svc_instance->ToString();
ss << (usable ? " usable" : " not usable");
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (!usable && GetLastCmdType(svc_instance) == Start) {
LOG(DEBUG, "Stopping service instance!");
StopServiceInstance(svc_instance, state);
SetLastCmdType(svc_instance, Stop);
} else if (usable) {
Expand All @@ -792,7 +841,11 @@ void InstanceManager::LoadbalancerObserver(DBTablePartBase *db_part,
if (!boost::filesystem::exists(dir, error)) {
boost::filesystem::create_directories(dir, error);
if (error) {
LOG(ERROR, error.message());
std::stringstream ss;
ss << "CreateDirectory error for ";
ss << UuidToString(loadbalancer->uuid()) << " ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
return;
}
}
Expand All @@ -801,7 +854,11 @@ void InstanceManager::LoadbalancerObserver(DBTablePartBase *db_part,
} else {
boost::filesystem::remove_all(pathgen.str(), error);
if (error) {
LOG(ERROR, error.message());
std::stringstream ss;
ss << "Removeall error for ";
ss << UuidToString(loadbalancer->uuid()) << " ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
return;
}
}
Expand Down
6 changes: 6 additions & 0 deletions src/vnsw/agent/oper/instance_manager.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ class InstanceState;
class InstanceTask;
class InstanceTaskQueue;

extern SandeshTraceBufferPtr InstanceManagerTraceBuf;
#define INSTANCE_MANAGER_TRACE(obj, ...) \
do { \
InstanceManager##obj::TraceMsg(InstanceManagerTraceBuf, __FILE__, __LINE__, __VA_ARGS__);\
} while (false);

/*
* Starts and stops network namespaces corresponding to service-instances.
*
Expand Down
2 changes: 1 addition & 1 deletion src/vnsw/agent/oper/instance_task.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ void InstanceTaskExecvp::Terminate() {
// instance manager has to rely on TaskTimeout delete the task.
bool InstanceTaskExecvp::Run() {
std::vector<std::string> argv;
LOG(DEBUG, "NetNS run command: " << cmd_);

is_running_ = true;

Expand Down Expand Up @@ -130,6 +129,7 @@ bool InstanceTaskExecvp::Run() {
this, boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred));
return true;

}

InstanceTaskQueue::InstanceTaskQueue(EventManager *evm) : evm_(evm),
Expand Down

0 comments on commit 37d5ed1

Please sign in to comment.