Skip to content

Commit

Permalink
Tracebuffer Support for Instance Manager
Browse files Browse the repository at this point in the history 
Currently there are no trace messages for Instancemanager. This change
adds the support of traceuffer.

closes-bug: #1527429

Conflicts:
	src/vnsw/agent/oper/agent.sandesh
	src/vnsw/agent/oper/instance_manager.cc

Change-Id: Ia39dce9f1280324935b6c54937baacc013167a59
  • Loading branch information
@divakardhar
divakardhar committed May 27, 2016
1 parent 34dabcd commit 41e6e9d
Show file tree
Hide file tree
Showing 4 changed files with 110 additions and 33 deletions.
9 changes: 9 additions & 0 deletions src/vnsw/agent/oper/agent.sandesh
View file
Expand Up @@ -1271,3 +1271,12 @@ request sandesh VrouterObjectLimitsReq {
response sandesh VrouterObjectLimitsResp {
1: VrouterObjectLimits vrouter_object_limit;
}

/**
* @description: Trace message for InstanceManager
* @type: Trace
* @severity: DEBUG
*/
trace sandesh InstanceManagerTrace {
1: string message;
}
126 changes: 94 additions & 32 deletions src/vnsw/agent/oper/instance_manager.cc
View file
Expand Up @@ -27,6 +27,8 @@
#include "base/util.h"

using boost::uuids::uuid;
SandeshTraceBufferPtr InstanceManagerTraceBuf(
SandeshTraceBufferCreate("InstanceManager", 1000));

static const char loadbalancer_config_path_default[] =
"/var/lib/contrail/loadbalancer/";
Expand Down Expand Up @@ -111,32 +113,40 @@ class InstanceManager::NamespaceStaleCleaner {
if (fs::exists(conf_path.str())) {
fs::remove_all(conf_path.str(), error);
if (error) {
LOG(ERROR, "Stale loadbalancer conf fle delete error"
<< error.message());
std::stringstream ss;
ss << "Stale loadbalancer cfg file delete error ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

if (fs::exists(sock_path.str())) {
fs::remove_all(sock_path.str(), error);
if (error) {
LOG(ERROR, "Stale loadbalancer sock fle delete error"
<< error.message());
std::stringstream ss;
ss << "Stale loadbalancer sock file delete error ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

if (fs::exists(json_path.str())) {
fs::remove_all(json_path.str(), error);
if (error) {
LOG(ERROR, "Stale loadbalancer json file delete error"
<< error.message());
std::stringstream ss;
ss << "Stale loadbalancer json file delete error ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

if (fs::exists(crt_path.str())) {
fs::remove_all(crt_path.str(), error);
if (error) {
LOG(ERROR, "Stale loadbalancer certificate file delete error"
<< error.message());
std::stringstream ss;
ss << "Stale loadbalancer certificate file delete error ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}
}
Expand Down Expand Up @@ -245,6 +255,10 @@ void InstanceManager::OnTaskTimeout(InstanceTaskQueue *task_queue) {
}

void InstanceManager::OnTaskTimeoutEventHandler(InstanceManagerChildEvent event) {
std::stringstream ss;
ss << "TaskTimeOut for the TaskQ " << event.task_queue;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

ScheduleNextTask(event.task_queue);
}

Expand All @@ -254,6 +268,10 @@ void InstanceManager::OnErrorEventHandler(InstanceManagerChildEvent event) {
return;
}

std::stringstream ss;
ss << "Error for the Task " << event.task << " " << event.errors;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

InstanceState *state = GetState(svc_instance);
if (state != NULL) {
state->set_errors(event.errors);
Expand All @@ -266,6 +284,10 @@ void InstanceManager::OnExitEventHandler(InstanceManagerChildEvent event) {
return;
}

std::stringstream ss;
ss << "Exit event for the Task " << event.task;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

UpdateStateStatusType(event);
for (std::vector<InstanceTaskQueue *>::iterator iter =
task_queues_.begin();
Expand Down Expand Up @@ -325,9 +347,6 @@ void InstanceManager::UpdateStateStatusType(InstanceManagerChildEvent event) {
}

state->set_status(error_status);
LOG(DEBUG, "NetNS update status for uuid: "
<< svc_instance->ToString()
<< " " << error_status);

if (error_status != 0) {
if (state->status_type() != InstanceState::Timeout) {
Expand All @@ -338,6 +357,11 @@ void InstanceManager::UpdateStateStatusType(InstanceManagerChildEvent event) {
} else if (state->status_type() == InstanceState::Stopping) {
state->set_status_type(InstanceState::Stopped);
}

std::stringstream ss;
ss << "For the task " << event.task << " error status " <<
error_status << " status type " << state->status_type();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}
}
Expand Down Expand Up @@ -456,6 +480,12 @@ bool InstanceManager::StartTask(InstanceTaskQueue *task_queue,

pid_t pid;
bool status = task->Run();

std::stringstream ss;
ss << "Run status for the task " << task << " " << status;
ss << " With running status " << task->is_running();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (status || task->is_running()) {
pid = task->pid();
if (state != NULL) {
Expand All @@ -468,13 +498,19 @@ bool InstanceManager::StartTask(InstanceTaskQueue *task_queue,
}
}
} else {
LOG(ERROR, "Instance task " << task << " attempt " << task->reattempts());

ss.str(std::string());
ss << "Run failure for the task " << task << " attempt " << task->reattempts();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (state) {
state->set_status_type(InstanceState::Reattempt);
state->set_cmd(task->cmd());
}
if (task->incr_reattempts() > netns_reattempts_) {
LOG(ERROR, "Instance task " << task << " reattempts exceeded");
ss.str(std::string());
ss << "Run failure for the task " << task << " attempts exceeded";
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
return false;
}
}
Expand Down Expand Up @@ -506,11 +542,11 @@ void InstanceManager::ScheduleNextTask(InstanceTaskQueue *task_queue) {
state->set_status_type(InstanceState::Timeout);
}

LOG(ERROR, "NetNS error timeout " << delay << " > " <<
netns_timeout_ << ", " << task->cmd());

LOG(ERROR, " Delay " << delay << "Timeout " <<
(netns_timeout_ * 2));
std::stringstream ss;
ss << "Timeout for the Task " << task << " delay " << delay;
ss << " netns timeout " << netns_timeout_ << " ";
ss << task->cmd();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (delay >= (netns_timeout_ * 2)) {
task->Terminate();
Expand All @@ -529,7 +565,10 @@ void InstanceManager::ScheduleNextTask(InstanceTaskQueue *task_queue) {

task_svc_instances_.erase(task);

LOG(ERROR, "Delete task " << task);
std::stringstream ss;
ss << "Delete of the Task " << task;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

DeleteState(svc_instance);

delete task;
Expand Down Expand Up @@ -604,24 +643,31 @@ void InstanceManager::StartServiceInstance(ServiceInstance *svc_instance,
InstanceState *state, bool update) {
const ServiceInstance::Properties &props = svc_instance->properties();
InstanceManagerAdapter *adapter = this->FindApplicableAdapter(props);
std::stringstream ss;
if (adapter != NULL) {
InstanceTask *task = adapter->CreateStartTask(props, update);
if (task != NULL) {

ss << "Starting the Task " << task << " " << task->cmd();
ss << " for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

task->set_on_error_cb(boost::bind(&InstanceManager::OnError,
this, _1, _2));
task->set_on_exit_cb(boost::bind(&InstanceManager::OnExit,
this, _1, _2));
state->set_properties(props);
RegisterSvcInstance(task, svc_instance);
std::stringstream info;
info << "Service run command queued: " << task->cmd();
Enqueue(task, props.instance_id);
LOG(DEBUG, info.str().c_str());
} else {
LOG(ERROR, "Error creating task!");
ss << "Error Starting the Task for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
} else {
LOG(ERROR, "Unknown virtualization type: " << props.virtualization_type);
ss << "Unknown virtualization type " << props.virtualization_type;
ss << " for " << svc_instance->ToString();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

Expand All @@ -630,23 +676,30 @@ void InstanceManager::StopServiceInstance(ServiceInstance *svc_instance,
InstanceState *state) {
const ServiceInstance::Properties &props = state->properties();
InstanceManagerAdapter *adapter = this->FindApplicableAdapter(props);
std::stringstream ss;
if (adapter != NULL) {
InstanceTask *task = adapter->CreateStopTask(props);
if (task != NULL) {
ss << "Stopping the Task " << task << " " << task->cmd();
ss << " for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

task->set_on_error_cb(boost::bind(&InstanceManager::OnError,
this, _1, _2));
task->set_on_exit_cb(boost::bind(&InstanceManager::OnExit,
this, _1, _2));
RegisterSvcInstance(task, svc_instance);
std::stringstream info;
info << "Service stop command queued: " << task->cmd();
Enqueue(task, props.instance_id);
LOG(DEBUG, info.str().c_str());
} else {
LOG(ERROR, "Error creating task!");
std::stringstream ss;
ss << "Error Stopping the Task for " << props.instance_id;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
} else {
LOG(ERROR, "Unknown virtualization type: " << props.virtualization_type);
ss << "Unknown virtualization type " << props.virtualization_type;
ss << " for " << svc_instance->ToString();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
}
}

Expand Down Expand Up @@ -699,7 +752,9 @@ void InstanceManager::StopStaleNetNS(ServiceInstance::Properties &props) {
c_argv[i] = argv[i].c_str();
}

LOG(ERROR, "Stale NetNS " << cmd);
std::stringstream ss;
ss << "StaleNetNS " << cmd;
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

pid_t pid = vfork();
if (pid == 0) {
Expand Down Expand Up @@ -767,10 +822,13 @@ void InstanceManager::EventObserver(
}

bool usable = svc_instance->IsUsable();
LOG(DEBUG, "NetNS event notification for uuid: " << svc_instance->ToString()
<< (usable ? " usable" : " not usable"));

std::stringstream ss;
ss << "NetNS event notification for uuid: " << svc_instance->ToString();
ss << (usable ? " usable" : " not usable");
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());

if (!usable && GetLastCmdType(svc_instance) == Start) {
LOG(DEBUG, "Stopping service instance!");
StopServiceInstance(svc_instance, state);
SetLastCmdType(svc_instance, Stop);
} else if (usable) {
Expand Down Expand Up @@ -798,7 +856,11 @@ void InstanceManager::LoadbalancerObserver(
if (!boost::filesystem::exists(dir, error)) {
boost::filesystem::create_directories(dir, error);
if (error) {
LOG(ERROR, error.message());
std::stringstream ss;
ss << "CreateDirectory error for ";
ss << UuidToString(loadbalancer->uuid()) << " ";
ss << error.message();
INSTANCE_MANAGER_TRACE(Trace, ss.str().c_str());
return;
}
}
Expand Down
6 changes: 6 additions & 0 deletions src/vnsw/agent/oper/instance_manager.h
View file
Expand Up @@ -18,6 +18,12 @@ class InstanceState;
class InstanceTask;
class InstanceTaskQueue;

extern SandeshTraceBufferPtr InstanceManagerTraceBuf;
#define INSTANCE_MANAGER_TRACE(obj, ...) \
do { \
InstanceManager##obj::TraceMsg(InstanceManagerTraceBuf, __FILE__, __LINE__, __VA_ARGS__);\
} while (false);

/*
* Starts and stops network namespaces corresponding to service-instances.
*
Expand Down
2 changes: 1 addition & 1 deletion src/vnsw/agent/oper/instance_task.cc
View file
Expand Up @@ -64,7 +64,6 @@ void InstanceTaskExecvp::Terminate() {
// instance manager has to rely on TaskTimeout delete the task.
bool InstanceTaskExecvp::Run() {
std::vector<std::string> argv;
LOG(DEBUG, "NetNS run command: " << cmd_);

is_running_ = true;

Expand Down Expand Up @@ -125,6 +124,7 @@ bool InstanceTaskExecvp::Run() {
this, boost::asio::placeholders::error,
boost::asio::placeholders::bytes_transferred));
return true;

}

InstanceTaskQueue::InstanceTaskQueue(EventManager *evm) : evm_(evm),
Expand Down

0 comments on commit 41e6e9d

Please sign in to comment.