Skip to content

Commit

Permalink
Merge "Adding anomaly detections for: per-vrouter drop-stats, per-VMI…
Browse files Browse the repository at this point in the history 
… interface stats (in vrouter), and database writes on collector. For database writes, the reporting messages is coverted from objectlog to UVE. Also, query schema is corrected for SandeshClient message stats. Vrouter-agent adds the capability of overriding DerivedStats arguments in conf file. Closes-Bug:1607018" into R3.1
  • Loading branch information
@opencontrail-ci-admin
Zuul authored and opencontrail-ci-admin committed Aug 4, 2016
2 parents c444121 + 7907f18 commit fb99daa
Show file tree
Hide file tree
Showing 17 changed files with 213 additions and 108 deletions.
34 changes: 28 additions & 6 deletions src/analytics/collector_uve.sandesh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ include "io/io.sandesh"
include "sandesh/library/common/sandesh_uve.sandesh"
include "database/gendb.sandesh"
include "database/cassandra/cql/cql.sandesh"
include "sandesh/library/common/derived_stats_results.sandesh"

/**
* structure to store sandesh stats on a per message type
Expand Down Expand Up @@ -131,12 +132,33 @@ struct CollectorStats {
* database operations in the contrail-collector
* @object: analytics-node
*/
objectlog sandesh CollectorDbStats {
1: string name (key="ObjectCollectorInfo")
2: optional list<gendb.DbTableInfo> table_info (tags=".table_name")
3: optional gendb.DbErrors errors (tags="")
4: optional list<gendb.DbTableInfo> statistics_table_info (tags=".table_name")
5: optional cql.DbStats cql_stats (tags="")

struct CollectorDbStats {
1: string name (key="ObjectCollectorInfo")
99: optional bool deleted
2: optional map<string, gendb.DbTableStat> table_info (tags=".__key", metric="diff")
3: optional gendb.DbErrors errors (tags="")
4: optional map<string, gendb.DbTableStat> stats_info (tags=".__key", metric="diff")
5: optional cql.DbStats cql_stats (tags="")

10: optional map<string, gendb.DbTableStat_P_>
table_info_1h (mstats="60-table_info:DSSum:", hidden="yes")
11: optional map<string, gendb.DbTableStat_P_>
table_info_10m (mstats="10-table_info:DSSum:")
12: optional map<string, derived_stats_results.AnomalyResult>
table_writes_anomaly (mstats="0-table_info.writes:DSAnomaly:EWM:0.2")

20: optional map<string, gendb.DbTableStat_P_>
stats_info_1h (mstats="60-stats_info:DSSum:", hidden="yes")
21: optional map<string, gendb.DbTableStat_P_>
stats_info_10m (mstats="10-stats_info:DSSum:")
22: optional map<string, derived_stats_results.AnomalyResult>
stats_writes_anomaly (mstats="0-stats_info.writes:DSAnomaly:EWM:0.2")

} (period="60")

uve sandesh CollectorDbStatsTrace {
1: CollectorDbStats data
}

/**
Expand Down
1 change: 1 addition & 0 deletions src/analytics/db_handler.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ class DbHandler {
uint64_t timestamp, GenDb::GenDbIf::DbAddColumnCb db_cb);

bool GetStats(uint64_t *queue_count, uint64_t *enqueues) const;

bool GetStats(std::vector<GenDb::DbTableInfo> *vdbti,
GenDb::DbErrors *dbe, std::vector<GenDb::DbTableInfo> *vstats_dbti);
bool GetCumulativeStats(std::vector<GenDb::DbTableInfo> *vdbti,
Expand Down
76 changes: 28 additions & 48 deletions src/analytics/viz.sandesh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1510,52 +1510,32 @@ const list<stat_table> _STAT_TABLES = [
{
'display_name' : 'Sandesh Client Message Stats',
'stat_type' : 'ModuleClientState',
'stat_attr' : 'tx_msg_agg_diff.counters',
'stat_attr' : 'tx_msg_diff',
'obj_table' : GENERATOR_INFO_TABLE,
'attributes': [
{ 'name' : 'tx_msg_agg_diff.counters.__key', 'datatype' : 'string', 'index' : true},
{ 'name' : 'tx_msg_agg_diff.counters.__value', 'datatype' : 'string', 'index' : false},
{ 'name' : 'tx_msg_diff.__key', 'datatype' : 'string', 'index' : true},
{ 'name' : 'tx_msg_diff.__value', 'datatype' : 'string', 'index' : false},
]
},
{
'display_name' : 'Generator Database Table Statistics',
'stat_type' : 'GeneratorDbStats',
'stat_attr' : 'table_info',
'obj_table' : GENERATOR_INFO_TABLE,
'attributes': [
{ 'name' : 'table_info.table_name', 'datatype' : 'string', 'index' : true},
{ 'name' : 'table_info.reads', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_info.read_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_info.writes', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_info.write_fails', 'datatype' : 'int', 'index' : false},
]
},
{
'display_name' : 'Generator Statistics Database Table',
'stat_type' : 'GeneratorDbStats',
'stat_attr' : 'statistics_table_info',
'obj_table' : GENERATOR_INFO_TABLE,
'attributes': [
{ 'name' : 'statistics_table_info.table_name', 'datatype' : 'string', 'index' : true},
{ 'name' : 'statistics_table_info.reads', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.read_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.writes', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.write_fails', 'datatype' : 'int', 'index' : false},
]
},
{
'display_name' : 'Generator Database Errors',
'stat_type' : 'GeneratorDbStats',
'stat_attr' : 'errors',
'display_name' : 'Sandesh Client Message-Type Stats',
'stat_type' : 'ModuleClientState',
'stat_attr' : 'msg_type_diff',
'obj_table' : GENERATOR_INFO_TABLE,
'attributes': [
{ 'name' : 'errors.write_tablespace_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'errors.read_tablespace_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'errors.write_table_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'errors.read_table_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'errors.write_column_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'errors.write_batch_column_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'errors.read_column_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'msg_type_diff.__key', 'datatype' : 'string', 'index' : true},
{ 'name' : 'msg_type_diff.messages_sent', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_no_queue', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_no_client', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_no_session', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_queue_level', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_client_send_failed', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_session_not_connected', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_header_write_failed', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_write_failed', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_wrong_client_sm_state', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_validation_failed', 'datatype' : 'string', 'index' : false},
{ 'name' : 'msg_type_diff.messages_sent_dropped_rate_limited', 'datatype' : 'string', 'index' : false},
]
},
{
Expand All @@ -1564,7 +1544,7 @@ const list<stat_table> _STAT_TABLES = [
'stat_attr' : 'table_info',
'obj_table' : COLLECTOR_INFO_TABLE,
'attributes': [
{ 'name' : 'table_info.table_name', 'datatype' : 'string', 'index' : true},
{ 'name' : 'table_info.__key', 'datatype' : 'string', 'index' : true},
{ 'name' : 'table_info.reads', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_info.read_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_info.writes', 'datatype' : 'int', 'index' : false},
Expand All @@ -1575,15 +1555,15 @@ const list<stat_table> _STAT_TABLES = [
{
'display_name' : 'Collector Statistics Database Table',
'stat_type' : 'CollectorDbStats',
'stat_attr' : 'statistics_table_info',
'stat_attr' : 'stats_info',
'obj_table' : COLLECTOR_INFO_TABLE,
'attributes': [
{ 'name' : 'statistics_table_info.table_name', 'datatype' : 'string', 'index' : true},
{ 'name' : 'statistics_table_info.reads', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.read_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.writes', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.write_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'statistics_table_info.write_back_pressure_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'stats_info.__key', 'datatype' : 'string', 'index' : true},
{ 'name' : 'stats_info.reads', 'datatype' : 'int', 'index' : false},
{ 'name' : 'stats_info.read_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'stats_info.writes', 'datatype' : 'int', 'index' : false},
{ 'name' : 'stats_info.write_fails', 'datatype' : 'int', 'index' : false},
{ 'name' : 'stats_info.write_back_pressure_fails', 'datatype' : 'int', 'index' : false},
]
},
{
Expand Down Expand Up @@ -2305,7 +2285,7 @@ const list<stat_table> _STAT_TABLES = [
'stat_attr' : 'table_stats',
'obj_table' : ROUTING_INSTANCE_TABLE,
'attributes': [
{ 'name' : 'table_stats.address_family', 'datatype' : 'string', 'index' : true},
{ 'name' : 'table_stats.__key', 'datatype' : 'string', 'index' : true},
{ 'name' : 'table_stats.prefixes', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_stats.primary_paths', 'datatype' : 'int', 'index' : false},
{ 'name' : 'table_stats.secondary_paths', 'datatype' : 'int', 'index' : false},
Expand Down
42 changes: 35 additions & 7 deletions src/analytics/viz_collector.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@

using std::stringstream;
using std::string;
using std::map;
using std::make_pair;
using boost::system::error_code;

VizCollector::VizCollector(EventManager *evm, unsigned short listen_port,
Expand Down Expand Up @@ -237,16 +239,42 @@ void VizCollector::SendDbStatistics() {
std::vector<GenDb::DbTableInfo> vdbti, vstats_dbti;
GenDb::DbErrors dbe;
db_handler->GetStats(&vdbti, &dbe, &vstats_dbti);
CollectorDbStats *snh(COLLECTOR_DB_STATS_CREATE());
snh->set_name(name_);
snh->set_table_info(vdbti);
snh->set_errors(dbe);
snh->set_statistics_table_info(vstats_dbti);

// TODO: Change DBStats to return a map directly
map<string,GenDb::DbTableStat> mtstat, msstat;

for (size_t idx=0; idx<vdbti.size(); idx++) {
GenDb::DbTableStat dtis;
dtis.set_reads(vdbti[idx].get_reads());
dtis.set_read_fails(vdbti[idx].get_read_fails());
dtis.set_writes(vdbti[idx].get_writes());
dtis.set_write_fails(vdbti[idx].get_write_fails());
dtis.set_write_back_pressure_fails(vdbti[idx].get_write_back_pressure_fails());
mtstat.insert(make_pair(vdbti[idx].get_table_name(), dtis));
}

for (size_t idx=0; idx<vstats_dbti.size(); idx++) {
GenDb::DbTableStat dtis;
dtis.set_reads(vstats_dbti[idx].get_reads());
dtis.set_read_fails(vstats_dbti[idx].get_read_fails());
dtis.set_writes(vstats_dbti[idx].get_writes());
dtis.set_write_fails(vstats_dbti[idx].get_write_fails());
dtis.set_write_back_pressure_fails(
vstats_dbti[idx].get_write_back_pressure_fails());
msstat.insert(make_pair(vstats_dbti[idx].get_table_name(), dtis));
}

CollectorDbStats cds;
cds.set_name(name_);
cds.set_table_info(mtstat);
cds.set_errors(dbe);
cds.set_stats_info(msstat);

cass::cql::DbStats cql_stats;
if (db_handler->GetCqlStats(&cql_stats)) {
snh->set_cql_stats(cql_stats);
cds.set_cql_stats(cql_stats);
}
COLLECTOR_DB_STATS_SEND_SANDESH(snh);
CollectorDbStatsTrace::Send(cds);
}

bool VizCollector::GetCqlMetrics(cass::cql::Metrics *metrics) {
Expand Down
13 changes: 13 additions & 0 deletions src/database/gendb.sandesh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,19 @@ struct DbTableInfo {
6: u64 write_back_pressure_fails
}

struct DbTableStat {
2: u64 reads
3: u64 read_fails
4: u64 writes
5: u64 write_fails
6: u64 write_back_pressure_fails
}

struct DbTableStat_P_ {
1: optional DbTableStat value
2: optional DbTableStat staging
}

struct DbErrors {
1: u64 write_tablespace_fails
2: u64 read_tablespace_fails
Expand Down
7 changes: 5 additions & 2 deletions src/opserver/opserver_util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,9 @@ def uve_attr_flatten(inp):
for idx in range(0,int(inp['map']['@size'])):
m_attr = inp['map']['element'][idx]
subst = {}
for sk,sv in inp['map'][sname][idx].iteritems():
subst[sk] = OpServerUtils.uve_attr_flatten(sv)
if inp['map'][sname][idx]:
for sk,sv in inp['map'][sname][idx].iteritems():
subst[sk] = OpServerUtils.uve_attr_flatten(sv)
fmap[m_attr] = subst
return fmap
else:
Expand Down Expand Up @@ -554,6 +555,8 @@ def messages_data_dict_to_str(messages_dict, message_type, sandesh_type):
@staticmethod
def _data_dict_to_str(data_dict, sandesh_type):
data_str = None
if not data_dict:
return ''
for key, value in data_dict.iteritems():
# Ignore if type is sandesh
if '@type' == key and value == 'sandesh':
Expand Down
3 changes: 2 additions & 1 deletion src/vnsw/agent/cfg/discovery_agent.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,8 @@ void DiscoveryAgentClient::DiscoverServices() {
agent->introspect_port(),
csf,
list,
NULL);
NULL,
param_->derived_stats_map());
}

//subscribe to Xmpp Server on controller
Expand Down
4 changes: 2 additions & 2 deletions src/vnsw/agent/cmn/agent.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,15 +433,15 @@ void Agent::InitCollector() {
event_manager(),
params_->http_server_port(), 0,
params_->collector_server_list(),
NULL);
NULL, params_->derived_stats_map());
} else {
Sandesh::InitGenerator(module_name(),
host_name(),
g_vns_constants.NodeTypeNames.find(node_type)->second,
instance_id_,
event_manager(),
params_->http_server_port(),
NULL);
NULL, params_->derived_stats_map());
}

}
Expand Down
48 changes: 44 additions & 4 deletions src/vnsw/agent/init/agent_param.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,12 +233,44 @@ bool AgentParam::ParseServerListArguments
return true;
}

void AgentParam::ParseCollector() {
std::map<string, std::map<string, string> >
AgentParam::ParseDerivedStats(const std::vector<std::string> &dsvec) {
std::map<string, std::map<string, string> > dsmap;
std::map<string, std::map<string, string> >::iterator dsiter;

for (size_t idx=0; idx!=dsvec.size(); idx++) {
size_t pos = dsvec[idx].find(':');
assert(pos != string::npos);
string dsfull = dsvec[idx].substr(0,pos);
string dsarg = dsvec[idx].substr(pos+1, string::npos);

size_t dpos = dsfull.find('.');
assert(dpos != string::npos);
string dsstruct = dsfull.substr(0,dpos);
string dsattr = dsfull.substr(dpos+1, string::npos);

dsiter = dsmap.find(dsstruct);
std::map<string, string> dselem;
if (dsiter!=dsmap.end()) dselem = dsiter->second;
dselem[dsattr] = dsarg;

dsmap[dsstruct] = dselem;
}
return dsmap;
}

void AgentParam::ParseCollectorDS() {
optional<string> opt_str;
if (opt_str = tree_.get_optional<string>("DEFAULT.collectors")) {
boost::split(collector_server_list_, opt_str.get(),
boost::is_any_of(" "));
}
if (opt_str = tree_.get_optional<string>("DEFAULT.derived_stats")) {
vector<string> dsvec;
boost::split(dsvec, opt_str.get(),
boost::is_any_of(" "));
derived_stats_map_ = ParseDerivedStats(dsvec);
}
}

void AgentParam::BuildAddressList(const string &val) {
Expand Down Expand Up @@ -639,10 +671,15 @@ void AgentParam::ParseServices() {
GetValueFromTree<uint32_t>(services_queue_limit_, "SERVICES.queue_limit");
}

void AgentParam::ParseCollectorArguments
void AgentParam::ParseCollectorDSArguments
(const boost::program_options::variables_map &var_map) {
GetOptValue< vector<string> >(var_map, collector_server_list_,
"DEFAULT.collectors");
vector<string> dsvec;
if (GetOptValue< vector<string> >(var_map, dsvec,
"DEFAULT.derived_stats")) {
derived_stats_map_ = ParseDerivedStats(dsvec);
}
}

void AgentParam::ParseVirtualHostArguments
Expand Down Expand Up @@ -926,7 +963,7 @@ void AgentParam::InitFromConfig() {
return;
}

ParseCollector();
ParseCollectorDS();
ParseVirtualHost();
ParseServerList("CONTROL-NODE.server", &xmpp_server_1_, &xmpp_server_2_);
ParseDns();
Expand All @@ -950,7 +987,7 @@ void AgentParam::InitFromConfig() {
}

void AgentParam::InitFromArguments() {
ParseCollectorArguments(var_map_);
ParseCollectorDSArguments(var_map_);
ParseVirtualHostArguments(var_map_);
ParseServerListArguments(var_map_, xmpp_server_1_, xmpp_server_2_,
"CONTROL-NODE.server");
Expand Down Expand Up @@ -1394,6 +1431,9 @@ AgentParam::AgentParam(bool enable_flow_options,
("DEFAULT.collectors",
opt::value<std::vector<std::string> >()->multitoken(),
"Collector server list")
("DEFAULT.derived_stats",
opt::value<std::vector<std::string> >()->multitoken(),
"Derived Stats Parameters")
("DEFAULT.flow_cache_timeout",
opt::value<uint16_t>()->default_value(Agent::kDefaultFlowCacheTimeout),
"Flow aging time in seconds")
Expand Down

0 comments on commit fb99daa

Please sign in to comment.