Skip to content

Commit

Permalink
Merge "- DerivedStats periodic UVE sends from sandesh client state ma…
Browse files Browse the repository at this point in the history 
…chine are now enabled. - In the periodic map case, DerivedStats object will be retained even if the update does not supply a raw value for the object. (Object are cleared during the periodic flush anyway). In the non-periodic case, such DerivedStats objects will be deleted - DerivedStats implementations can now choose to not return a result even if they have been fed samples. - A DerivedStats implementation has been added to convert per-category aggregate counts into diffs ("DerivedStatsCategoryCounter"). The first use is for generator sent message statistics. The categories used are either "ok", or the error string ("no_session", "rate_limited", etc.)"
  • Loading branch information
@opencontrail-ci-admin
Zuul authored and opencontrail-ci-admin committed May 17, 2016
2 parents de01e10 + 70981e6 commit c8af604
Show file tree
Hide file tree
Showing 15 changed files with 547 additions and 252 deletions.
156 changes: 127 additions & 29 deletions compiler/generate/t_cpp_generator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,8 @@ void generate_sandesh_async_creator_helper(ofstream &out, t_sandesh *tsandesh, b
void generate_sandesh_static_rate_limit_mutex_def(ofstream& out, t_sandesh* tsandesh);
void generate_sandesh_static_rate_limit_buffer_def(ofstream& out, t_sandesh* tsandesh);
void derived_stats_info(t_struct* tstruct,
// val is rawtype,resulttype,annotation
map<string,tuple<string,string,string> >& dsmap,
// val is rawtype,resulttype,algo,annotation
map<string,tuple<string,string,string,string> >& dsmap,
// val is set of ds attributes
map<string,set<string> >& rawmap);

Expand Down Expand Up @@ -472,6 +472,7 @@ void t_cpp_generator::init_generator() {
#else
"#include <boost/shared_ptr.hpp>" << endl <<
"#include <sandesh/derived_stats.h>" << endl <<
"#include <sandesh/derived_stats_algo.h>" << endl <<
"#include <boost/pointer_cast.hpp>" << endl <<
#endif
"#include <base/trace.h>" << endl;
Expand Down Expand Up @@ -2184,8 +2185,8 @@ void t_cpp_generator::cache_attr_info(t_struct* tstruct,


void t_cpp_generator::derived_stats_info(t_struct* tstruct,
// val is rawtype,resulttype,annotation
map<string,tuple<string,string,string> >& dsmap,
// val is rawtype,resulttype,algo,annotation
map<string,tuple<string,string,string,string> >& dsmap,
// val is set of ds attributes
map<string,set<string> >& rawmap) {
const vector<t_field*>& members = tstruct->get_members();
Expand All @@ -2208,7 +2209,11 @@ void t_cpp_generator::derived_stats_info(t_struct* tstruct,
const string &tstr = jt->second;
size_t pos = tstr.find(':');
string rawattr = tstr.substr(0,pos);
string anno = tstr.substr(pos+1, string::npos);

string residual = tstr.substr(pos+1, string::npos);
size_t rpos = residual.find(':');
string algo = residual.substr(0,rpos);
string anno = residual.substr(rpos+1, string::npos);

string rawtype;
vector<t_field*>::const_iterator s_iter;
Expand All @@ -2231,7 +2236,7 @@ void t_cpp_generator::derived_stats_info(t_struct* tstruct,
}
// map of derived stats
dsmap.insert(make_pair((*m_iter)->get_name(),
make_tuple(rawtype, restype, anno)));
make_tuple(rawtype, restype, algo, anno)));

// map of raw attributes
map<string,set<string> >::iterator r_iter = rawmap.find(rawattr);
Expand Down Expand Up @@ -2332,10 +2337,10 @@ void t_cpp_generator::generate_struct_definition(ofstream& out,
std::map<string, CacheAttribute> cache_attrs;
cache_attr_info(tstruct, cache_attrs);

map<string,tuple<string,string,string> > dsinfo;
map<string,tuple<string,string,string,string> > dsinfo;
map<string,set<string> > rawmap;
derived_stats_info(tstruct, dsinfo, rawmap);
map<string,tuple<string,string,string> >::iterator ds_iter;
map<string,tuple<string,string,string,string> >::iterator ds_iter;
#endif

if (!pointers) {
Expand Down Expand Up @@ -2406,16 +2411,18 @@ void t_cpp_generator::generate_struct_definition(ofstream& out,

if (cat == PERIODIC) {
out << ", __dsobj_" << ds_iter->first <<
"(new DerivedStatsPeriodicIf<" <<
"(new ::contrail::sandesh::DerivedStatsPeriodicIf< ::contrail::sandesh::" <<
ds_iter->second.get<2>() << ", " <<
ds_iter->second.get<0>() << ", " << ds_iter->second.get<1>() <<
"Elem, " << ds_iter->second.get<1>() <<
">(\"" << ds_iter->second.get<2>() << "\"))";
">(\"" << ds_iter->second.get<3>() << "\"))";
} else {

out << ", __dsobj_" << ds_iter->first <<
"(new DerivedStatsIf<" <<
"(new ::contrail::sandesh::DerivedStatsIf< ::contrail::sandesh::" <<
ds_iter->second.get<2>() << ", " <<
ds_iter->second.get<0>() << ", " << ds_iter->second.get<1>() <<
">(\"" << ds_iter->second.get<2>() << "\"))";
">(\"" << ds_iter->second.get<3>() << "\"))";
}
}
if (is_table) {
Expand Down Expand Up @@ -2458,25 +2465,27 @@ void t_cpp_generator::generate_struct_definition(ofstream& out,
declare_field(*m_iter, false, pointers && !(*m_iter)->get_type()->is_xception(), !read) << endl;
}
#ifdef SANDESH
if (is_table) {
indent(out) << "std::string table_;" << endl;
}

for (ds_iter = dsinfo.begin(); ds_iter != dsinfo.end(); ++ds_iter) {
CacheAttribute cat = (cache_attrs.find(ds_iter->first))->second;

if (cat == PERIODIC) {
indent(out) << "boost::shared_ptr<DerivedStatsPeriodicIf<" <<
indent(out) << "boost::shared_ptr< ::contrail::sandesh::DerivedStatsPeriodicIf< ::contrail::sandesh::" <<
ds_iter->second.get<2>() << ", " <<
ds_iter->second.get<0>() << ", " << ds_iter->second.get<1>() <<
"Elem, " << ds_iter->second.get<1>() <<
"> > __dsobj_" << ds_iter->first << ";" << endl;
} else {
indent(out) << "boost::shared_ptr<DerivedStatsIf<" <<
indent(out) << "boost::shared_ptr< ::contrail::sandesh::DerivedStatsIf< ::contrail::sandesh::" <<
ds_iter->second.get<2>() << ", " <<
ds_iter->second.get<0>() << ", " << ds_iter->second.get<1>() <<
"> > __dsobj_" << ds_iter->first << ";" << endl;
}

}
if (is_table) {
indent(out) << "std::string table_;" << endl;
}

#endif

// Add the __isset data member if we need it, using the definition from above
Expand Down Expand Up @@ -2529,15 +2538,11 @@ void t_cpp_generator::generate_struct_definition(ofstream& out,
indent()<< "}" << endl;

#ifdef SANDESH
ds_iter = dsinfo.find((*m_iter)->get_name());
if (ds_iter == dsinfo.end()) {
out << endl << indent() << type_name((*m_iter)->get_type(), false, true)
<< " get_";
out << (*m_iter)->get_name() << "() const {" << endl;
out << indent() << indent() << "return " << (*m_iter)->get_name() << ";" << endl;
out << indent() << "}" << endl;
}

out << endl << indent() << type_name((*m_iter)->get_type(), false, true)
<< " get_";
out << (*m_iter)->get_name() << "() const {" << endl;
out << indent() << indent() << "return " << (*m_iter)->get_name() << ";" << endl;
out << indent() << "}" << endl;
#endif
}
out << endl;
Expand Down Expand Up @@ -2607,6 +2612,99 @@ void t_cpp_generator::generate_struct_definition(ofstream& out,
}
#ifdef SANDESH
out << indent() << "std::string log() const;" << endl;
if (!pointers) {
out << indent() <<
tstruct->get_name() << " operator+(const " << tstruct->get_name() <<
"& right) const" << endl;
scope_up(out);
out << indent() << tstruct->get_name() << " result;" << endl;
for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) {
std::string nn = (*m_iter)->get_name();
ds_iter = dsinfo.find((*m_iter)->get_name());
std::string sn;
if (ds_iter == dsinfo.end()) sn = std::string("set_");
else sn = std::string("__set_");

t_type* type = get_true_type((*m_iter)->get_type());
if (type->is_base_type()) {
t_base_type::t_base tbase = ((t_base_type*)type)->get_base();
switch (tbase) {
case t_base_type::TYPE_BYTE:
case t_base_type::TYPE_I16:
case t_base_type::TYPE_I32:
case t_base_type::TYPE_I64:
case t_base_type::TYPE_DOUBLE:
case t_base_type::TYPE_U16:
case t_base_type::TYPE_U32:
case t_base_type::TYPE_U64:
if ((*m_iter)->get_req() != t_field::T_OPTIONAL) {
out <<
indent() << "result." << sn << nn << "(get_" << nn <<
"() + right.get_" << nn << "());" << endl << endl;
} else {
out <<
indent() << "if ((__isset." << nn << ") && (right.__isset." <<
nn << ")) result." << sn << nn << "(get_" << nn <<
"() + right.get_" << nn << "());" << endl <<
indent() << "else if (__isset." << nn << ") result." << sn <<
nn << "(get_" << nn << "());" << endl <<
indent() << "else if (right.__isset." << nn << ") result." << sn <<
nn << "(right.get_" << nn << "());" << endl << endl;
}
default:
continue;
}
}
}
indent(out) << "return result;" << endl;
scope_down(out);

out << indent() <<
tstruct->get_name() << " operator-(const " << tstruct->get_name() <<
"& right) const" << endl;
scope_up(out);
out << indent() << tstruct->get_name() << " result;" << endl;
for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) {
std::string nn = (*m_iter)->get_name();
ds_iter = dsinfo.find((*m_iter)->get_name());
std::string sn;
if (ds_iter == dsinfo.end()) sn = std::string("set_");
else sn = std::string("__set_");
t_type* type = get_true_type((*m_iter)->get_type());
if (type->is_base_type()) {
t_base_type::t_base tbase = ((t_base_type*)type)->get_base();
switch (tbase) {
case t_base_type::TYPE_BYTE:
case t_base_type::TYPE_I16:
case t_base_type::TYPE_I32:
case t_base_type::TYPE_I64:
case t_base_type::TYPE_DOUBLE:
case t_base_type::TYPE_U16:
case t_base_type::TYPE_U32:
case t_base_type::TYPE_U64:
if ((*m_iter)->get_req() != t_field::T_OPTIONAL) {
out <<
indent() << "result." << sn << nn << "(get_" << nn <<
"() - right.get_" << nn << "());" << endl << endl;
} else {
out <<
indent() << "if ((__isset." << nn << ") && (right.__isset." <<
nn << ")) {" << endl <<
indent() << " result." << sn << nn << "(get_" << nn <<
"() - right.get_" << nn << "()); }" << endl <<
indent() << "else if (__isset." << nn << ") result." << sn <<
nn << "(get_" << nn << "());" << endl <<
indent() << "if (!result.get_" << nn << "()) result.__isset." <<
nn << " = false;" << endl;
}
default:
continue;
}
}
}
indent(out) << "return result;" << endl;
scope_down(out);
}
out << indent() << "size_t GetSize() const;" << endl;
out << indent() << "std::string __listkey(void) {" << endl;
out << indent() << " std::string __result;" << endl;
Expand Down Expand Up @@ -3316,7 +3414,7 @@ void t_cpp_generator::generate_sandesh_updater(ofstream& out,

t_struct* ts = (t_struct*)t;

map<string,tuple<string,string,string> > dsinfo;
map<string,tuple<string,string,string,string> > dsinfo;
map<string,set<string> > rawmap;
derived_stats_info(ts, dsinfo, rawmap);

Expand Down Expand Up @@ -3745,7 +3843,7 @@ void t_cpp_generator::generate_sandesh_uve_creator(
"g_sandesh_constants.SANDESH_SYNC_HINT);" << endl;
indent(out) << "snh->Dispatch();" << endl;
indent_down();
indent(out) << "}" << endl;
indent(out) << "} else snh->Release();" << endl;
indent_down();
indent(out) << "}" << endl << endl;

Expand Down
12 changes: 10 additions & 2 deletions library/common/derived_stats_results.sandesh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,20 @@ struct NullResult {

struct EWMResult {
3: u64 samples
5: double alpha
6: double avg
6: double mean
7: double stddev
}

struct PercentileResult {
3: u64 samples
4: map<string,double> percentiles;
}

struct CategoryResultSubElem {
1: string category
2: u64 count
}
struct CategoryResult {
1: u64 samples
2: list<CategoryResultSubElem> results (tags=".category")
}
72 changes: 39 additions & 33 deletions library/common/sandesh_uve.sandesh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
*/

include "io/io.sandesh"
include "sandesh/library/common/derived_stats_results.sandesh"

struct SandeshMessageStats {
1: u64 messages_sent;
Expand All @@ -21,42 +22,42 @@ struct SandeshMessageStats {
8: u64 bytes_received_dropped;
// Send
// Messages
51: u64 messages_sent_dropped_no_queue;
52: u64 messages_sent_dropped_no_client;
53: u64 messages_sent_dropped_no_session;
54: u64 messages_sent_dropped_queue_level;
55: u64 messages_sent_dropped_client_send_failed;
56: u64 messages_sent_dropped_session_not_connected;
57: u64 messages_sent_dropped_header_write_failed;
58: u64 messages_sent_dropped_write_failed;
59: u64 messages_sent_dropped_wrong_client_sm_state;
60: u64 messages_sent_dropped_validation_failed;
61: u64 messages_sent_dropped_rate_limited;
51: optional u64 messages_sent_dropped_no_queue;
52: optional u64 messages_sent_dropped_no_client;
53: optional u64 messages_sent_dropped_no_session;
54: optional u64 messages_sent_dropped_queue_level;
55: optional u64 messages_sent_dropped_client_send_failed;
56: optional u64 messages_sent_dropped_session_not_connected;
57: optional u64 messages_sent_dropped_header_write_failed;
58: optional u64 messages_sent_dropped_write_failed;
59: optional u64 messages_sent_dropped_wrong_client_sm_state;
60: optional u64 messages_sent_dropped_validation_failed;
61: optional u64 messages_sent_dropped_rate_limited;
// Bytes
81: u64 bytes_sent_dropped_no_queue;
82: u64 bytes_sent_dropped_no_client;
83: u64 bytes_sent_dropped_no_session;
84: u64 bytes_sent_dropped_queue_level;
85: u64 bytes_sent_dropped_client_send_failed;
86: u64 bytes_sent_dropped_session_not_connected;
87: u64 bytes_sent_dropped_header_write_failed;
88: u64 bytes_sent_dropped_write_failed;
89: u64 bytes_sent_dropped_wrong_client_sm_state;
90: u64 bytes_sent_dropped_validation_failed;
91: u64 bytes_sent_dropped_rate_limited;
81: optional u64 bytes_sent_dropped_no_queue;
82: optional u64 bytes_sent_dropped_no_client;
83: optional u64 bytes_sent_dropped_no_session;
84: optional u64 bytes_sent_dropped_queue_level;
85: optional u64 bytes_sent_dropped_client_send_failed;
86: optional u64 bytes_sent_dropped_session_not_connected;
87: optional u64 bytes_sent_dropped_header_write_failed;
88: optional u64 bytes_sent_dropped_write_failed;
89: optional u64 bytes_sent_dropped_wrong_client_sm_state;
90: optional u64 bytes_sent_dropped_validation_failed;
91: optional u64 bytes_sent_dropped_rate_limited;
// Receive
// Messages
101: u64 messages_received_dropped_no_queue;
102: u64 messages_received_dropped_queue_level;
103: u64 messages_received_dropped_create_failed;
104: u64 messages_received_dropped_control_msg_failed;
105: u64 messages_received_dropped_decoding_failed;
101: optional u64 messages_received_dropped_no_queue;
102: optional u64 messages_received_dropped_queue_level;
103: optional u64 messages_received_dropped_create_failed;
104: optional u64 messages_received_dropped_control_msg_failed;
105: optional u64 messages_received_dropped_decoding_failed;
// Bytes
131: u64 bytes_received_dropped_no_queue;
132: u64 bytes_received_dropped_queue_level;
133: u64 bytes_received_dropped_create_failed;
134: u64 bytes_received_dropped_control_msg_failed;
135: u64 bytes_received_dropped_decoding_failed;
131: optional u64 bytes_received_dropped_no_queue;
132: optional u64 bytes_received_dropped_queue_level;
133: optional u64 bytes_received_dropped_create_failed;
134: optional u64 bytes_received_dropped_control_msg_failed;
135: optional u64 bytes_received_dropped_decoding_failed;
}

struct SandeshMessageBasicStats {
Expand Down Expand Up @@ -232,12 +233,17 @@ struct ModuleClientState {
1: string name (key="ObjectGeneratorInfo")
2: optional bool deleted
3: optional SandeshClientInfo client_info
4: optional SandeshGeneratorBasicStats msg_stats
5: optional u64 sm_queue_count
6: optional SandeshStateMachineStats sm_stats
7: optional SandeshSessionStats session_stats
8: optional io.SocketIOStats session_rx_socket_stats
9: optional io.SocketIOStats session_tx_socket_stats

10: optional map<string, SandeshMessageStats> msg_type_agg (hidden="yes")
11: optional map<string, SandeshMessageStats> msg_type_diff (stats="msg_type_agg:DSDiff:")

12: optional list<derived_stats_results.CategoryResultSubElem> tx_msg_agg (hidden="yes")
13: optional derived_stats_results.CategoryResult tx_msg_agg_diff (stats="tx_msg_agg:DSCategoryCount:", tags="")
}

uve sandesh SandeshModuleClientTrace {
Expand Down

0 comments on commit c8af604

Please sign in to comment.