Dynamic sandesh UVE implementation

Added a new sandesh type named "dynamicuve".
Defined sandesh of type "dynamicuve" in sandesh_dynamic_uve.sandesh
All dynamic UVEs would be derived from this base class defined in
sandesh_dynamic_uve.sandesh
The newly added sandesh type "dynamicuve" is only for parsing/code
generation. Dynamic UVEs are sent to collector with same type as sandesh
UVE. Therefore, dynamic UVEs are processed in the same way as sandesh
UVEs in the collector/analytics-api and hence there is no difference in
the interpretation of dynamic UVEs from sandesh UVEs from the user stand
point.

Related change:
Presently, _SANDESH_REQUEST_LIST, _SANDESH_UVE_LIST and
_SANDESH_ALARM_LIST contain the class name as string. Modified these
lists to contain class object instead of class name. pysandesh library
would store the class object instead of the class name and corresponding
module.

Change-Id: I9f15ff6613e4f2a23334291f165e7bcdd4bf6c62
Closes-Bug: #1469966

compiler/generate/t_py_generator.cc

@@ -33,6 +33,7 @@
 
 #ifdef SANDESH
 #include <cassert>
+#include "boost/tuple/tuple.hpp"
 #endif
 using namespace std;
 
@@ -163,9 +164,7 @@ class t_py_generator : public t_generator {
   void generate_py_sandesh_response(std::ofstream& out, t_sandesh* tsandesh);
   void generate_py_sandesh_uve(std::ofstream& out, t_sandesh* tsandesh);
   void generate_py_sandesh_uve_list(std::ofstream& out);
-  void generate_py_sandesh_uve_data_list(std::ofstream& out);
   void generate_py_sandesh_alarm_list(std::ofstream& out);
-  void generate_py_sandesh_alarm_data_list(std::ofstream& out);
   void generate_py_sandesh_trace(std::ofstream& out, t_sandesh* tsandesh);
   void generate_py_sandesh_sizeof(std::ofstream& out, t_sandesh* tsandesh);
 #endif
@@ -344,22 +343,12 @@ class t_py_generator : public t_generator {
   /**
    * Sandesh UVE List
    */
-  std::vector<std::string> sandesh_uve_list_;
-
-  /**
-   * Sandesh UVE Data List
-   */
-  std::vector<std::string> sandesh_uve_data_list_;
+  std::vector<boost::tuple<std::string, std::string> > sandesh_uve_list_;
 
   /**
    * Sandesh Alarm List
    */
-  std::vector<std::string> sandesh_alarm_list_;
-
-  /**
-   * Sandesh Alarm Data List
-   */
-  std::vector<std::string> sandesh_alarm_data_list_;
+  std::vector<boost::tuple<std::string, std::string> > sandesh_alarm_list_;
 #endif
 
   /**
@@ -574,9 +563,7 @@ void t_py_generator::close_generator() {
 #ifdef SANDESH
   generate_py_sandesh_request_list(f_types_);
   generate_py_sandesh_uve_list(f_types_);
-  generate_py_sandesh_uve_data_list(f_types_);
   generate_py_sandesh_alarm_list(f_types_);
-  generate_py_sandesh_alarm_data_list(f_types_);
 #endif
   // Close types file
   f_types_.close();
@@ -1212,7 +1199,7 @@ void t_py_generator::generate_py_struct_writer(ofstream& out,
 
 #ifdef SANDESH
   indent(out) <<
-    "if oprot.writeStructBegin('" << name << "') < 0: return -1" << endl;
+    "if oprot.writeStructBegin(self.__class__.__name__) < 0: return -1" << endl;
 #else
   indent(out) <<
     "oprot.writeStructBegin('" << name << "')" << endl;
@@ -1335,6 +1322,8 @@ void t_py_generator::generate_py_sandesh_definition(ofstream& out,
     base_class = "sandesh_base.SandeshResponse";
   } else if (sandesh_type->is_sandesh_uve()) {
     base_class = "sandesh_base.SandeshUVE";
+  } else if (sandesh_type->is_sandesh_dynamic_uve()) {
+    base_class = "sandesh_base.SandeshDynamicUVE";
   } else if (sandesh_type->is_sandesh_alarm()) {
     base_class = "sandesh_base.SandeshAlarm";
   } else if (sandesh_type->is_sandesh_trace() ||
@@ -1444,6 +1433,7 @@ void t_py_generator::generate_py_sandesh_definition(ofstream& out,
         "', level=SandeshLevel.SYS_INFO";
     }
     if (sandesh_type->is_sandesh_uve() ||
+        sandesh_type->is_sandesh_dynamic_uve() ||
         sandesh_type->is_sandesh_alarm()) {
       out << ", table=None";
     }
@@ -1495,6 +1485,7 @@ void t_py_generator::generate_py_sandesh_definition(ofstream& out,
       indent(out) << "self._level = level" << endl;
     }
     if (sandesh_type->is_sandesh_uve() ||
+        sandesh_type->is_sandesh_dynamic_uve() ||
         sandesh_type->is_sandesh_alarm()) {
       indent(out) << "if table is not None:" << endl;
       indent_up();
@@ -1702,7 +1693,6 @@ void t_py_generator::generate_py_sandesh_reader(ofstream& out,
 
 void t_py_generator::generate_py_sandesh_writer(ofstream& out,
                                                 t_sandesh* tsandesh) {
-  string name = tsandesh->get_name();
   const vector<t_field*>& fields = tsandesh->get_sorted_members();
   vector<t_field*>::const_iterator f_iter;
 
@@ -1723,7 +1713,7 @@ void t_py_generator::generate_py_sandesh_writer(ofstream& out,
   indent_down();
 
   indent(out) <<
-    "if oprot.writeSandeshBegin('" << name << "') < 0: return -1" << endl;
+    "if oprot.writeSandeshBegin(self.__class__.__name__) < 0: return -1" << endl;
   for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) {
     // Write field header
     indent(out) <<
@@ -1923,7 +1913,7 @@ void t_py_generator::generate_py_sandesh_request_list(ofstream& out) {
   out << "_SANDESH_REQUEST_LIST = [" << endl;
   std::vector<std::string>::const_iterator it;
   for (it = sandesh_req_list_.begin(); it != sandesh_req_list_.end(); ++it) {
-    indent(out) << "'" << *it << "'," << endl;
+    indent(out) << *it << "," << endl;
   }
   out << "]" << endl;
 }
@@ -2195,8 +2185,8 @@ void t_py_generator::generate_py_sandesh_logger(ofstream &out,
           << "self._level])" << endl;
         indent(out) << log_str << ".write(']: ')" << endl;
       }
-      indent(out) << log_str << ".write('" << tsandesh->get_name() <<
-        ": ')                                                                                                                                                                         " << endl;
+      indent(out) << log_str << ".write(self.__class__.__name__ + " <<
+        "': ')" << endl;
     }
     indent(out) << 
       "if self." << (*f_iter)->get_name() << " is not None:" << endl;
@@ -2363,55 +2353,32 @@ void t_py_generator::generate_py_sandesh_uve(std::ofstream& out,
   const t_base_type *sandesh_type = (t_base_type *)tsandesh->get_type();
   if (sandesh_type->is_sandesh_uve()) {
     // Update sandesh UVE list
-    sandesh_uve_list_.push_back(tsandesh->get_name());
-    // Update sandesh UVE data list
-    sandesh_uve_data_list_.push_back(ts->get_name());
+    sandesh_uve_list_.push_back(boost::make_tuple(tsandesh->get_name(),
+                                                  ts->get_name()));
   } else if (sandesh_type->is_sandesh_alarm()) {
     // Update sandesh Alarm list
-    sandesh_alarm_list_.push_back(tsandesh->get_name());
-    // Update sandesh Alarm data list
-    sandesh_alarm_data_list_.push_back(ts->get_name());
+    sandesh_alarm_list_.push_back(boost::make_tuple(tsandesh->get_name(),
+                                                    ts->get_name()));
   }
 }
 
 void t_py_generator::generate_py_sandesh_uve_list(ofstream& out) {
   out << endl << endl;
   out << "_SANDESH_UVE_LIST = [" << endl;
-  std::vector<std::string>::const_iterator it;
+  std::vector<boost::tuple<std::string, std::string> >::const_iterator it;
   for (it = sandesh_uve_list_.begin(); it != sandesh_uve_list_.end(); ++it) {
-    indent(out) << "'" << *it << "'," << endl;
-  }
-  out << "]" << endl;
-}
-
-void t_py_generator::generate_py_sandesh_uve_data_list(ofstream& out) {
-  out << endl << endl;
-  out << "_SANDESH_UVE_DATA_LIST = [" << endl;
-  std::vector<std::string>::const_iterator it;
-  for (it = sandesh_uve_data_list_.begin(); it != sandesh_uve_data_list_.end(); ++it) {
-    indent(out) << "'" << *it << "'," << endl;
+    indent(out) << "(" << it->get<0>() << ", " << it->get<1>() << ")," << endl;
   }
   out << "]" << endl;
 }
 
 void t_py_generator::generate_py_sandesh_alarm_list(ofstream& out) {
   out << endl << endl;
   out << "_SANDESH_ALARM_LIST = [" << endl;
-  std::vector<std::string>::const_iterator it;
+  std::vector<boost::tuple<std::string, std::string> >::const_iterator it;
   for (it = sandesh_alarm_list_.begin(); it != sandesh_alarm_list_.end();
        ++it) {
-    indent(out) << "'" << *it << "'," << endl;
-  }
-  out << "]" << endl;
-}
-
-void t_py_generator::generate_py_sandesh_alarm_data_list(ofstream& out) {
-  out << endl << endl;
-  out << "_SANDESH_ALARM_DATA_LIST = [" << endl;
-  std::vector<std::string>::const_iterator it;
-  for (it = sandesh_alarm_data_list_.begin();
-       it != sandesh_alarm_data_list_.end(); ++it) {
-    indent(out) << "'" << *it << "'," << endl;
+    indent(out) << "(" << it->get<0>() << ", " << it->get<1>() << ")," << endl;
   }
   out << "]" << endl;
 }
@@ -4098,6 +4065,8 @@ string t_py_generator::type_to_enum(t_type* type) {
       return "TTytpe.STRING";
     case t_base_type::TYPE_SANDESH_UVE:
       return "TTytpe.STRING";
+    case t_base_type::TYPE_SANDESH_DYNAMIC_UVE:
+      return "TTytpe.STRING";
     case t_base_type::TYPE_SANDESH_ALARM:
       return "TTytpe.STRING";
     case t_base_type::TYPE_SANDESH_OBJECT:

compiler/globals.h

@@ -79,6 +79,7 @@ extern t_type* g_type_sandesh_trace;
 extern t_type* g_type_sandesh_trace_object;
 extern t_type* g_type_sandesh_buffer;
 extern t_type* g_type_sandesh_uve;
+extern t_type* g_type_sandesh_dynamic_uve;
 extern t_type* g_type_sandesh_alarm;
 extern t_type* g_type_sandesh_object;
 extern t_type* g_type_sandesh_flow;

compiler/main.cc

@@ -89,6 +89,7 @@ t_type* g_type_sandesh_trace;
 t_type* g_type_sandesh_trace_object;
 t_type* g_type_sandesh_buffer;
 t_type* g_type_sandesh_uve;
+t_type* g_type_sandesh_dynamic_uve;
 t_type* g_type_sandesh_alarm;
 t_type* g_type_sandesh_object;
 t_type* g_type_sandesh_flow;
@@ -1386,6 +1387,7 @@ int main(int argc, char** argv) {
   g_type_sandesh_trace_object = new t_base_type("traceobject", t_base_type::TYPE_SANDESH_TRACE_OBJECT);
   g_type_sandesh_buffer   = new t_base_type("buffer",   t_base_type::TYPE_SANDESH_BUFFER);
   g_type_sandesh_uve      = new t_base_type("uve",      t_base_type::TYPE_SANDESH_UVE);
+  g_type_sandesh_dynamic_uve = new t_base_type("dynamicuve", t_base_type::TYPE_SANDESH_DYNAMIC_UVE);
   g_type_sandesh_alarm    = new t_base_type("alarm",    t_base_type::TYPE_SANDESH_ALARM);
   g_type_sandesh_object   = new t_base_type("object",   t_base_type::TYPE_SANDESH_OBJECT);
   g_type_sandesh_flow     = new t_base_type("flow",     t_base_type::TYPE_SANDESH_FLOW);
@@ -1429,6 +1431,7 @@ int main(int argc, char** argv) {
   delete g_type_sandesh_trace_object;
   delete g_type_sandesh_buffer;
   delete g_type_sandesh_uve;
+  delete g_type_sandesh_dynamic_uve;
   delete g_type_sandesh_alarm;
   delete g_type_sandesh_object;
   delete g_type_sandesh_flow;

compiler/parse/t_base_type.h

@@ -45,6 +45,7 @@ class t_base_type : public t_type {
     TYPE_SANDESH_TRACE_OBJECT,
     TYPE_SANDESH_BUFFER,
     TYPE_SANDESH_UVE,
+    TYPE_SANDESH_DYNAMIC_UVE,
     TYPE_SANDESH_ALARM,
     TYPE_SANDESH_OBJECT,
     TYPE_SANDESH_FLOW,
@@ -124,6 +125,10 @@ class t_base_type : public t_type {
     return base_ == TYPE_SANDESH_UVE;
   }
 
+  bool is_sandesh_dynamic_uve() const {
+    return base_ == TYPE_SANDESH_DYNAMIC_UVE;
+  }
+
   bool is_sandesh_alarm() const {
     return base_ == TYPE_SANDESH_ALARM;
   }
@@ -222,6 +227,7 @@ class t_base_type : public t_type {
       case TYPE_SANDESH_TRACE_OBJECT : return "traceobject"; break;
       case TYPE_SANDESH_BUFFER   : return      "buffer"; break;
       case TYPE_SANDESH_UVE      : return         "uve"; break;
+      case TYPE_SANDESH_DYNAMIC_UVE  : return     "dynamicuve"; break;
       case TYPE_SANDESH_ALARM    : return       "alarm"; break;
       case TYPE_SANDESH_OBJECT   : return      "object"; break;
       case TYPE_SANDESH_FLOW     : return        "flow"; break;

compiler/parse/t_scope.h

@@ -172,6 +172,8 @@ class t_scope {
               break;
             case t_base_type::TYPE_SANDESH_UVE:
               break;
+            case t_base_type::TYPE_SANDESH_DYNAMIC_UVE:
+              break;
             case t_base_type::TYPE_SANDESH_OBJECT:
               break;
             case t_base_type::TYPE_SANDESH_FLOW:

compiler/sandeshl.ll

@@ -186,6 +186,7 @@ format_str    (%([1-9]$)?([+-]?[ -#0])?([1-9]+)?(\.[1-9]+)?(hh|h|l|ll|L|z|j|t)?(
 "flowlog"            { return tok_flow;                 } 
 "buffer"             { return tok_buffer;               }
 "uve"                { return tok_uve;                  }
+"dynamicuve"         { return tok_dynamic_uve;          }
 "alarm"              { return tok_alarm;                }
 "sandesh"            { return tok_sandesh;              }
 "staticfunction"     { return tok_staticfunction;       }

compiler/sandeshy.yy

@@ -170,6 +170,7 @@ const int struct_is_union = 1;
  %token tok_flow
  %token tok_buffer
  %token tok_uve
+ %token tok_dynamic_uve
  %token tok_trace_object
  %token tok_alarm
 
@@ -895,6 +896,11 @@ SandeshType:
     {
       $$ = g_type_sandesh_uve;
     }
+|
+  tok_dynamic_uve
+    {
+      $$ = g_type_sandesh_dynamic_uve;
+    }
 |
   tok_object
     {

library/common/sandesh_dynamic_uve.sandesh

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
+ */
+
+/*
+ *  sandesh_dynamic_uve.sandesh
+ *
+ *  Sandesh Dynamic UVE definition
+ */
+
+
+struct DynamicElement {
+    1: string attribute
+    2: string value
+}
+
+struct DynamicObject {
+    1: string  name (key="")
+    2: list<DynamicElement> elements
+    3: optional bool deleted
+}
+
+dynamicuve sandesh DynamicObjectUVE {
+    1: DynamicObject  data
+}

library/python/SConscript

@@ -114,6 +114,8 @@ sandesh_hdr_pkg = env.SandeshGenPy('#tools/sandesh/library/common/sandesh.sandes
 sandesh_ctrl_pkg = env.SandeshGenPy('#tools/sandesh/library/common/sandesh_ctrl.sandesh', 'pysandesh/')
 # Auto-generated Sandesh UVE package
 sandesh_uve_pkg = env.SandeshGenPy('#tools/sandesh/library/common/sandesh_uve.sandesh', 'pysandesh/')
+# Auto-generated Sandesh Dynamic UVE package
+sandesh_dynamic_uve_pkg = env.SandeshGenPy('#tools/sandesh/library/common/sandesh_dynamic_uve.sandesh', 'pysandesh/')
 # Auto-generated Sandesh Alarm package
 sandesh_alarm_pkg = env.SandeshGenPy('#tools/sandesh/library/common/sandesh_alarm.sandesh', 'pysandesh/')
 # Auto-generated Sandesh Trace package
@@ -127,7 +129,7 @@ process_info_pkg = env.SandeshGenPy('#controller/src/base/sandesh/process_info.s
 sdist_depends = [local_modules_rules, web_srv_rules,
                  pysandesh_modules_rules, transport_modules_rules,
                  protocol_modules_rules, sandesh_hdr_pkg,
-                 sandesh_ctrl_pkg, sandesh_uve_pkg,
+                 sandesh_ctrl_pkg, sandesh_uve_pkg, sandesh_dynamic_uve_pkg,
                  sandesh_alarm_pkg, sandesh_trace_pkg,
                  fysom_sources_rules,
                  io_pkg, process_info_pkg]