Use map instead of list for reporting cpu/mem usage in NodeStatus

The new DerivedStats infra supports maps and not lists. - Updating new NodeStatus structure to use maps instead of lists. - Corresponding changes on client. - Marked the deprecated fields, these will be deleted after UI moves to new structures. List of contrail-stats test commands attached to the bug. Change-Id: I3e4f2f38d11dff149cacda98162303917281c4cf Closes-Bug:1601909 (cherry picked from commit 3f025bb)
Juniper · Jul 19, 2016 · e3df29f · e3df29f
1 parent 4d5bdd4
commit e3df29f
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 30 deletions.
diff --git a/src/analytics/viz.sandesh b/src/analytics/viz.sandesh
@@ -1906,7 +1906,7 @@ const list<stat_table> _STAT_TABLES  = [
       'obj_table' : COLLECTOR_INFO_TABLE,
       'attributes': [
           { 'name' : 'disk_usage_info.partition_type', 'datatype' : 'string', 'index' : false },
-          { 'name' : 'disk_usage_info.partition_name', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'disk_usage_info.__key', 'datatype' : 'string', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_used_1k', 'datatype' : 'int', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_available_1k', 'datatype' : 'int', 'index' : false },
         ]
@@ -1918,7 +1918,7 @@ const list<stat_table> _STAT_TABLES  = [
       'obj_table' : DATABASE_INFO_TABLE,
       'attributes': [
           { 'name' : 'disk_usage_info.partition_type', 'datatype' : 'string', 'index' : false },
-          { 'name' : 'disk_usage_info.partition_name', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'disk_usage_info.__key', 'datatype' : 'string', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_used_1k', 'datatype' : 'int', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_available_1k', 'datatype' : 'int', 'index' : false },
         ]
@@ -1930,7 +1930,7 @@ const list<stat_table> _STAT_TABLES  = [
       'obj_table' : CONFIG_NODE_TABLE,
       'attributes': [
           { 'name' : 'disk_usage_info.partition_type', 'datatype' : 'string', 'index' : false },
-          { 'name' : 'disk_usage_info.partition_name', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'disk_usage_info.__key', 'datatype' : 'string', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_used_1k', 'datatype' : 'int', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_available_1k', 'datatype' : 'int', 'index' : false },
         ]
@@ -1942,7 +1942,7 @@ const list<stat_table> _STAT_TABLES  = [
       'obj_table' : BGP_ROUTER_TABLE,
       'attributes': [
           { 'name' : 'disk_usage_info.partition_type', 'datatype' : 'string', 'index' : false },
-          { 'name' : 'disk_usage_info.partition_name', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'disk_usage_info.__key', 'datatype' : 'string', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_used_1k', 'datatype' : 'int', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_available_1k', 'datatype' : 'int', 'index' : false },
         ]
@@ -1954,7 +1954,7 @@ const list<stat_table> _STAT_TABLES  = [
       'obj_table' : VROUTER_TABLE,
       'attributes': [
           { 'name' : 'disk_usage_info.partition_type', 'datatype' : 'string', 'index' : false },
-          { 'name' : 'disk_usage_info.partition_name', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'disk_usage_info.__key', 'datatype' : 'string', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_used_1k', 'datatype' : 'int', 'index' : false },
           { 'name' : 'disk_usage_info.partition_space_available_1k', 'datatype' : 'int', 'index' : false },
         ]
@@ -1965,8 +1965,7 @@ const list<stat_table> _STAT_TABLES  = [
       'stat_attr' : 'process_mem_cpu_usage',
       'obj_table' : COLLECTOR_INFO_TABLE,
       'attributes': [
-          { 'name' : 'process_mem_cpu_usage.module_id', 'datatype' : 'string', 'index' : true, 'suffixes': ['process_mem_cpu_usage.inst_id']},
-          { 'name' : 'process_mem_cpu_usage.inst_id', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'process_mem_cpu_usage.__key', 'datatype' : 'string', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_virt', 'datatype' : 'int', 'index' : false },
           { 'name' : 'process_mem_cpu_usage.cpu_share', 'datatype' : 'double', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_res', 'datatype' : 'int', 'index' : false },
@@ -1978,8 +1977,7 @@ const list<stat_table> _STAT_TABLES  = [
       'stat_attr' : 'process_mem_cpu_usage',
       'obj_table' : DATABASE_INFO_TABLE,
       'attributes': [
-          { 'name' : 'process_mem_cpu_usage.module_id', 'datatype' : 'string', 'index' : true, 'suffixes': ['process_mem_cpu_usage.inst_id']},
-          { 'name' : 'process_mem_cpu_usage.inst_id', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'process_mem_cpu_usage.__key', 'datatype' : 'string', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_virt', 'datatype' : 'int', 'index' : false },
           { 'name' : 'process_mem_cpu_usage.cpu_share', 'datatype' : 'double', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_res', 'datatype' : 'int', 'index' : false },
@@ -1991,8 +1989,7 @@ const list<stat_table> _STAT_TABLES  = [
       'stat_attr' : 'process_mem_cpu_usage',
       'obj_table' : CONFIG_NODE_TABLE,
       'attributes': [
-          { 'name' : 'process_mem_cpu_usage.module_id', 'datatype' : 'string', 'index' : true, 'suffixes': ['process_mem_cpu_usage.inst_id']},
-          { 'name' : 'process_mem_cpu_usage.inst_id', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'process_mem_cpu_usage.__key', 'datatype' : 'string', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_virt', 'datatype' : 'int', 'index' : false },
           { 'name' : 'process_mem_cpu_usage.cpu_share', 'datatype' : 'double', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_res', 'datatype' : 'int', 'index' : false },
@@ -2004,8 +2001,7 @@ const list<stat_table> _STAT_TABLES  = [
       'stat_attr' : 'process_mem_cpu_usage',
       'obj_table' : BGP_ROUTER_TABLE,
       'attributes': [
-          { 'name' : 'process_mem_cpu_usage.module_id', 'datatype' : 'string', 'index' : true, 'suffixes': ['process_mem_cpu_usage.inst_id']},
-          { 'name' : 'process_mem_cpu_usage.inst_id', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'process_mem_cpu_usage.__key', 'datatype' : 'string', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_virt', 'datatype' : 'int', 'index' : false },
           { 'name' : 'process_mem_cpu_usage.cpu_share', 'datatype' : 'double', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_res', 'datatype' : 'int', 'index' : false },
@@ -2017,8 +2013,7 @@ const list<stat_table> _STAT_TABLES  = [
       'stat_attr' : 'process_mem_cpu_usage',
       'obj_table' : VROUTER_TABLE,
       'attributes': [
-          { 'name' : 'process_mem_cpu_usage.module_id', 'datatype' : 'string', 'index' : true, 'suffixes': ['process_mem_cpu_usage.inst_id']},
-          { 'name' : 'process_mem_cpu_usage.inst_id', 'datatype' : 'string', 'index' : false },
+          { 'name' : 'process_mem_cpu_usage.__key', 'datatype' : 'string', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_virt', 'datatype' : 'int', 'index' : false },
           { 'name' : 'process_mem_cpu_usage.cpu_share', 'datatype' : 'double', 'index' : true},
           { 'name' : 'process_mem_cpu_usage.mem_res', 'datatype' : 'int', 'index' : false },

diff --git a/src/base/sandesh/cpuinfo.sandesh b/src/base/sandesh/cpuinfo.sandesh
@@ -52,8 +52,8 @@ response sandesh CpuLoadInfoResp {
  * this is used to store cpu and memory statistics on a per process basis
  */
 struct ProcessCpuInfo {
-    1: string                              module_id
-    2: string                              inst_id
+    1: string                              module_id        // deprecated
+    2: string                              inst_id          // deprecated
     3: u32                                 mem_virt
     4: double                              cpu_share
     5: u32                                 mem_res

diff --git a/src/base/sandesh/nodeinfo.sandesh b/src/base/sandesh/nodeinfo.sandesh
@@ -30,11 +30,13 @@ struct NodeStatus {
     4: optional list<process_info.ProcessStatus> process_status (aggtype="union")
     // Sent by node manager
     5: optional list<process_info.ProcessInfo> process_info (aggtype="union")
-    6: optional list<process_info.DiskPartitionUsageStats> disk_usage_info (tags="")
+    // map key is partition name
+    6: optional map<string,process_info.DiskPartitionUsageStats> disk_usage_info (tags="")
     7: optional string description
     8: optional list<string> all_core_file_list
     9: optional string build_info
-   10: optional list<cpuinfo.ProcessCpuInfo> process_mem_cpu_usage (tags=".module_id:.inst_id,.cpu_share")
+    // map key is <module_id>:<inst_id>
+   10: optional map<string,cpuinfo.ProcessCpuInfo> process_mem_cpu_usage (tags=".__key,.cpu_share")
    11: optional cpuinfo.SystemMemCpuUsage system_mem_cpu_usage(tags=".cpu_share")
    12: optional cpuinfo.SystemCpuInfo system_cpu_info
 }

diff --git a/src/base/sandesh/process_info.sandesh b/src/base/sandesh/process_info.sandesh
@@ -92,7 +92,7 @@ struct ProcessStatus {
  */
 struct DiskPartitionUsageStats {
     1: string                              partition_type
-    2: string                              partition_name
+    2: string                              partition_name       // deprecated
     3: u64                                 partition_space_used_1k
     4: u64                                 partition_space_available_1k
     5: u16                                 percentage_partition_space_used

diff --git a/src/nodemgr/common/event_manager.py b/src/nodemgr/common/event_manager.py
@@ -385,7 +385,7 @@ def get_system_mem_cpu_usage(self):
         return system_mem_cpu_usage_data.get_sys_mem_cpu_info()
 
     def get_all_processes_mem_cpu_usage(self):
-        process_mem_cpu_usage = []
+        process_mem_cpu_usage = {}
         for key in self.process_state_db:
             pstat = self.process_state_db[key]
             if (pstat.process_state == 'PROCESS_STATE_RUNNING'):
@@ -396,9 +396,8 @@ def get_all_processes_mem_cpu_usage(self):
                     sys.stderr.write("NoSuchProcess: process name:%s pid:%d\n"
                                      % (pstat.pname, pstat.pid))
                 else:
-                    process_mem_cpu.module_id = pstat.pname
-                    process_mem_cpu.inst_id = "0"   # ??
-                    process_mem_cpu_usage.append(process_mem_cpu)
+                    process_mem_cpu.__key = pstat.pname
+                    process_mem_cpu_usage[process_mem_cpu.__key] = process_mem_cpu
 
         # walk through all processes being monitored by nodemgr,
         # not spawned by supervisord
@@ -416,13 +415,12 @@ def get_all_processes_mem_cpu_usage(self):
                     sys.stderr.write("NoSuchProcess: process name:%s pid:%d\n"
                                      % (pname, pid))
                 else:
-                    process_mem_cpu.module_id = pname
-                    process_mem_cpu.inst_id = "0"
-                    process_mem_cpu_usage.append(process_mem_cpu)
+                    process_mem_cpu.__key = pname
+                    process_mem_cpu_usage[process_mem_cpu.__key] = process_mem_cpu
         return process_mem_cpu_usage
 
     def get_disk_usage(self):
-        disk_usage_info = []
+        disk_usage_info = {}
         partition = subprocess.Popen(
             "df -T -t ext2 -t ext3 -t ext4 -t xfs",
             shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -436,7 +434,7 @@ def get_disk_usage(self):
             disk_usage_stat = DiskPartitionUsageStats()
             try:
                 disk_usage_stat.partition_type = str(partition_type)
-                disk_usage_stat.partition_name = str(partition_name)
+                disk_usage_stat.__key = str(partition_name)
                 disk_usage_stat.partition_space_used_1k = \
                     int(partition_space_used_1k)
                 disk_usage_stat.partition_space_available_1k = \
@@ -450,7 +448,7 @@ def get_disk_usage(self):
             except ValueError:
                 sys.stderr.write("Failed to get local disk space usage" + "\n")
             else:
-                disk_usage_info.append(disk_usage_stat)
+                disk_usage_info[partition_name] = disk_usage_stat
         return disk_usage_info
     # end get_disk_usage