New feature in api server to send every api requests statistics to an…

…alytics

including response_time, object_type, response status. Also enabling the contrail
log query of vnc api stats through webui
Closes-Bug: 1481116
Raise user exception when the UUID is not of the standard format.
Closes-Bug: 1486290

Change-Id: Ic1ab3e3fb9dc4620b4610456d6f10db4aa52e51f
(cherry picked from commit ebe1ecf)
(cherry picked from commit a81b02e)
(cherry picked from commit d5bc2e3)

src/analytics/viz.sandesh

@@ -1274,6 +1274,24 @@ const list<stat_table> _STAT_TABLES  = [
           { 'name' : 'if_stats.out_bw_usage', 'datatype' : 'int', 'index' : false }
         ]
     }
+    {
+      'display_name' : 'Api Server Statistics',
+      'stat_type' : 'VncApiStatsLog',
+      'stat_attr' : 'api_stats',
+      'obj_table' : CONFIG_OBJECT_TABLE,
+      'attributes' : [
+          { 'name' : 'api_stats.operation_type', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.user', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.useragent', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.remote_ip', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.domain_name', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.project_name', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.object_type', 'datatype' : 'string', 'index' : true },
+          { 'name' : 'api_stats.response_time_in_usec', 'datatype' : 'double', 'index' : true },
+          { 'name' : 'api_stats.response_size', 'datatype' : 'int', 'index' : true },
+          { 'name' : 'api_stats.response_code', 'datatype' : 'int', 'index' : true }
+        ]
+    }
 ]
 
 const list<query_table> _TABLES = [

src/config/api-server/vnc_cfg_api_server.py

@@ -88,6 +88,7 @@
 import vnc_auth_keystone
 import vnc_perms
 from cfgm_common import vnc_cpu_info
+from cfgm_common.vnc_api_stats import log_api_stats
 
 from pysandesh.sandesh_base import *
 from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
@@ -287,6 +288,7 @@ def _validate_perms_in_request(self, resource_class, obj_type, obj_dict):
                     bottle.abort(code, err_msg)
     # end _validate_perms_in_request
 
+    @log_api_stats
     def http_resource_create(self, resource_type):
         r_class = self.get_resource_class(resource_type)
         obj_type = resource_type.replace('-', '_')
@@ -435,6 +437,7 @@ def http_resource_create(self, resource_type):
         return {resource_type: rsp_body}
     # end http_resource_create
 
+    @log_api_stats
     def http_resource_read(self, resource_type, id):
         r_class = self.get_resource_class(resource_type)
         obj_type = resource_type.replace('-', '_')
@@ -528,6 +531,7 @@ def http_resource_read(self, resource_type, id):
         return {resource_type: rsp_body}
     # end http_resource_read
 
+    @log_api_stats
     def http_resource_update(self, resource_type, id):
         r_class = self.get_resource_class(resource_type)
         obj_type = resource_type.replace('-', '_')
@@ -625,6 +629,7 @@ def http_resource_update(self, resource_type, id):
         return {resource_type: rsp_body}
     # end http_resource_update
 
+    @log_api_stats
     def http_resource_delete(self, resource_type, id):
         r_class = self.get_resource_class(resource_type)
         obj_type = resource_type.replace('-', '_')
@@ -765,6 +770,7 @@ def http_resource_delete(self, resource_type, id):
             self.config_log(err_msg, level=SandeshLevel.SYS_NOTICE)
     # end http_resource_delete
 
+    @log_api_stats
     def http_resource_list(self, resource_type):
         r_class = self.get_resource_class(resource_type)
         obj_type = resource_type.replace('-', '_')
@@ -2176,6 +2182,8 @@ def _http_post_common(self, request, obj_type, obj_dict):
         apiConfig.operation = 'post'
         apiConfig.body = str(request.json)
         if uuid_in_req:
+            if uuid_in_req != str(uuid.UUID(uuid_in_req)):
+                bottle.abort(400, 'Invalid UUID format: ' + uuid_in_req)
             try:
                 fq_name = self._db_conn.uuid_to_fq_name(uuid_in_req)
                 bottle.abort(

src/config/common/SConscript

@@ -37,6 +37,7 @@ local_sources = [
                   'vnc_kombu.py',
                   'vnc_db.py',
                   'dependency_tracker.py',
+                  'vnc_api_stats.py',
                  ]
 local_sources_rules = []
 for file in local_sources:

src/config/common/tests/test_common.py

@@ -22,7 +22,8 @@
 import kombu
 import discoveryclient.client as disc_client
 import cfgm_common.zkclient
-from cfgm_common.uve.vnc_api.ttypes import VncApiConfigLog, VncApiError
+from cfgm_common.uve.vnc_api.ttypes import (VncApiConfigLog, VncApiError,
+    VncApiStatsLog)
 from cfgm_common import imid
 
 from test_utils import *
@@ -239,6 +240,7 @@ def setup_common_flexmock():
     flexmock(kombu.Producer, __new__=FakeKombu.Producer)
 
     flexmock(VncApiConfigLog, __new__=FakeApiConfigLog)
+    flexmock(VncApiStatsLog, __new__=FakeVncApiStatsLog)
 #end setup_common_flexmock
 
 @contextlib.contextmanager

src/config/common/tests/test_utils.py

@@ -1027,3 +1027,20 @@ def edit_config(self, target, config, test_option, default_operation):
 netconf_managers = {}
 def fake_netconf_connect(host, *args, **kwargs):
     return netconf_managers.setdefault(host, FakeNetconfManager(args, kwargs))
+
+
+class FakeVncApiStatsLog(object):
+    _all_logs = []
+    send = stub
+    def __init__(self, *args, **kwargs):
+        FakeVncApiStatsLog._all_logs.append(kwargs['api_stats'])
+
+    @classmethod
+    def _print(cls):
+        for log in cls._all_logs:
+            x = copy.deepcopy(log.__dict__)
+            #body = x.pop('body')
+            #pprint(json.loads(body))
+            pprint(x)
+            print "\n"
+# class FakeVncApiStatsLog

src/config/common/vnc_api_stats.py

@@ -0,0 +1,70 @@
+#
+# Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
+#
+
+import bottle
+from datetime import datetime
+
+from uve.vnc_api.ttypes import VncApiStats, VncApiStatsLog
+from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
+
+
+def log_api_stats(func):
+    def wrapper(api_server_obj, resource_type, *args, **kwargs):
+        try:
+            statistics = VncApiStatistics(
+                obj_type=resource_type.replace('-', '_'))
+            response = func(api_server_obj, resource_type, *args, **kwargs)
+            statistics.response_size = len(str(response))
+            statistics.response_code = bottle.response.status_code
+            return response
+        except Exception as err_response:
+            if isinstance(err_response, bottle.HTTPError):
+                statistics.response_size = len(err_response.body)
+                statistics.response_code = err_response.status_code
+            else:
+                statistics.response_size = 0
+                # 520 Unknown Error
+                statistics.response_code = 520
+            raise
+        finally:
+            # Collect api stats and send to analytics
+            statistics.collect()
+            statistics.sendwith(api_server_obj._sandesh)
+    return wrapper
+
+
+class VncApiStatistics(object):
+    def __init__(self, obj_type):
+        self.obj_type = obj_type
+        self.response_size = 0
+        self.time_start = datetime.now()
+
+    def collect(self):
+        self.time_finish = datetime.now()
+        domain_name = bottle.request.headers.get('X-Domain-Name', 'None')
+        if domain_name.lower() == 'none':
+            domain_name = 'default-domain'
+        project_name = bottle.request.headers.get('X-Project-Name', 'None')
+        if project_name.lower() == 'none':
+            project_name = 'default-project'
+
+        # Create api stats object
+        self.api_stats = VncApiStats(
+            object_type=self.obj_type,
+            operation_type=bottle.request.method,
+            user=bottle.request.headers.get('X-User-Name'),
+            useragent=bottle.request.headers.get('X-Contrail-Useragent',
+                bottle.request.headers.get('User-Agent')),
+            remote_ip=bottle.request.headers.get('Host'),
+            domain_name=domain_name,
+            project_name=project_name,
+            response_time_in_usec=(self.time_finish -
+                self.time_start).total_seconds() * 1000000,
+            response_size=self.response_size,
+            response_code=self.response_code,
+        )
+
+    def sendwith(self, sandesh):
+        stats_log = VncApiStatsLog(api_stats=self.api_stats, sandesh=sandesh)
+        stats_log.send(sandesh=sandesh)

src/config/uve/vnc_api.sandesh

@@ -35,3 +35,20 @@ objectlog sandesh VncApiConfigLog {
 systemlog sandesh VncApiError {
     1: string api_error_msg;
 }
+
+struct VncApiStats {
+    1: string operation_type // (GET, PUT, POST, DELETE)
+    2: string user
+    3: string useragent
+    4: string remote_ip
+    5: string domain_name
+    6: string project_name
+    7: string object_type (key="ConfigObjectTable")
+    8: double response_time_in_usec
+    9: u64 response_size
+    10: u64 response_code
+}
+
+objectlog sandesh VncApiStatsLog {
+    1: VncApiStats api_stats (tags=".operation_type, .user, .useragent, .remote_ip, .domain_name, .project_name, .object_type, .response_time_in_usec, .response_size, .response_code")
+}