Fixes: contrail-status should indicate ifmap as down if database is e…

…mpty

In the health-check for IFMAP, adding a check to check whether the
default global system config is there.

If not publish the IFMAP server as Down with the discovery server
with relevant reason and also bring the ConnectionStatus for IFMAP
in API server to down so that the contrail-status reports that
the IFMAP server connection is not functional.

Change-Id: Ia5c1589055f7795ca4a145403c583588df94cec5
Closes-Bug: 1546689

src/config/api-server/vnc_cfg_api_server.py

@@ -3185,15 +3185,16 @@ def publish_self_to_discovery(self):
             self.api_server_task = self._disc.publish(
                 API_SERVER_DISCOVERY_SERVICE_NAME, data)
 
-    def publish_ifmap_to_discovery(self):
+    def publish_ifmap_to_discovery(self, state = 'up', msg = ''):
         # publish ifmap server
         data = {
             'ip-address': self._args.ifmap_server_ip,
             'port': self._args.ifmap_server_port,
         }
         if self._disc:
             self.ifmap_task = self._disc.publish(
-                IFMAP_SERVER_DISCOVERY_SERVICE_NAME, data)
+                                  IFMAP_SERVER_DISCOVERY_SERVICE_NAME,
+                                  data, state, msg)
     # end publish_ifmap_to_discovery
 
     def un_publish_self_to_discovery(self):

src/config/api-server/vnc_cfg_ifmap.py

@@ -23,7 +23,7 @@
 from cfgm_common.uve.vnc_api.ttypes import *
 from cfgm_common import ignore_exceptions
 from cfgm_common.ifmap.client import client
-from cfgm_common.ifmap.request import NewSessionRequest, PublishRequest
+from cfgm_common.ifmap.request import NewSessionRequest, PublishRequest, SearchRequest
 from cfgm_common.ifmap.id import Identity
 from cfgm_common.ifmap.operations import PublishUpdateOperation,\
     PublishDeleteOperation
@@ -117,6 +117,7 @@ def __init__(self, db_client_mgr, ifmap_srv_ip, ifmap_srv_port,
             name = 'IfMap', status = ConnectionStatus.INIT, message = '',
             server_addrs = ["%s:%s" % (ifmap_srv_ip, ifmap_srv_port)])
         self._conn_state = ConnectionStatus.INIT
+        self._is_ifmap_up = False
 
         self.reset()
 
@@ -454,6 +455,7 @@ def _publish(requests, traces, publish_discovery=False):
                               error_msg=msg)
             if publish_discovery and ok:
                 self._get_api_server().publish_ifmap_to_discovery()
+                self._is_ifmap_up = True
         # end _publish
 
         while True:
@@ -545,6 +547,7 @@ def _publish_to_ifmap(self, oper_body):
 
                 self.reset()
                 self._get_api_server().un_publish_ifmap_to_discovery()
+                self._is_ifmap_up = False
                 # this will block till connection is re-established
                 self._init_conn()
                 self._publish_config_root()
@@ -705,9 +708,69 @@ def _health_checker(self):
                         ns_prefix='contrail', elements='')
                 request_str = self._build_request('healthcheck', 'self', [meta])
                 self._publish_to_ifmap_enqueue('update', request_str, do_trace=False)
+
+                # Confirm the existence of the following default global entities in IFMAP.
+                search_list = ['contrail:global-system-config:default-global-system-config']
+                mapclient = self._mapclient
+                srch_params = {}
+                srch_params['max-depth'] = '0'
+                srch_params['max-size'] = '500000'
+                srch_params['result-filter'] = None
+
+                for each_entity in search_list:
+                    start_id = str(
+                        Identity(name = each_entity,
+                        type='other', other_type='extended'))
+
+                    srch_req = SearchRequest(mapclient.get_session_id(), start_id,
+                                             search_parameters = srch_params)
+
+                    result = mapclient.call('search', srch_req)
+                    soap_doc = etree.fromstring(result)
+                    result_items = soap_doc.xpath(
+                        '/env:Envelope/env:Body/ifmap:response/searchResult/resultItem',
+                        namespaces=self._NAMESPACES)
+
+                    # The above IFMAP search call does not return the success of
+                    # the search query. To determine the success of the query
+                    # we verify whether the response has a 'metadata' child
+                    # and that in turn has "perms" sub children, then we can assume
+                    # that the query has succeeded. If not, we raise an 
+                    # exception that the IFMAP does not contain basic Contrail
+                    # entities.
+                    for each_result in result_items:
+                        result_children = each_result.getchildren()
+                        metadata = [x for x in result_children if x.tag == 'metadata']
+                        if len(metadata) != 0:
+                            perms = [x for x in metadata[0] if "perms" in x.tag]
+                            if len(perms) != 0:
+                                continue
+                        raise Exception("%s not found in IFMAP DB" % each_entity)
+
+                # If we had unpublished the IFMAP server to discovery server earlier
+                # publish it back now since it is valid now.
+                if not self._is_ifmap_up:
+                    self._get_api_server().publish_ifmap_to_discovery('up', '')
+                    self._is_ifmap_up = True
+                    ConnectionState.update(conn_type = ConnType.IFMAP,
+                                           name = 'IfMap',
+                                           status = ConnectionStatus.UP,
+                                           message = '',
+                                           server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
+                                                                      self._ifmap_srv_port)])
             except Exception as e:
-                log_str = 'Healthcheck to IFMAP failed: %s' %(str(e))
+                log_str = 'IFMAP Healthcheck failed: %s' %(str(e))
                 self.config_log(log_str, level=SandeshLevel.SYS_ERR)
+                if self._is_ifmap_up:
+                    self._get_api_server().publish_ifmap_to_discovery('down',
+                                                   'IFMAP DB - Invalid state')
+                    self._is_ifmap_up = False
+                    ConnectionState.update(conn_type = ConnType.IFMAP,
+                                           name = 'IfMap',
+                                           status = ConnectionStatus.DOWN,
+                                           message = 'Invalid IFMAP DB State',
+                                           server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
+                                                                      self._ifmap_srv_port)])
             finally:
                 gevent.sleep(
                     self._get_api_server().get_ifmap_health_check_interval())

src/discovery/client.py

@@ -295,12 +295,14 @@ def exportJson(self, o):
                               for k, v in obj.__dict__.iteritems()))
         return obj_json
 
-    def _publish_int(self, service, data):
+    def _publish_int(self, service, data, oper_state = 'up', msg = ''):
         self.syslog('Publish service "%s", data "%s"' % (service, data))
         payload = {
-            service        : data,
-            'service-type' : service,
-            'remote-addr'  : self._remote_addr
+            service             : data,
+            'service-type'      : service,
+            'remote-addr'       : self._remote_addr,
+            'oper-state'        : oper_state,
+            'oper-state-reason' : msg
         }
         emsg = None
         cookie = None
@@ -366,9 +368,9 @@ def publish_obj(self, obj):
     # end publish
 
     # API publish service and data
-    def publish(self, service, data):
+    def publish(self, service, data, state = 'up', msg = ''):
         self.pub_data[service] = data
-        self._publish_int(service, data)
+        self._publish_int(service, data, state, msg)
         return self.hbtask
     # end