vnc_cfg_ifmap.py

#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#

"""
Layer that transforms VNC config objects to ifmap representation
"""
from cfgm_common.zkclient import ZookeeperClient, IndexAllocator
from gevent import ssl, monkey
monkey.patch_all()
import gevent
import gevent.event
from gevent.queue import Queue, Empty

import time
from pprint import pformat

from lxml import etree
import StringIO

import socket
from netaddr import IPNetwork, IPAddress
from context import get_request

from cfgm_common.uve.vnc_api.ttypes import *
from cfgm_common import ignore_exceptions
from cfgm_common.ifmap.client import client
from cfgm_common.ifmap.request import NewSessionRequest, PublishRequest,\
    EndSessionRequest
from cfgm_common.ifmap.id import Identity
from cfgm_common.ifmap.operations import PublishUpdateOperation,\
    PublishDeleteOperation
from cfgm_common.ifmap.response import newSessionResult
from cfgm_common.ifmap.metadata import Metadata
from cfgm_common.imid import escape, entity_is_present,\
    get_ifmap_id_from_fq_name, get_fq_name_from_ifmap_id, ifmap_wipe
from cfgm_common.exceptions import ResourceExhaustionError, ResourceExistsError
from cfgm_common.vnc_cassandra import VncCassandraClient
from vnc_rdbms import VncServerRDBMSClient
from cfgm_common.vnc_kombu import VncKombuClient
from cfgm_common.utils import cgitb_hook
from cfgm_common import vnc_greenlets
from cfgm_common import SGID_MIN_ALLOC

import copy
from cfgm_common import jsonutils as json
import uuid
import datetime
import pycassa
import pycassa.util
import pycassa.cassandra.ttypes
from pycassa.system_manager import *
from pycassa.util import *

import signal, os

from provision_defaults import *
from cfgm_common.exceptions import *
from vnc_quota import *
from pysandesh.connection_info import ConnectionState
from pysandesh.gen_py.process_info.ttypes import ConnectionStatus
from pysandesh.gen_py.process_info.ttypes import ConnectionType as ConnType
from pysandesh.gen_py.sandesh.ttypes import SandeshLevel
from sandesh_common.vns.constants import USERAGENT_KEYSPACE_NAME
from sandesh.traces.ttypes import DBRequestTrace, MessageBusNotifyTrace, \
    IfmapTrace

@ignore_exceptions
def get_trace_id():
    try:
        req_id = gevent.getcurrent().trace_request_id
    except Exception:
        req_id = 'req-%s' %(str(uuid.uuid4()))
        gevent.getcurrent().trace_request_id = req_id

    return req_id
# end get_trace_id

@ignore_exceptions
def trace_msg(trace_objs=[], trace_name='', sandesh_hdl=None, error_msg=None):
    for trace_obj in trace_objs:
        if error_msg:
            trace_obj.error = error_msg
        trace_obj.trace_msg(name=trace_name, sandesh=sandesh_hdl)
# end trace_msg

def build_idperms_ifmap_obj(prop_field, values):
    prop_xml = u'<uuid><uuid-mslong>'
    prop_xml += unicode(json.dumps(values[u'uuid'][u'uuid_mslong']))
    prop_xml += u'</uuid-mslong><uuid-lslong>'
    prop_xml += unicode(json.dumps(values[u'uuid'][u'uuid_lslong']))
    prop_xml += u'</uuid-lslong></uuid><enable>'
    prop_xml += unicode(json.dumps(values[u'enable']))
    prop_xml += u'</enable>'
    return prop_xml

class VncIfmapClient(object):

    # * Not all properties in an object needs to be published
    #   to IfMap.
    # * In some properties, not all fields are relevant
    #   to be publised to IfMap.
    # If the property is not relevant at all, define the property
    # with None. If it is partially relevant, then define the fn.
    # which would handcraft the generated xml for the object.
    IFMAP_PUBLISH_SKIP_LIST = {
        # Format - <prop_field> : None | <Handler_fn>
        u"perms2" : None,
        u"id_perms" : build_idperms_ifmap_obj
    }

    def handler(self, signum, frame):
        file = open("/tmp/api-server-ifmap-cache.txt", "w")
        file.write(pformat(self._id_to_metas))
        file.close()

    def __init__(self, db_client_mgr, ifmap_srv_ip, ifmap_srv_port,
                 uname, passwd, ssl_options):
        self._ifmap_srv_ip = ifmap_srv_ip
        self._ifmap_srv_port = ifmap_srv_port
        self._username = uname
        self._password = passwd
        self._ssl_options = ssl_options
        self._dequeue_greenlet = None
        self._CONTRAIL_XSD = "http://www.contrailsystems.com/vnc_cfg.xsd"
        self._IPERMS_NAME = "id-perms"
        self._NAMESPACES = {
            'env':   "http://www.w3.org/2003/05/soap-envelope",
            'ifmap':   "http://www.trustedcomputinggroup.org/2010/IFMAP/2",
            'meta':
            "http://www.trustedcomputinggroup.org/2010/IFMAP-METADATA/2",
            'contrail':   self._CONTRAIL_XSD
        }

        self._db_client_mgr = db_client_mgr
        self._sandesh = db_client_mgr._sandesh

        ConnectionState.update(conn_type = ConnType.IFMAP,
            name = 'IfMap', status = ConnectionStatus.INIT, message = '',
            server_addrs = ["%s:%s" % (ifmap_srv_ip, ifmap_srv_port)])
        self._conn_state = ConnectionStatus.INIT
        self._is_ifmap_up = False
        self._queue = Queue(self._get_api_server()._args.ifmap_queue_size)

        self.reset()

        # Set the signal handler
        signal.signal(signal.SIGUSR2, self.handler)

        self._init_conn()
        self._publish_config_root()
        self._health_checker_greenlet =\
               vnc_greenlets.VncGreenlet('VNC IfMap Health Checker',
                                         self._health_checker)
    # end __init__
    @classmethod
    def object_alloc(cls, obj_class, parent_res_type, fq_name):
        res_type = obj_class.resource_type
        my_fqn = ':'.join(fq_name)
        parent_fqn = ':'.join(fq_name[:-1])

        my_imid = 'contrail:%s:%s' %(res_type, my_fqn)
        if parent_fqn:
            if parent_res_type is None:
                err_msg = "Parent: %s type is none for: %s" % (parent_fqn,
                                                               my_fqn)
                return False, (409, err_msg)
            parent_imid = 'contrail:' + parent_res_type + ':' + parent_fqn
        else: # parent is config-root
            parent_imid = 'contrail:config-root:root'

        # Normalize/escape special chars
        my_imid = escape(my_imid)
        parent_imid = escape(parent_imid)

        return True, (my_imid, parent_imid)
    # end object_alloc

    def object_set(self, obj_type, my_imid, existing_metas, obj_dict):
        obj_class = self._db_client_mgr.get_resource_class(obj_type)
        update = {}

        # Properties Meta
        for prop_field in obj_class.prop_fields:
            field = obj_dict.get(prop_field)
            if field is None:
                continue
            # construct object of xsd-type and get its xml repr
            # e.g. virtual_network_properties
            prop_field_types = obj_class.prop_field_types[prop_field]
            is_simple = not prop_field_types['is_complex']
            prop_type = prop_field_types['xsd_type']
            # e.g. virtual-network-properties
            prop_meta = obj_class.prop_field_metas[prop_field]

            if prop_field in VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST:
                # Field not relevant, skip publishing to IfMap
                if not VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST[prop_field]:
                    continue
                # Call the handler fn to generate the relevant fields.
                if callable(VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST[prop_field]):
                    prop_xml = VncIfmapClient.IFMAP_PUBLISH_SKIP_LIST[prop_field](
                                        prop_field, field)
                    meta = Metadata(prop_meta, '',
                        {'ifmap-cardinality':'singleValue'}, ns_prefix='contrail',
                        elements=prop_xml)
                else:
                    log_str = '%s is marked for partial publish\
                               to Ifmap but handler not defined' %(
                                prop_field)
                    self.config_log(log_str, level=SandeshLevel.SYS_DEBUG)
                    continue
            elif is_simple:
                norm_str = escape(str(field))
                meta = Metadata(prop_meta, norm_str,
                       {'ifmap-cardinality':'singleValue'}, ns_prefix = 'contrail')
            else: # complex type
                prop_cls = cfgm_common.utils.str_to_class(prop_type,
                                                          __name__)
                buf = cStringIO.StringIO()
                # perms might be inserted at server as obj.
                # obj construction diff from dict construction.
                if isinstance(field, dict):
                    prop_cls(**field).exportChildren(
                        buf, level=1, name_=prop_meta, pretty_print=False)
                elif isinstance(field, list):
                    for elem in field:
                        if isinstance(elem, dict):
                            prop_cls(**elem).exportChildren(
                                buf, level=1, name_=prop_meta, pretty_print=False)
                        else:
                            elem.exportChildren(
                                buf, level=1, name_=prop_meta, pretty_print=False)
                else: # object
                    field.exportChildren(
                        buf, level=1, name_=prop_meta, pretty_print=False)
                prop_xml = buf.getvalue()
                buf.close()
                meta = Metadata(prop_meta, '',
                    {'ifmap-cardinality':'singleValue'}, ns_prefix='contrail',
                    elements=prop_xml)

            # If obj is new (existing metas is none) or
            # if obj does not have this prop_meta (or)
            # or if the prop_meta is different from what we have currently,
            # then update
            if (not existing_metas or
                not prop_meta in existing_metas or
                    ('' in existing_metas[prop_meta] and
                    str(meta) != str(existing_metas[prop_meta]['']))):
                self._update_id_self_meta(update, meta)
        # end for all property types

        # References Meta
        for ref_field in obj_class.ref_fields:
            refs = obj_dict.get(ref_field)
            if not refs:
                continue
            for ref in refs:
                ref_fq_name = ref['to']
                ref_fld_types_list = list(
                    obj_class.ref_field_types[ref_field])
                ref_res_type = ref_fld_types_list[0]
                ref_link_type = ref_fld_types_list[1]
                ref_meta = obj_class.ref_field_metas[ref_field]
                ref_imid = get_ifmap_id_from_fq_name(ref_res_type, ref_fq_name)
                ref_data = ref.get('attr')
                if ref_data:
                    buf = cStringIO.StringIO()
                    attr_cls = cfgm_common.utils.str_to_class(ref_link_type,
                                                              __name__)
                    attr_cls(**ref_data).exportChildren(
                        buf, level=1, name_=ref_meta, pretty_print=False)
                    ref_link_xml = buf.getvalue()
                    buf.close()
                else:
                    ref_link_xml = ''
                meta = Metadata(ref_meta, '',
                    {'ifmap-cardinality':'singleValue'}, ns_prefix = 'contrail',
                    elements=ref_link_xml)
                self._update_id_pair_meta(update, ref_imid, meta)
        # end for all ref types

        self._publish_update(my_imid, update)
        return (True, '')
    # end object_set

    def object_create(self, obj_ids, obj_dict):
        obj_type = obj_ids['type']
        if not 'parent_type' in obj_dict:
            # parent is config-root
            parent_type = 'config-root'
            parent_imid = 'contrail:config-root:root'
        else:
            parent_type = obj_dict['parent_type']
            parent_imid = obj_ids.get('parent_imid', None)

        # Parent Link Meta
        update = {}
        parent_cls = self._db_client_mgr.get_resource_class(parent_type)
        parent_link_meta = parent_cls.children_field_metas.get('%ss' %(obj_type))
        if parent_link_meta:
            meta = Metadata(parent_link_meta, '',
                       {'ifmap-cardinality':'singleValue'}, ns_prefix = 'contrail')
            self._update_id_pair_meta(update, obj_ids['imid'], meta)
            self._publish_update(parent_imid, update)

        (ok, result) = self.object_set(obj_type, obj_ids['imid'], None, obj_dict)
        return (ok, result)
    # end object_create

    def _object_read_to_meta_index(self, ifmap_id):
        # metas is a dict where key is meta-name and val is list of dict of
        # form [{'meta':meta}, {'id':id1, 'meta':meta}, {'id':id2, 'meta':meta}]
        metas = {}
        if ifmap_id in self._id_to_metas:
            metas = self._id_to_metas[ifmap_id].copy()
        return metas
    # end _object_read_to_meta_index

    def object_update(self, obj_type, ifmap_id, new_obj_dict):
        obj_cls = self._db_client_mgr.get_resource_class(obj_type)
        # read in refs from ifmap to determine which ones become inactive after update
        existing_metas = self._object_read_to_meta_index(ifmap_id)

        if not existing_metas:
            # UPDATE notify queued before CREATE notify, Skip publish to IFMAP.
            return (True, '')

        # remove properties that are no longer active
        props = obj_cls.prop_field_metas
        for prop, meta in props.items():
            if meta in existing_metas and new_obj_dict.get(prop) is None:
                self._delete_id_self_meta(ifmap_id, meta)

        # remove refs that are no longer active
        delete_list = []
        refs = dict((obj_cls.ref_field_metas[rf],
                     obj_cls.ref_field_types[rf][0])
                    for rf in obj_cls.ref_fields)
        #refs = {'virtual-network-qos-forwarding-class': 'qos-forwarding-class',
        #        'virtual-network-network-ipam': 'network-ipam',
        #        'virtual-network-network-policy': 'network-policy',
        #        'virtual-network-route-table': 'route-table'}
        for meta, ref_res_type in refs.items():
            old_set = set(existing_metas.get(meta, {}).keys())
            new_set = set()
            ref_obj_type = self._db_client_mgr.get_resource_class(
                ref_res_type).object_type
            for ref in new_obj_dict.get(ref_obj_type+'_refs', []):
                to_imid = get_ifmap_id_from_fq_name(ref_res_type, ref['to'])
                new_set.add(to_imid)

            for inact_ref in old_set - new_set:
                delete_list.append((inact_ref, meta))

        if delete_list:
            self._delete_id_pair_meta_list(ifmap_id, delete_list)

        (ok, result) = self.object_set(obj_type, ifmap_id, existing_metas, new_obj_dict)
        return (ok, result)
    # end object_update

    def object_delete(self, obj_ids):
        ifmap_id = obj_ids['imid']
        parent_imid = obj_ids.get('parent_imid')
        existing_metas = self._object_read_to_meta_index(ifmap_id)
        meta_list = []
        for meta_name, meta_infos in existing_metas.items():
            # Delete all refs/links in the object.
            # Refs are identified when the key is a non-empty string.
            meta_list.extend(
                [(k, meta_name)
                  for k in meta_infos if k != ''])

        if parent_imid:
            # Remove link from parent
            meta_list.append((parent_imid, None))

        if meta_list:
            self._delete_id_pair_meta_list(ifmap_id, meta_list)

        # Remove all property metadata associated with this ident
        self._delete_id_self_meta(ifmap_id, None)

        return (True, '')
    # end object_delete

    def _init_conn(self):
        self._mapclient = client(("%s" % (self._ifmap_srv_ip),
                                  "%s" % (self._ifmap_srv_port)),
                                 self._username, self._password,
                                 self._NAMESPACES, self._ssl_options)

        connected = False
        while not connected:
            try:
                resp_xml = self._mapclient.call('newSession', NewSessionRequest())
            except socket.error as e:
                msg = 'Failed to establish IF-MAP connection: %s' % str(e)
                self.config_log(msg, level=SandeshLevel.SYS_WARN)
                time.sleep(3)
                continue

            resp_doc = etree.parse(StringIO.StringIO(resp_xml))
            err_codes = resp_doc.xpath(
                '/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
                namespaces=self._NAMESPACES)
            if not err_codes:
                connected = True
            else:
                msg = "Failed to establish IF-MAP connection: %s" % err_codes
                self.config_log(msg, level=SandeshLevel.SYS_WARN)
                session_id = self._mapclient.get_session_id()
                try:
                    self._mapclient.call('endSession',
                                         EndSessionRequest(session_id))
                except socket.error as e:
                    msg = "Failed to end the IF-MAP session %s: %s" %\
                          (session_id, str(e))
                    self.config_log(msg, level=SandeshLevel.SYS_WARN)
                time.sleep(3)

        ConnectionState.update(conn_type = ConnType.IFMAP,
            name = 'IfMap', status = ConnectionStatus.UP, message = '',
            server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
                                       self._ifmap_srv_port)])
        self._conn_state = ConnectionStatus.UP
        msg = 'IFMAP connection ESTABLISHED'
        self.config_log(msg, level=SandeshLevel.SYS_NOTICE)

        self._mapclient.set_session_id(
            newSessionResult(resp_xml).get_session_id())
        self._mapclient.set_publisher_id(
            newSessionResult(resp_xml).get_publisher_id())
    # end _init_conn

    def _get_api_server(self):
        return self._db_client_mgr._api_svr_mgr
    # end _get_api_server

    def reset(self):
        self._id_to_metas = {}
        while not self._queue.empty():
            self._queue.get_nowait()

        if (self._dequeue_greenlet is not None and
                gevent.getcurrent() != self._dequeue_greenlet):
            self._dequeue_greenlet.kill()
        self._dequeue_greenlet =\
              vnc_greenlets.VncGreenlet("VNC IfMap Dequeue",
                                        self._ifmap_dequeue_task)

    # end reset

    def _publish_config_root(self):
        # Remove all resident data
        result = ifmap_wipe(self._mapclient)
        if result is None:
            msg = "Cannot purge the IF-MAP server before publishing root graph"
            self.config_log(msg, level=SandeshLevel.SYS_WARN)
        # Build default config-root
        buf = cStringIO.StringIO()
        perms = Provision.defaults.perms
        perms.exportChildren(buf, level=1, pretty_print=False)
        id_perms_xml = buf.getvalue()
        buf.close()
        update = {}
        meta = Metadata(self._IPERMS_NAME, '',
                        {'ifmap-cardinality': 'singleValue'},
                        ns_prefix='contrail', elements=id_perms_xml)
        self._update_id_self_meta(update, meta)
        self._publish_update("contrail:config-root:root", update)
    # end _publish_config_root

    def config_log(self, msg, level):
        self._db_client_mgr.config_log(msg, level)
    # end config_log

    @ignore_exceptions
    def _generate_ifmap_trace(self, oper, body):
        req_id = get_trace_id()
        ifmap_trace = IfmapTrace(request_id=req_id)
        ifmap_trace.operation = oper
        ifmap_trace.body = body

        return ifmap_trace
    # end _generate_ifmap_trace

    def _publish_to_ifmap_enqueue(self, oper, oper_body, do_trace=True):
        # safety check, if we proceed ifmap-server reports error
        # asking for update|delete in publish
        if not oper_body:
            return
        self._queue.put((oper, oper_body, do_trace))
    # end _publish_to_ifmap_enqueue

    def _ifmap_dequeue_task(self):
        while True:
            try:
                self._publish_to_ifmap_dequeue()
            except Exception as e:
                tb = cfgm_common.utils.detailed_traceback()
                self.config_log(tb, level=SandeshLevel.SYS_ERR)

    def _publish_to_ifmap_dequeue(self):
        def _publish(requests, traces, publish_discovery=False):
            if not requests:
                return
            ok = False
            # Continue to trying publish requests until the queue is full.
            # When queue is full, ifmap is totally resync from db
            while not ok:
                ok, err_msg = self._publish_to_ifmap(''.join(requests))
                if ok:
                    trace_msg(traces, 'IfmapTraceBuf', self._sandesh)
                else:
                    trace_msg(traces, 'IfmapTraceBuf', self._sandesh,
                              error_msg=err_msg)
                if publish_discovery and ok:
                    self._get_api_server().publish_ifmap_to_discovery()
                    self._is_ifmap_up = True
                if not ok:
                    msg = ("%s. IF-MAP sending queue size: %d/%d" %
                           (err_msg, self._queue.qsize(),
                            self._get_api_server()._args.ifmap_queue_size))
                    self.config_log(msg, level=SandeshLevel.SYS_WARN)
                    gevent.sleep(1)
        # end _publish

        while True:
            # block until there is data in the queue
            (oper, oper_body, do_trace) = self._queue.get()
            requests = []
            requests_len = 0
            traces = []
            while True:
                # drain the queue till empty or max message size
                # or change of oper because ifmap does not like
                # different operations in same message
                if oper == 'publish_discovery':
                    _publish(requests, traces, True)
                    break
                if do_trace:
                    trace = self._generate_ifmap_trace(oper, oper_body)
                    traces.append(trace)
                requests.append(oper_body)
                requests_len += len(oper_body)
                if (requests_len >
                    self._get_api_server()._args.ifmap_max_message_size):
                    _publish(requests, traces)
                    break
                old_oper = oper
                try:
                    (oper, oper_body, do_trace) = self._queue.get_nowait()
                    if oper != old_oper:
                        _publish(requests, traces)
                        requests = []
                        requests_len = 0
                        traces = []
                        continue
                except Empty:
                    _publish(requests, traces)
                    break
    # end _publish_to_ifmap_dequeue

    def _publish_to_ifmap(self, oper_body):
        try:
            not_published = True
            retry_count = 0
            resp_xml = None
            while not_published:
                sess_id = self._mapclient.get_session_id()
                req_xml = PublishRequest(sess_id, oper_body)
                resp_xml = self._mapclient.call('publish', req_xml)

                resp_doc = etree.parse(StringIO.StringIO(resp_xml))
                err_codes = resp_doc.xpath(
                    '/env:Envelope/env:Body/ifmap:response/errorResult/@errorCode',
                    namespaces=self._NAMESPACES)
                if err_codes:
                    if retry_count == 0:
                        log_str = 'Error publishing to ifmap, req: %s, resp: %s' \
                                  %(req_xml, resp_xml)
                        self.config_log(log_str, level=SandeshLevel.SYS_ERR)

                    ConnectionState.update(
                        conn_type = ConnType.IFMAP,
                        name = 'IfMap',
                        status = ConnectionStatus.INIT,
                        message = 'Session lost, renew it',
                        server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
                                                   self._ifmap_srv_port)])
                    self._conn_state = ConnectionStatus.INIT
                    self._is_ifmap_up = False
                    retry_count = retry_count + 1
                    self._init_conn()

                    if self._ifmap_restarted():
                        msg = "IF-MAP servers restarted, re-populate it"
                        self.config_log(msg, level=SandeshLevel.SYS_ERR)

                        self.reset()
                        self._get_api_server().publish_ifmap_to_discovery(
                            'down', msg)

                        self._publish_config_root()
                        self._db_client_mgr.db_resync()
                        self._publish_to_ifmap_enqueue('publish_discovery', 1)

                else: # successful publish
                    not_published = False
                    break
            # end while not_published

            if retry_count:
                log_str = 'Success publishing to ifmap after %d tries' \
                          %(retry_count)
                self.config_log(log_str, level=SandeshLevel.SYS_ERR)

            return True, resp_xml
        except Exception as e:
            # Failed to publish the operation due to unknown error.
            # Probably a connection issue with the ifmap server.
            msg = "Failed to publish request %s: %s" % (oper_body, str(e))
            return False, msg
    # end _publish_to_ifmap

    def _build_request(self, id1_name, id2_name, meta_list, delete=False):
            request = ''
            id1 = unicode(Identity(name=id1_name, type="other",
                                   other_type="extended"))
            if id2_name != 'self':
                id2 = unicode(Identity(name=id2_name, type="other",
                                       other_type="extended"))
            else:
                id2 = None
            for m in meta_list:
                if delete:
                    filter = unicode(m) if m else None
                    op = PublishDeleteOperation(id1=id1, id2=id2,
                                                filter=filter)
                else:
                    op = PublishUpdateOperation(id1=id1, id2=id2,
                                                metadata=unicode(m),
                                                lifetime='forever')
                request += unicode(op)
            return request

    def _delete_id_self_meta(self, self_imid, meta_name):
        contrail_metaname = 'contrail:' + meta_name if meta_name else None
        del_str = self._build_request(self_imid, 'self', [contrail_metaname],
                                      True)
        self._publish_to_ifmap_enqueue('delete', del_str)

        try:

            # del meta from cache and del id if this was last meta
            if meta_name:
                del self._id_to_metas[self_imid][meta_name]
                if not self._id_to_metas[self_imid]:
                    del self._id_to_metas[self_imid]
            else:
                del self._id_to_metas[self_imid]

        except KeyError:
            # Case of delete received for an id which we do not know about.
            # Could be a case of duplicate delete.
            # There is nothing for us to do here. Just log and proceed.
            msg = "Delete received for unknown imid(%s) meta_name(%s)." % \
                  (self_imid, meta_name)
            self.config_log(msg, level=SandeshLevel.SYS_DEBUG)

    # end _delete_id_self_meta

    def _delete_id_pair_meta_list(self, id1, meta_list):
        del_str = ''
        for id2, metadata in meta_list:
            contrail_metadata = 'contrail:' + metadata if metadata else None
            del_str += self._build_request(id1, id2, [contrail_metadata], True)

        self._publish_to_ifmap_enqueue('delete', del_str)

        # del meta,id2 from cache and del id if this was last meta
        def _id_to_metas_delete(id1, id2, meta_name):
            if id1 not in self._id_to_metas:
                return
            if meta_name not in self._id_to_metas[id1]:
                return
            if not self._id_to_metas[id1][meta_name]:
                del self._id_to_metas[id1][meta_name]
                if not self._id_to_metas[id1]:
                    del self._id_to_metas[id1]
                return

            # if meta is prop, noop
            if id2 in self._id_to_metas[id1][meta_name]:
                del self._id_to_metas[id1][meta_name][id2]
        #end _id_to_metas_delete

        for id2, metadata in meta_list:
            if metadata:
                # replace with remaining refs
                _id_to_metas_delete(id1, id2, metadata)
                _id_to_metas_delete(id2, id1, metadata)
            else: # no meta specified remove all links from id1 to id2
                for meta_name in self._id_to_metas.get(id1, {}).keys():
                    _id_to_metas_delete(id1, id2, meta_name)
                for meta_name in self._id_to_metas.get(id2, {}).keys():
                    _id_to_metas_delete(id2, id1, meta_name)
    # end _delete_id_pair_meta_list

    def _update_id_self_meta(self, update, meta):
        """ update: dictionary of the type
                update[<id> | 'self'] = list(metadata)
        """
        mlist = update.setdefault('self', [])
        mlist.append(meta)
    # end _update_id_self_meta

    def _update_id_pair_meta(self, update, to_id, meta):
        mlist = update.setdefault(to_id, [])
        mlist.append(meta)
     # end _update_id_pair_meta

    def _publish_update(self, self_imid, update):
        requests = []
        self_metas = self._id_to_metas.setdefault(self_imid, {})
        for id2, metalist in update.items():
            request = self._build_request(self_imid, id2, metalist)

            # remember what we wrote for diffing during next update
            old_metalist = []
            for m in metalist:
                meta_name = m._Metadata__name[9:]

                # Objects have two types of members - Props and refs/links.
                # Props are cached in id_to_metas as
                #        id_to_metas[self_imid][meta_name]['']
                #        (with empty string as key)

                # Links are cached in id_to_metas as
                #        id_to_metas[self_imid][meta_name][id2]
                #        id2 is used as a key

                if id2 == 'self':
                    self_metas[meta_name] = {'' : m}
                    continue

                if meta_name in self_metas:
                    old_metalist.append(self_metas[meta_name])
                    # Update the link/ref
                    self_metas[meta_name][id2] = m
                else:
                    # Create a new link/ref
                    self_metas[meta_name] = {id2 : m}

                # Reverse linking from id2 to id1
                self._id_to_metas.setdefault(id2, {})

                if meta_name in self._id_to_metas[id2]:
                    self._id_to_metas[id2][meta_name][self_imid] = m
                else:
                    self._id_to_metas[id2][meta_name] = {self_imid : m}

            old_request = self._build_request(self_imid, id2, old_metalist)
            if request != old_request:
                requests.append(request)

        upd_str = ''.join(requests)
        self._publish_to_ifmap_enqueue('update', upd_str)
    # end _publish_update

    def _ifmap_restarted(self):
        return not entity_is_present(self._mapclient, 'config-root', ['root'])

    def _health_checker(self):
        while True:
            try:
                # do the healthcheck only if we are connected
                if self._conn_state == ConnectionStatus.DOWN:
                    continue
                meta = Metadata('display-name', '',
                        {'ifmap-cardinality': 'singleValue'},
                        ns_prefix='contrail', elements='')
                request_str = self._build_request('healthcheck', 'self', [meta])
                self._publish_to_ifmap_enqueue('update', request_str, do_trace=False)

                # Confirm the existence of the following default global entities in IFMAP.
                search_list = [
                    ('global-system-config', ['default-global-system-config']),
                ]
                for type, fq_name in search_list:
                    if not entity_is_present(self._mapclient, type, fq_name):
                        raise Exception("%s not found in IFMAP DB" % ':'.join(fq_name))

                # If we had unpublished the IFMAP server to discovery server earlier
                # publish it back now since it is valid now.
                if not self._is_ifmap_up:
                    self._get_api_server().publish_ifmap_to_discovery('up', '')
                    self._is_ifmap_up = True
                    ConnectionState.update(conn_type = ConnType.IFMAP,
                                           name = 'IfMap',
                                           status = ConnectionStatus.UP,
                                           message = '',
                                           server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
                                                                      self._ifmap_srv_port)])
            except Exception as e:
                log_str = 'IFMAP Healthcheck failed: %s' %(str(e))
                self.config_log(log_str, level=SandeshLevel.SYS_ERR)
                if self._is_ifmap_up:
                    self._get_api_server().publish_ifmap_to_discovery('down',
                                                   'IFMAP DB - Invalid state')
                    self._is_ifmap_up = False
                    ConnectionState.update(conn_type = ConnType.IFMAP,
                                           name = 'IfMap',
                                           status = ConnectionStatus.DOWN,
                                           message = 'Invalid IFMAP DB State',
                                           server_addrs = ["%s:%s" % (self._ifmap_srv_ip,
                                                                      self._ifmap_srv_port)])
            finally:
                gevent.sleep(
                    self._get_api_server().get_ifmap_health_check_interval())
    # end _health_checker

# end class VncIfmapClient


class VncServerCassandraClient(VncCassandraClient):
    # Useragent datastore keyspace + tables (used by neutron plugin currently)
    _USERAGENT_KEYSPACE_NAME = USERAGENT_KEYSPACE_NAME
    _USERAGENT_KV_CF_NAME = 'useragent_keyval_table'

    @classmethod
    def get_db_info(cls):
        db_info = VncCassandraClient.get_db_info() + \
                  [(cls._USERAGENT_KEYSPACE_NAME, [cls._USERAGENT_KV_CF_NAME])]
        return db_info
    # end get_db_info

    def __init__(self, db_client_mgr, cass_srv_list, reset_config, db_prefix,
                      cassandra_credential, walk, obj_cache_entries,
                      obj_cache_exclude_types):
        self._db_client_mgr = db_client_mgr
        keyspaces = self._UUID_KEYSPACE.copy()
        keyspaces[self._USERAGENT_KEYSPACE_NAME] = {
            self._USERAGENT_KV_CF_NAME: {}}
        super(VncServerCassandraClient, self).__init__(
            cass_srv_list, db_prefix, keyspaces, None, self.config_log,
            generate_url=db_client_mgr.generate_url, reset_config=reset_config,
            credential=cassandra_credential, walk=walk,
            obj_cache_entries=obj_cache_entries,
            obj_cache_exclude_types=obj_cache_exclude_types)
        self._useragent_kv_cf = self._cf_dict[self._USERAGENT_KV_CF_NAME]
    # end __init__

    def config_log(self, msg, level):
        self._db_client_mgr.config_log(msg, level)
    # end config_log

    def prop_collection_update(self, obj_type, obj_uuid, updates):
        obj_class = self._db_client_mgr.get_resource_class(obj_type)
        bch = self._obj_uuid_cf.batch()
        for oper_param in updates:
            oper = oper_param['operation']
            prop_name = oper_param['field']
            if prop_name in obj_class.prop_list_fields:
                if oper == 'add':
                    prop_elem_val = oper_param['value']
                    prop_elem_pos = oper_param.get('position') or str(uuid.uuid4())
                    self._add_to_prop_list(bch, obj_uuid,
                        prop_name, prop_elem_val, prop_elem_pos)
                elif oper == 'modify':
                    prop_elem_val = oper_param['value']
                    prop_elem_pos = oper_param['position']
                    # modify is practically an insert so use add
                    self._add_to_prop_list(bch, obj_uuid,
                        prop_name, prop_elem_val, prop_elem_pos)
                elif oper == 'delete':
                    prop_elem_pos = oper_param['position']
                    self._delete_from_prop_list(bch, obj_uuid,
                        prop_name, prop_elem_pos)
            elif prop_name in obj_class.prop_map_fields:
                key_name = obj_class.prop_map_field_key_names[prop_name]
                if oper == 'set':
                    prop_elem_val = oper_param['value']
                    position = prop_elem_val[key_name]
                    self._set_in_prop_map(bch, obj_uuid,
                        prop_name, prop_elem_val, position)
                elif oper == 'delete':
                    position = oper_param['position']
                    self._delete_from_prop_map(bch, obj_uuid,
                        prop_name, position)
        # end for all updates

        self.update_last_modified(bch, obj_type, obj_uuid)
        bch.send()
    # end prop_collection_update

    def ref_update(self, obj_type, obj_uuid, ref_obj_type, ref_uuid,
                   ref_data, operation):
        bch = self._obj_uuid_cf.batch()
        if operation == 'ADD':
            self._create_ref(bch, obj_type, obj_uuid, ref_obj_type, ref_uuid,
                             ref_data)
        elif operation == 'DELETE':
            self._delete_ref(bch, obj_type, obj_uuid, ref_obj_type, ref_uuid)
        else:
            pass
        self.update_last_modified(bch, obj_type, obj_uuid)
        bch.send()
    # end ref_update

    def ref_relax_for_delete(self, obj_uuid, ref_uuid):
        bch = self._obj_uuid_cf.batch()
        self._relax_ref_for_delete(bch, obj_uuid, ref_uuid)
        bch.send()
    # end ref_relax_for_delete

    def _relax_ref_for_delete(self, bch, obj_uuid, ref_uuid):
        send = False
        if bch is None:
            send = True
            bch = self._obj_uuid_cf.batch()
        bch.insert(ref_uuid, {'relaxbackref:%s' % (obj_uuid):
                               json.dumps(None)})
        if send:
            bch.send()
    # end _relax_ref_for_delete

    def get_relaxed_refs(self, obj_uuid):
        relaxed_cols = self.get(self._OBJ_UUID_CF_NAME, obj_uuid,
                                start='relaxbackref:', finish='relaxbackref;')
        if not relaxed_cols:
            return []

        return [col.split(':')[1] for col in relaxed_cols]
    # end get_relaxed_refs

    def is_latest(self, id, tstamp):
        id_perms = self.uuid_to_obj_perms(id)
        if id_perms['last_modified'] == tstamp:
            return True
        else:
            return False
    # end is_latest

    # Insert new perms. Called on startup when walking DB
    def update_perms2(self, obj_uuid):
        bch = self._obj_uuid_cf.batch()
        perms2 = copy.deepcopy(Provision.defaults.perms2)
        perms2_json = json.dumps(perms2, default=lambda o: dict((k, v)
                               for k, v in o.__dict__.iteritems()))
        perms2 = json.loads(perms2_json)
        self._update_prop(bch, obj_uuid, 'perms2', {'perms2': perms2})
        bch.send()
        return perms2

    def enable_domain_sharing(self, obj_uuid, perms2):
        share_item = {
            'tenant': 'domain:%s' % obj_uuid,
            'tenant_access': cfgm_common.DOMAIN_SHARING_PERMS
        }
        perms2['share'].append(share_item)
        bch = self._obj_uuid_cf.batch()
        self._update_prop(bch, obj_uuid, 'perms2', {'perms2': perms2})
        bch.send()

    def uuid_to_obj_dict(self, id):
        obj_cols = self.get(self._OBJ_UUID_CF_NAME, id)
        if not obj_cols:
            raise NoIdError(id)
        return obj_cols
    # end uuid_to_obj_dict

    def uuid_to_obj_perms(self, id):
        return self.get_one_col(self._OBJ_UUID_CF_NAME, id, 'prop:id_perms')
    # end uuid_to_obj_perms

    # fetch perms2 for an object
    def uuid_to_obj_perms2(self, id):
        return self.get_one_col(self._OBJ_UUID_CF_NAME, id, 'prop:perms2')
    # end uuid_to_obj_perms2

    def useragent_kv_store(self, key, value):
        columns = {'value': value}
        self._useragent_kv_cf.insert(key, columns)
    # end useragent_kv_store

    def useragent_kv_retrieve(self, key):
        if key:
            if isinstance(key, list):
                rows = self._useragent_kv_cf.multiget(key)
                return [rows[row].get('value') for row in rows]
            else:
                try:
                    columns = self._useragent_kv_cf.get(key)
                except pycassa.NotFoundException:
                    raise NoUserAgentKey
                return columns.get('value')
        else:  # no key specified, return entire contents
            kv_list = []
            for ua_key, ua_cols in self._useragent_kv_cf.get_range():
                kv_list.append({'key': ua_key, 'value': ua_cols.get('value')})
            return kv_list
    # end useragent_kv_retrieve

    def useragent_kv_delete(self, key):
        try:
            self._useragent_kv_cf.remove(key)
        except pycassa.NotFoundException:
            raise NoUserAgentKey
    # end useragent_kv_delete

# end class VncCassandraClient


class VncServerKombuClient(VncKombuClient):
    def __init__(self, db_client_mgr, rabbit_ip, rabbit_port, ifmap_db,
                 rabbit_user, rabbit_password, rabbit_vhost, rabbit_ha_mode,
                 rabbit_health_check_interval, **kwargs):
        self._db_client_mgr = db_client_mgr
        self._sandesh = db_client_mgr._sandesh
        self._ifmap_db = ifmap_db
        listen_port = db_client_mgr.get_server_port()
        q_name = 'vnc_config.%s-%s' %(socket.gethostname(), listen_port)
        super(VncServerKombuClient, self).__init__(
            rabbit_ip, rabbit_port, rabbit_user, rabbit_password, rabbit_vhost,
            rabbit_ha_mode, q_name, self._dbe_subscribe_callback,
            self.config_log, heartbeat_seconds=rabbit_health_check_interval,
            **kwargs)

    # end __init__

    def prepare_to_consume(self):
        self._db_client_mgr.wait_for_resync_done()
    # prepare_to_consume

    def config_log(self, msg, level):
        self._db_client_mgr.config_log(msg, level)
    # end config_log

    def uuid_to_fq_name(self, uuid):
        self._db_client_mgr.uuid_to_fq_name(uuid)
    # end uuid_to_fq_name

    def dbe_uve_trace(self, oper, typ, uuid, body):
        self._db_client_mgr.dbe_uve_trace(oper, typ, uuid, body)
    # end dbe_uve_trace

    def dbe_oper_publish_pending(self):
        return self.num_pending_messages()
    # end dbe_oper_publish_pending

    @ignore_exceptions
    def _generate_msgbus_notify_trace(self, oper_info):
        req_id = oper_info.get('request-id',
            'req-%s' %(str(uuid.uuid4())))
        gevent.getcurrent().trace_request_id = req_id

        notify_trace = MessageBusNotifyTrace(request_id=req_id)
        notify_trace.operation = oper_info.get('oper', '')
        notify_trace.body = json.dumps(oper_info)

        return notify_trace
    # end _generate_msgbus_notify_trace

    def _dbe_subscribe_callback(self, oper_info):
        self._db_client_mgr.wait_for_resync_done()
        try:
            msg = "Notification Message: %s" %(pformat(oper_info))
            self.config_log(msg, level=SandeshLevel.SYS_DEBUG)
            trace = self._generate_msgbus_notify_trace(oper_info)

            if oper_info['oper'] == 'CREATE':
                self._dbe_create_notification(oper_info)
            if oper_info['oper'] == 'UPDATE':
                self._dbe_update_notification(oper_info)
            elif oper_info['oper'] == 'DELETE':
                self._dbe_delete_notification(oper_info)

            trace_msg([trace], 'MessageBusNotifyTraceBuf', self._sandesh)
        except Exception as e:
            string_buf = cStringIO.StringIO()
            cgitb_hook(file=string_buf, format="text")
            errmsg = string_buf.getvalue()
            self.config_log(string_buf.getvalue(),
                level=SandeshLevel.SYS_ERR)
            trace_msg([trace], name='MessageBusNotifyTraceBuf',
                              sandesh=self._sandesh, error_msg=errmsg)
    #end _dbe_subscribe_callback

    def dbe_create_publish(self, obj_type, obj_ids, obj_dict):
        req_id = get_trace_id()
        oper_info = {'request-id': req_id,
                     'oper': 'CREATE',
                     'type': obj_type,
                     'obj_dict': obj_dict}
        oper_info.update(obj_ids)
        self.publish(oper_info)
    # end dbe_create_publish

    def _dbe_create_notification(self, obj_info):
        try:
            (ok, result) = self._db_client_mgr.dbe_read(obj_info['type'], obj_info)
            if not ok:
                raise Exception(result)
            obj_dict = result
        except NoIdError as e:
            # No error, we will hear a delete shortly
            return

        self.dbe_uve_trace("CREATE", obj_info['type'], obj_info['uuid'], obj_dict)

        try:
            r_class = self._db_client_mgr.get_resource_class(obj_info['type'])
            if r_class:
                r_class.dbe_create_notification(obj_info, obj_dict)
        except Exception as e:
            err_msg = ("Failed in type specific dbe_create_notification " +
                       str(e))
            self.config_log(err_msg, level=SandeshLevel.SYS_ERR)
            raise
        finally:
            (ok, result) = self._ifmap_db.object_create(obj_info, obj_dict)
            if not ok:
                self.config_log(result, level=SandeshLevel.SYS_ERR)
                raise Exception(result)
    #end _dbe_create_notification

    def dbe_update_publish(self, obj_type, obj_ids):
        oper_info = {'oper': 'UPDATE', 'type': obj_type}
        oper_info.update(obj_ids)
        self.publish(oper_info)
    # end dbe_update_publish

    def _dbe_update_notification(self, obj_info):
        try:
            (ok, result) = self._db_client_mgr.dbe_read(obj_info['type'], obj_info)
        except NoIdError as e:
            # No error, we will hear a delete shortly
            return

        new_obj_dict = result

        self.dbe_uve_trace("UPDATE", obj_info['type'], obj_info['uuid'], new_obj_dict)

        try:
            r_class = self._db_client_mgr.get_resource_class(obj_info['type'])
            if r_class:
                r_class.dbe_update_notification(obj_info)
        except:
            msg = "Failed to invoke type specific dbe_update_notification"
            self.config_log(msg, level=SandeshLevel.SYS_ERR)
            raise
        finally:
            ifmap_id = self._db_client_mgr.uuid_to_ifmap_id(
                r_class.resource_type, obj_info['uuid'])
            (ok, ifmap_result) = self._ifmap_db.object_update(
                obj_info['type'], ifmap_id, new_obj_dict)
            if not ok:
                raise Exception(ifmap_result)
    #end _dbe_update_notification

    def dbe_delete_publish(self, obj_type, obj_ids, obj_dict):
        oper_info = {'oper': 'DELETE', 'type': obj_type, 'obj_dict': obj_dict}
        oper_info.update(obj_ids)
        self.publish(oper_info)
    # end dbe_delete_publish

    def _dbe_delete_notification(self, obj_info):
        obj_dict = obj_info['obj_dict']

        self.dbe_uve_trace(
            "DELETE", obj_info['type'], obj_info['uuid'], obj_dict)

        db_client_mgr = self._db_client_mgr
        db_client_mgr._object_db.cache_uuid_to_fq_name_del(obj_dict['uuid'])

        try:
            r_class = self._db_client_mgr.get_resource_class(obj_info['type'])
            if r_class:
                r_class.dbe_delete_notification(obj_info, obj_dict)
        except:
            msg = "Failed to invoke type specific dbe_delete_notification"
            self.config_log(msg, level=SandeshLevel.SYS_ERR)
            raise
        finally:
            (ok, ifmap_result) = self._ifmap_db.object_delete(obj_info)
            if not ok:
                self.config_log(ifmap_result, level=SandeshLevel.SYS_ERR)
                raise Exception(ifmap_result)
    #end _dbe_delete_notification

# end class VncKombuClient


class VncZkClient(object):
    _SUBNET_PATH = "/api-server/subnets"
    _FQ_NAME_TO_UUID_PATH = "/fq-name-to-uuid"
    _MAX_SUBNET_ADDR_ALLOC = 65535

    _VN_ID_ALLOC_PATH = "/id/virtual-networks/"
    _VN_MAX_ID = 1 << 24

    _SG_ID_ALLOC_PATH = "/id/security-groups/id/"
    _SG_MAX_ID = 1 << 32

    def __init__(self, instance_id, zk_server_ip, reset_config, db_prefix,
                 sandesh_hdl):
        self._db_prefix = db_prefix
        if db_prefix:
            client_pfx = db_prefix + '-'
            zk_path_pfx = db_prefix + '/'
        else:
            client_pfx = ''
            zk_path_pfx = ''

        client_name = '%sapi-%s' %(client_pfx, instance_id)
        self._subnet_path = zk_path_pfx + self._SUBNET_PATH
        self._fq_name_to_uuid_path = zk_path_pfx + self._FQ_NAME_TO_UUID_PATH
        _vn_id_alloc_path = zk_path_pfx + self._VN_ID_ALLOC_PATH
        _sg_id_alloc_path = zk_path_pfx + self._SG_ID_ALLOC_PATH
        self._zk_path_pfx = zk_path_pfx

        self._sandesh = sandesh_hdl
        self._reconnect_zk_greenlet = None
        while True:
            try:
                self._zk_client = ZookeeperClient(client_name, zk_server_ip,
                                                  self._sandesh)
                # set the lost callback to always reconnect
                self._zk_client.set_lost_cb(self.reconnect_zk)
                break
            except gevent.event.Timeout as e:
                pass

        if reset_config:
            self._zk_client.delete_node(self._subnet_path, True)
            self._zk_client.delete_node(self._fq_name_to_uuid_path, True)
            self._zk_client.delete_node(_vn_id_alloc_path, True)
            self._zk_client.delete_node(_vn_id_alloc_path, True)

        self._subnet_allocators = {}

        # Initialize the virtual network ID allocator
        self._vn_id_allocator = IndexAllocator(self._zk_client,
                                               _vn_id_alloc_path,
                                               self._VN_MAX_ID)

        # Initialize the security group ID allocator
        self._sg_id_allocator = IndexAllocator(self._zk_client,
                                               _sg_id_alloc_path,
                                               self._SG_MAX_ID)
        # 0 is not a valid sg id any more. So, if it was previously allocated,
        # delete it and reserve it
        if self._sg_id_allocator.read(0) != '__reserved__':
            self._sg_id_allocator.delete(0)
        self._sg_id_allocator.reserve(0, '__reserved__')
    # end __init__

    def master_election(self, path, func, *args):
        self._zk_client.master_election(
            self._zk_path_pfx + path, os.getpid(),
            func, *args)
    # end master_election

    def _reconnect_zk(self):
        self._zk_client.connect()
        self._reconnect_zk_greenlet = None
    # end

    def reconnect_zk(self):
        if self._reconnect_zk_greenlet is None:
            self._reconnect_zk_greenlet =\
                   vnc_greenlets.VncGreenlet("VNC IfMap ZK Reconnect",
                                             self._reconnect_zk)
    # end

    def create_subnet_allocator(self, subnet, subnet_alloc_list,
                                addr_from_start, should_persist,
                                start_subnet, size, alloc_unit):
        # TODO handle subnet resizing change, ignore for now
        if subnet not in self._subnet_allocators:
            if addr_from_start is None:
                addr_from_start = False
            self._subnet_allocators[subnet] = IndexAllocator(
                self._zk_client, self._subnet_path+'/'+subnet+'/',
                size=size/alloc_unit, start_idx=start_subnet/alloc_unit,
                reverse=not addr_from_start,
                alloc_list=[{'start': x['start']/alloc_unit, 'end':x['end']/alloc_unit}
                            for x in subnet_alloc_list],
                max_alloc=self._MAX_SUBNET_ADDR_ALLOC/alloc_unit)
    # end create_subnet_allocator

    def delete_subnet_allocator(self, subnet):
        self._subnet_allocators.pop(subnet, None)
        IndexAllocator.delete_all(self._zk_client,
                                  self._subnet_path+'/'+subnet+'/')
    # end delete_subnet_allocator

    def _get_subnet_allocator(self, subnet):
        return self._subnet_allocators.get(subnet)
    # end _get_subnet_allocator

    def subnet_is_addr_allocated(self, subnet, addr):
        allocator = self._get_subnet_allocator(subnet)
        return allocator.read(addr)
    # end subnet_is_addr_allocated

    def subnet_set_in_use(self, subnet, addr):
        allocator = self._get_subnet_allocator(subnet)
        allocator.set_in_use(addr)
    # end subnet_set_in_use

    def subnet_reset_in_use(self, subnet, addr):
        allocator = self._get_subnet_allocator(subnet)
        allocator.reset_in_use(addr)
    # end subnet_reset_in_use

    def subnet_reserve_req(self, subnet, addr, value):
        allocator = self._get_subnet_allocator(subnet)
        return allocator.reserve(addr, value)
    # end subnet_reserve_req

    def subnet_alloc_count(self, subnet):
        allocator = self._get_subnet_allocator(subnet)
        return allocator.get_alloc_count()
    # end subnet_alloc_count

    def subnet_alloc_req(self, subnet, value=None):
        allocator = self._get_subnet_allocator(subnet)
        try:
            return allocator.alloc(value=value)
        except ResourceExhaustionError:
            return None
    # end subnet_alloc_req

    def subnet_free_req(self, subnet, addr):
        allocator = self._get_subnet_allocator(subnet)
        if allocator:
            allocator.delete(addr)
    # end subnet_free_req

    def create_fq_name_to_uuid_mapping(self, obj_type, fq_name, id):
        fq_name_str = ':'.join(fq_name)
        zk_path = self._fq_name_to_uuid_path+'/%s:%s' %(obj_type, fq_name_str)
        self._zk_client.create_node(zk_path, id)
    # end create_fq_name_to_uuid_mapping

    def get_fq_name_to_uuid_mapping(self, obj_type, fq_name):
        fq_name_str = ':'.join(fq_name)
        zk_path = self._fq_name_to_uuid_path+'/%s:%s' %(obj_type, fq_name_str)
        obj_uuid, znode_stat = self._zk_client.read_node(
            zk_path, include_timestamp=True)

        return obj_uuid, znode_stat.ctime
    # end get_fq_name_to_uuid_mapping

    def delete_fq_name_to_uuid_mapping(self, obj_type, fq_name):
        fq_name_str = ':'.join(fq_name)
        zk_path = self._fq_name_to_uuid_path+'/%s:%s' %(obj_type, fq_name_str)
        self._zk_client.delete_node(zk_path)
    # end delete_fq_name_to_uuid_mapping

    def is_connected(self):
        return self._zk_client.is_connected()
    # end is_connected

    def alloc_vn_id(self, name):
        if name is not None:
            return self._vn_id_allocator.alloc(name)

    def free_vn_id(self, vn_id):
        if vn_id is not None and vn_id < self._VN_MAX_ID:
            self._vn_id_allocator.delete(vn_id)

    def get_vn_from_id(self, vn_id):
        if vn_id is not None and vn_id < self._VN_MAX_ID:
            return self._vn_id_allocator.read(vn_id)

    def alloc_sg_id(self, name):
        if name is not None:
            return self._sg_id_allocator.alloc(name) + SGID_MIN_ALLOC

    def free_sg_id(self, sg_id):
        if (sg_id is not None and
                sg_id > SGID_MIN_ALLOC and
                sg_id < self._SG_MAX_ID):
            self._sg_id_allocator.delete(sg_id - SGID_MIN_ALLOC)

    def get_sg_from_id(self, sg_id):
        if (sg_id is not None and
                sg_id > SGID_MIN_ALLOC and
                sg_id < self._SG_MAX_ID):
            return self._sg_id_allocator.read(sg_id - SGID_MIN_ALLOC)
# end VncZkClient


class VncDbClient(object):
    def __init__(self, api_svr_mgr, ifmap_srv_ip, ifmap_srv_port, uname,
                 passwd, db_srv_list,
                 rabbit_servers, rabbit_port, rabbit_user, rabbit_password,
                 rabbit_vhost, rabbit_ha_mode, reset_config=False,
                 zk_server_ip=None, db_prefix='', db_credential=None,
                 obj_cache_entries=0, obj_cache_exclude_types=None,
                 db_engine='cassandra', connection=None,
                 **kwargs):
        self._db_engine = db_engine
        self._api_svr_mgr = api_svr_mgr
        self._sandesh = api_svr_mgr._sandesh

        self._UVEMAP = {
            "virtual_network" : ("ObjectVNTable", False),
            "service_instance" : ("ObjectSITable", False),
            "virtual_router" : ("ObjectVRouter", True),
            "analytics_node" : ("ObjectCollectorInfo", True),
            "database_node" : ("ObjectDatabaseInfo", True),
            "config_node" : ("ObjectConfigNode", True),
            "service_chain" : ("ServiceChain", False),
            "physical_router" : ("ObjectPRouter", True),
            "bgp_router": ("ObjectBgpRouter", True),
        }

        # certificate auth
        ssl_options = None
        if api_svr_mgr._args.use_certs:
            ssl_options = {
                'keyfile': api_svr_mgr._args.keyfile,
                'certfile': api_svr_mgr._args.certfile,
                'ca_certs': api_svr_mgr._args.ca_certs,
                'cert_reqs': ssl.CERT_REQUIRED,
                'ciphers': 'ALL'
            }

        self._db_resync_done = gevent.event.Event()

        if api_svr_mgr.get_worker_id() == 0:
            msg = "Connecting to ifmap on %s:%s as %s" \
                  % (ifmap_srv_ip, ifmap_srv_port, uname)
            self.config_log(msg, level=SandeshLevel.SYS_NOTICE)

            self._ifmap_db = VncIfmapClient(
                self, ifmap_srv_ip, ifmap_srv_port, uname, passwd, ssl_options)
        else:
            self._ifmap_db = None

        msg = "Connecting to zookeeper on %s" % (zk_server_ip)
        self.config_log(msg, level=SandeshLevel.SYS_NOTICE)

        if db_engine == 'cassandra':
            self._zk_db = VncZkClient(api_svr_mgr.get_worker_id(), zk_server_ip,
                                      reset_config, db_prefix, self.config_log)
            def db_client_init():
                msg = "Connecting to database on %s" % (db_srv_list)
                self.config_log(msg, level=SandeshLevel.SYS_NOTICE)

                if api_svr_mgr.get_worker_id() == 0:
                    walk = False # done as part of db_resync()
                else:
                    walk = True

                self._object_db = VncServerCassandraClient(
                    self, db_srv_list, reset_config, db_prefix,
                    db_credential, walk, obj_cache_entries,
                    obj_cache_exclude_types)

            self._zk_db.master_election("/api-server-election", db_client_init)
        elif db_engine == 'rdbms':
            self._object_db = VncServerRDBMSClient(self,
                server_list=db_srv_list, reset_config=reset_config,
                generate_url=self.generate_url,
                connection=connection,
                db_prefix=db_prefix, credential=db_credential)
            self._zk_db = self._object_db

        self._msgbus = VncServerKombuClient(self, rabbit_servers,
            rabbit_port, self._ifmap_db,
            rabbit_user, rabbit_password,
            rabbit_vhost, rabbit_ha_mode,
            api_svr_mgr.get_rabbit_health_check_interval(),
            **kwargs)
    # end __init__

    def _update_default_quota(self):
        """ Read the default quotas from the configuration
        and update it in the project object if not already
        updated.
        """
        default_quota = QuotaHelper.default_quota

        proj_id = self.fq_name_to_uuid('project',
                                       ['default-domain', 'default-project'])
        try:
            (ok, result) = self.dbe_read('project', {'uuid':proj_id})
        except NoIdError as e:
            ok = False
            result = 'Project Not Found: %s' %(proj_id)
        if not ok:
            self.config_log("Updating default quota failed: %s." %(result),
                level=SandeshLevel.SYS_ERR)
            return

        proj_dict = result
        quota = QuotaType()

        proj_dict['quota'] = default_quota
        self.dbe_update('project', {'uuid':proj_id}, proj_dict)
    # end _update_default_quota

    def get_api_server(self):
        return self._api_svr_mgr
    # end get_api_server

    def db_resync(self):
        # Read contents from cassandra and publish to ifmap
        start_time = datetime.datetime.utcnow()
        self._object_db.walk(self._dbe_resync)
        self._ifmap_db._publish_to_ifmap_enqueue('publish_discovery', 1)
        self.config_log("Cassandra DB walk completed.",
            level=SandeshLevel.SYS_INFO)
        self._update_default_quota()
        end_time = datetime.datetime.utcnow()
        msg = "Time elapsed in syncing ifmap: %s" % (str(end_time - start_time))
        self.config_log(msg, level=SandeshLevel.SYS_DEBUG)
        self._db_resync_done.set()
    # end db_resync

    def wait_for_resync_done(self):
        self._db_resync_done.wait()
    # end wait_for_resync_done

    def db_check(self):
        # Read contents from cassandra and report any read exceptions
        check_results = self._object_db.walk(self._dbe_check)

        return check_results
    # end db_check

    def db_read(self):
        # Read contents from cassandra
        read_results = self._object_db.walk(self._dbe_read)
        return read_results

    # end db_check

    def _uuid_to_longs(self, id):
        msb_id = id.int >> 64
        lsb_id = id.int & ((1 << 64) - 1)
        return msb_id, lsb_id
    # end _uuid_to_longs

    def set_uuid(self, obj_type, obj_dict, id, do_lock=True):
        if do_lock:
            # set the mapping from name to uuid in zk to ensure single creator
            fq_name = obj_dict['fq_name']
            try:
                self._zk_db.create_fq_name_to_uuid_mapping(obj_type, fq_name,
                    str(id))
            except ResourceExistsError as rexist:
                # see if stale and if so delete stale
                _, ctime = self._zk_db.get_fq_name_to_uuid_mapping(
                                       obj_type, fq_name)
                epoch_msecs = ctime
                try:
                    self._object_db.uuid_to_fq_name(str(id))
                    # not stale
                    raise ResourceExistsError(fq_name, str(id), 'cassandra')
                except NoIdError:
                    lock_msecs = float(time.time()*1000 - epoch_msecs)
                    stale_msecs_cfg = 1000 * float(
                        self._api_svr_mgr.get_args().stale_lock_seconds)
                    if (lock_msecs < stale_msecs_cfg):
                        # not stale, race in create
                        raise rexist

                    # stale, release old and create new lock
                    msg = 'Releasing stale lock(%s sec) for %s %s' \
                        %(float(lock_msecs)/1000, obj_type, fq_name)
                    self.config_log(msg, level=SandeshLevel.SYS_NOTICE)
                    self._zk_db.delete_fq_name_to_uuid_mapping(
                        obj_type, fq_name)
                    self._zk_db.create_fq_name_to_uuid_mapping(
                        obj_type, fq_name, str(id))
        # end do_lock

        # set uuid in the perms meta
        mslong, lslong = self._uuid_to_longs(id)
        obj_dict['id_perms']['uuid'] = {}
        obj_dict['id_perms']['uuid']['uuid_mslong'] = mslong
        obj_dict['id_perms']['uuid']['uuid_lslong'] = lslong

        obj_dict['uuid'] = str(id)

        return True
    # end set_uuid

    def _alloc_set_uuid(self, obj_type, obj_dict):
        id = uuid.uuid4()
        ok = self.set_uuid(obj_type, obj_dict, id)

        return (ok, obj_dict['uuid'])
    # end _alloc_set_uuid

    def match_uuid(self, obj_dict, obj_uuid):
        new_mslong, new_lslong = self._uuid_to_longs(uuid.UUID(obj_uuid))
        old_mslong = obj_dict['id_perms']['uuid']['uuid_mslong']
        old_lslong = obj_dict['id_perms']['uuid']['uuid_lslong']
        if new_mslong == old_mslong and new_lslong == old_lslong:
            return True

        return False
    # end match_uuid

    def update_subnet_uuid(self, subnets):
        updated = False
        if subnets is None:
            return updated

        for subnet in subnets:
            if subnet.get('subnet_uuid'):
                continue
            subnet_uuid = str(uuid.uuid4())
            subnet['subnet_uuid'] = subnet_uuid
            updated = True

        return updated
    # end update_subnet_uuid

    def update_bgp_router_type(self, obj_dict):
        """ Sets router_type property based on the vendor property only
        if router_type is not set.
        """
        router_params = obj_dict['bgp_router_parameters']
        if 'router_type' not in router_params:
            router_type = 'router'
            if router_params['vendor'] == 'contrail':
                router_type = 'control-node'
            router_params.update({'router_type': router_type})
            obj_uuid = obj_dict.get('uuid')
            self._object_db.object_update('bgp_router', obj_uuid, obj_dict)
    # end update_bgp_router_type

    def iip_update_subnet_uuid(self, iip_dict):
        """ Set the subnet uuid as instance-ip attribute """
        for vn_ref in iip_dict.get('virtual_network_refs', []):
            (ok, results) = self._object_db.object_read(
                'virtual_network', [vn_ref['uuid']],
                field_names=['network_ipam_refs'])
            if not ok:
                return
            vn_dict = results[0]
            for ipam in vn_dict.get('network_ipam_refs', []):
                subnets = ipam['attr']['ipam_subnets']
                for subnet in subnets:
                    pfx = subnet['subnet']['ip_prefix']
                    pfx_len = subnet['subnet']['ip_prefix_len']
                    cidr = '%s/%s' % (pfx, pfx_len)
                    if (IPAddress(iip_dict['instance_ip_address']) in
                            IPNetwork(cidr)):
                        iip_dict['subnet_uuid'] = subnet['subnet_uuid']
                        self._object_db.object_update('instance-ip',
                                                         iip_dict['uuid'],
                                                         iip_dict)
                        return

    def _dbe_resync(self, obj_type, obj_uuids):
        obj_class = cfgm_common.utils.obj_type_to_vnc_class(obj_type, __name__)
        obj_fields = list(obj_class.prop_fields) + list(obj_class.ref_fields)
        (ok, obj_dicts) = self._object_db.object_read(
                               obj_type, obj_uuids, field_names=obj_fields)
        uve_trace_list = []
        for obj_dict in obj_dicts:
            try:
                obj_uuid = obj_dict['uuid']
                uve_trace_list.append(("RESYNC", obj_type, obj_uuid, obj_dict))

                if obj_type == 'virtual_network':
                    # TODO remove backward compat (use RT instead of VN->LR ref)
                    for router in obj_dict.get('logical_router_refs', []):
                        self._object_db._delete_ref(None,
                                                       obj_type,
                                                       obj_uuid,
                                                       'logical_router',
                                                       router['uuid'])
                    if 'network_ipam_refs' in obj_dict:
                        ipam_refs = obj_dict['network_ipam_refs']
                        do_update = False
                        for ipam in ipam_refs:
                            vnsn = ipam['attr']
                            ipam_subnets = vnsn['ipam_subnets']
                            if (self.update_subnet_uuid(ipam_subnets)):
                                if not do_update:
                                    do_update = True
                        if do_update:
                            self.cassandra_db.object_update(
                                'virtual_network', obj_uuid, obj_dict)

                elif obj_type == 'virtual_machine_interface':
                    device_owner = obj_dict.get('virtual_machine_interface_device_owner')
                    li_back_refs = obj_dict.get('logical_interface_back_refs', [])
                    if not device_owner and li_back_refs:
                        obj_dict['virtual_machine_interface_device_owner'] = 'PhysicalRouter'
                        self._object_db.object_update('virtual_machine_interface',
                                                    obj_uuid, obj_dict)

                # create new perms if upgrading
                perms2 = obj_dict.get('perms2')
                if perms2 is None:
                    perms2 = self._object_db.update_perms2(obj_uuid)
                if obj_type == 'domain' and len(perms2['share']) == 0:
                    self._object_db.enable_domain_sharing(obj_uuid, perms2)

                if (obj_type == 'bgp_router' and
                        'bgp_router_parameters' in obj_dict and
                        'router_type' not in obj_dict['bgp_router_parameters']):
                    self.update_bgp_router_type(obj_dict)

                if obj_type == 'instance_ip' and 'subnet_uuid' not in obj_dict:
                    self.iip_update_subnet_uuid(obj_dict)

                # Ifmap alloc
                parent_res_type = obj_dict.get('parent_type', None)
                (ok, result) = self._ifmap_db.object_alloc(
                    obj_class, parent_res_type, obj_dict['fq_name'])
                if not ok:
                    msg = "%s(%s), dbe_resync:ifmap_alloc error: %s" % (
                            obj_type, obj_uuid, result[1])
                    self.config_log(msg, level=SandeshLevel.SYS_ERR)
                    continue
                (my_imid, parent_imid) = result

                # Ifmap create
                obj_ids = {'type': obj_type, 'uuid': obj_uuid, 'imid': my_imid,
                           'parent_imid': parent_imid}
                (ok, result) = self._ifmap_db.object_create(obj_ids, obj_dict)
            except Exception as e:
                tb = cfgm_common.utils.detailed_traceback()
                self.config_log(tb, level=SandeshLevel.SYS_ERR)
                continue
        # end for all objects

        # Send UVEs resync with a pool of workers
        uve_workers = gevent.pool.Group()
        def format_args_for_dbe_uve_trace(args):
            return self.dbe_uve_trace(*args)
        uve_workers.map(format_args_for_dbe_uve_trace, uve_trace_list)
    # end _dbe_resync


    def _dbe_check(self, obj_type, obj_uuids):
        for obj_uuid in obj_uuids:
            try:
                (ok, obj_dict) = self._object_db.object_read(obj_type, [obj_uuid])
            except Exception as e:
                return {'uuid': obj_uuid, 'type': obj_type, 'error': str(e)}
     # end _dbe_check

    def _dbe_read(self, obj_type, obj_uuids):
        results = []
        for obj_uuid in obj_uuids:
            try:
                (ok, obj_dict) = self._object_db.object_read(obj_type, [obj_uuid])
                result_dict = obj_dict[0]
                result_dict['type'] = obj_type
                result_dict['uuid'] = obj_uuid
                results.append(result_dict)
            except Exception as e:
                self.config_object_error(
                    obj_uuid, None, obj_type, '_dbe_read:cassandra_read', str(e))
                continue

        return results
    # end _dbe_read

    @ignore_exceptions
    def _generate_db_request_trace(self, oper, obj_type, obj_ids, obj_dict):
        req_id = get_trace_id()

        body = dict(obj_dict)
        body['type'] = obj_type
        body.update(obj_ids)
        db_trace = DBRequestTrace(request_id=req_id)
        db_trace.operation = oper
        db_trace.body = json.dumps(body)
        return db_trace
    # end _generate_db_request_trace

    # Public Methods
    # Returns created ifmap_id
    def dbe_alloc(self, obj_type, obj_dict, uuid_requested=None):
        try:
            if uuid_requested:
                obj_uuid = uuid_requested
                ok = self.set_uuid(obj_type, obj_dict,
                                   uuid.UUID(uuid_requested), False)
            else:
                (ok, obj_uuid) = self._alloc_set_uuid(obj_type, obj_dict)
        except ResourceExistsError as e:
            return (False, (409, str(e)))

        parent_res_type = obj_dict.get('parent_type')
        obj_class = self.get_resource_class(obj_type)
        (ok, result) = VncIfmapClient.object_alloc(obj_class, parent_res_type,
                                                   obj_dict['fq_name'])
        if not ok:
            self.dbe_release(obj_type, obj_dict['fq_name'])
            return False, result

        (my_imid, parent_imid) = result
        obj_ids = {
            'uuid': obj_dict['uuid'],
            'imid': my_imid, 'parent_imid': parent_imid}

        return (True, obj_ids)
    # end dbe_alloc

    def dbe_uve_trace(self, oper, type, uuid, obj_dict):
        if type not in self._UVEMAP:
            return

        if type == 'bgp_router':
            if 'bgp_router_parameters' not in obj_dict or\
               'router_type' not in obj_dict['bgp_router_parameters'] or\
               obj_dict['bgp_router_parameters']['router_type'] != 'control-node':
                return

        oper = oper.upper()
        req_id = get_trace_id()
        if 'fq_name' not in obj_dict:
            obj_dict['fq_name'] = self.uuid_to_fq_name(uuid)
        obj_json = {k: json.dumps(obj_dict[k]) for k in obj_dict or {}}

        db_trace = DBRequestTrace(request_id=req_id)
        db_trace.operation = oper
        db_trace.body = "name=%s type=%s value=%s" % (obj_dict['fq_name'],
                                                      type,
                                                      json.dumps(obj_dict))
        uve_table, global_uve = self._UVEMAP[type]
        if global_uve:
            uve_name = obj_dict['fq_name'][-1]
        else:
            uve_name = ':'.join(obj_dict['fq_name'])
        contrail_config = ContrailConfig(name=uve_name,
                                         elements=obj_json,
                                         deleted=oper=='DELETE')
        contrail_config_msg = ContrailConfigTrace(data=contrail_config,
                                                  table=uve_table,
                                                  sandesh=self._sandesh)

        contrail_config_msg.send(sandesh=self._sandesh)
        trace_msg([db_trace], 'DBUVERequestTraceBuf', self._sandesh)

    def dbe_trace(oper):
        def wrapper1(func):
            def wrapper2(self, obj_type, obj_ids, obj_dict):
                trace = self._generate_db_request_trace(oper, obj_type,
                                                        obj_ids, obj_dict)
                try:
                    ret = func(self, obj_type, obj_ids, obj_dict)
                    trace_msg([trace], 'DBRequestTraceBuf',
                              self._sandesh)
                    return ret
                except Exception as e:
                    trace_msg([trace], 'DBRequestTraceBuf',
                              self._sandesh, error_msg=str(e))
                    raise

            return wrapper2
        return wrapper1
    # dbe_trace

    # create/update indexes if object is shared
    def build_shared_index(oper):
        def wrapper1(func):
            def wrapper2(self, obj_type, obj_ids, obj_dict):

                obj_uuid = obj_ids['uuid']
                # fetch current share information to identify what might have changed
                try:
                    cur_perms2 = self.uuid_to_obj_perms2(obj_uuid)
                except Exception as e:
                    cur_perms2 = self.get_default_perms2()
                    pass

                # don't build sharing indexes if operation (create/update) failed
                (ok, result) = func(self, obj_type, obj_ids, obj_dict)
                if not ok:
                    return (ok, result)

                # many updates don't touch perms2
                new_perms2 = obj_dict.get('perms2', None)
                if not new_perms2:
                    return (ok, result)

                share_perms = new_perms2['share']
                global_access = new_perms2['global_access']

                # msg = 'RBAC: BSL perms new %s, cur %s' % (new_perms2, cur_perms2)
                # self.config_log(msg, level=SandeshLevel.SYS_NOTICE)

                # change in global access?
                if cur_perms2['global_access'] != global_access:
                    if global_access:
                        self._object_db.set_shared(obj_type, obj_uuid, rwx = global_access)
                    else:
                        self._object_db.del_shared(obj_type, obj_uuid)

                # change in shared list? Construct temporary sets to compare
                cur_shared_list = set(item['tenant']+':'+str(item['tenant_access']) for item in cur_perms2['share'])
                new_shared_list = set(item['tenant']+':'+str(item['tenant_access']) for item in new_perms2['share'])
                if cur_shared_list == new_shared_list:
                    return (ok, result)

                # delete sharing if no longer in shared list
                for share_info in cur_shared_list - new_shared_list:
                    # sharing information => [share-type, uuid, rwx bits]
                    (share_type, share_id, share_perms)  = cfgm_common.utils.shareinfo_from_perms2(share_info)
                    self._object_db.del_shared(obj_type, obj_uuid,
                        share_id = share_id, share_type = share_type)

                # share this object with specified tenants
                for share_info in new_shared_list - cur_shared_list:
                    # sharing information => [share-type, uuid, rwx bits]
                    (share_type, share_id, share_perms)  = cfgm_common.utils.shareinfo_from_perms2(share_info)
                    self._object_db.set_shared(obj_type, obj_uuid,
                        share_id = share_id, share_type = share_type, rwx = int(share_perms))

                return (ok, result)
            return wrapper2
        return wrapper1

    @dbe_trace('create')
    @build_shared_index('create')
    def dbe_create(self, obj_type, obj_ids, obj_dict):
        (ok, result) = self._object_db.object_create(
            obj_type, obj_ids['uuid'], obj_dict)

        if ok:
            # publish to ifmap via msgbus
            self._msgbus.dbe_create_publish(obj_type, obj_ids, obj_dict)

        return (ok, result)
    # end dbe_create

    # input id is ifmap-id + uuid
    def dbe_read(self, obj_type, obj_ids, obj_fields=None,
                 ret_readonly=False):
        try:
            (ok, cassandra_result) = self._object_db.object_read(
                obj_type, [obj_ids['uuid']], obj_fields,
                ret_readonly=ret_readonly)
        except NoIdError as e:
            # if NoIdError is for obj itself (as opposed to say for parent
            # or ref), let caller decide if this can be handled gracefully
            # by re-raising
            if e._unknown_id == obj_ids['uuid']:
                raise

            return (False, str(e))

        return (ok, cassandra_result[0])
    # end dbe_read

    def dbe_count_children(self, obj_type, obj_id, child_type):
        try:
            (ok, cassandra_result) = self._object_db.object_count_children(
                obj_type, obj_id, child_type)
        except NoIdError as e:
            return (False, str(e))

        return (ok, cassandra_result)
    # end dbe_count_children

    def dbe_read_multi(self, obj_type, obj_ids_list, obj_fields=None):
        if not obj_ids_list:
            return (True, [])

        try:
            (ok, cassandra_result) = self._object_db.object_read(
                obj_type, [obj_id['uuid'] for obj_id in obj_ids_list],
                obj_fields, ret_readonly=True)
        except NoIdError as e:
            return (False, str(e))

        return (ok, cassandra_result)
    # end dbe_read_multi

    def dbe_get_relaxed_refs(self, obj_id):
        return self._object_db.get_relaxed_refs(obj_id)
    # end dbe_get_relaxed_refs

    def dbe_is_latest(self, obj_ids, tstamp):
        try:
            is_latest = self._object_db.is_latest(obj_ids['uuid'], tstamp)
            return (True, is_latest)
        except Exception as e:
            return (False, str(e))
    # end dbe_is_latest

    @dbe_trace('update')
    @build_shared_index('update')
    def dbe_update(self, obj_type, obj_ids, new_obj_dict):
        (ok, cassandra_result) = self._object_db.object_update(
            obj_type, obj_ids['uuid'], new_obj_dict)

        # publish to ifmap via message bus (rabbitmq)
        self._msgbus.dbe_update_publish(obj_type, obj_ids)

        return (ok, cassandra_result)
    # end dbe_update

    def _owner_id(self):
        env = get_request().headers.environ
        tenant_uuid = env.get('HTTP_X_PROJECT_ID')
        domain = env.get('HTTP_X_DOMAIN_ID')
        if domain is None:
            domain = env.get('HTTP_X_USER_DOMAIN_ID')
            try:
                domain = str(uuid.UUID(domain))
            except ValueError:
                if domain == 'default':
                    domain = 'default-domain'
                domain = self._db_conn.fq_name_to_uuid('domain', [domain])
        if domain:
            domain = domain.replace('-','')
        return domain, tenant_uuid

    def dbe_list_rdbms(self, obj_type, parent_uuids=None, back_ref_uuids=None,
                 obj_uuids=None, is_count=False, filters=None,
                 paginate_start=None, paginate_count=None, is_detail=False,
                 field_names=None, include_shared=False):
        domain = None
        tenant_id = None
        if include_shared:
            domain, tenant_id = self._owner_id()

        return self._object_db.object_list(
                 obj_type, parent_uuids=parent_uuids,
                 back_ref_uuids=back_ref_uuids, obj_uuids=obj_uuids,
                 count=is_count, filters=filters, is_detail=is_detail, field_names=field_names, tenant_id=tenant_id, domain=domain)

    def dbe_list(self, obj_type, parent_uuids=None, back_ref_uuids=None,
                 obj_uuids=None, is_count=False, filters=None,
                 paginate_start=None, paginate_count=None, is_detail=False,
                 field_names=None, include_shared=False):
        if self._db_engine == 'rdbms':
            return self.dbe_list_rdbms(obj_type, parent_uuids, back_ref_uuids,
                 obj_uuids, is_count, filters,
                 paginate_start, paginate_count, is_detail,
                 field_names, include_shared)
        (ok, result) = self._object_db.object_list(
                 obj_type, parent_uuids=parent_uuids,
                 back_ref_uuids=back_ref_uuids, obj_uuids=obj_uuids,
                 count=is_count, filters=filters)

        obj_class = cfgm_common.utils.obj_type_to_vnc_class(obj_type, __name__)
        if not ok or is_count:
            return (ok, result)

        # include objects shared with tenant
        if include_shared:
            domain, tenant_uuid = self._owner_id()
            shares = self.get_shared_objects(obj_type, tenant_uuid, domain)
            owned_objs = set([obj_uuid for (fq_name, obj_uuid) in result])
            for (obj_uuid, obj_perm) in shares:
                # skip owned objects already included in results
                if obj_uuid in owned_objs:
                    continue
                try:
                    fq_name = self.uuid_to_fq_name(obj_uuid)
                    result.append((fq_name, obj_uuid))
                except NoIdError:
                    # uuid no longer valid. Delete?
                    pass
        # end shared

        if is_detail:
            obj_fields = list(obj_class.prop_fields) + \
                         list(obj_class.ref_fields)
        else:
            obj_fields = [u'id_perms']

        if field_names:
            obj_fields.extend(field_names)

        obj_ids_list = [{'uuid': obj_uuid} for _, obj_uuid in result]
        return self.dbe_read_multi(obj_type, obj_ids_list, obj_fields)

    # end dbe_list

    @dbe_trace('delete')
    def dbe_delete(self, obj_type, obj_ids, obj_dict):
        (ok, cassandra_result) = self._object_db.object_delete(
            obj_type, obj_ids['uuid'])

        # publish to ifmap via message bus (rabbitmq)
        self._msgbus.dbe_delete_publish(obj_type, obj_ids, obj_dict)

        # finally remove mapping in zk
        fq_name = get_fq_name_from_ifmap_id(obj_ids['imid'])
        self.dbe_release(obj_type, fq_name)

        return ok, cassandra_result
    # end dbe_delete

    def dbe_release(self, obj_type, obj_fq_name):
        self._zk_db.delete_fq_name_to_uuid_mapping(obj_type, obj_fq_name)
    # end dbe_release

    def dbe_oper_publish_pending(self):
        return self._msgbus.dbe_oper_publish_pending()
    # end dbe_oper_publish_pending

    def useragent_kv_store(self, key, value):
        self._object_db.useragent_kv_store(key, value)
    # end useragent_kv_store

    def useragent_kv_retrieve(self, key):
        return self._object_db.useragent_kv_retrieve(key)
    # end useragent_kv_retrieve

    def useragent_kv_delete(self, key):
        return self._object_db.useragent_kv_delete(key)
    # end useragent_kv_delete

    def subnet_is_addr_allocated(self, subnet, addr):
        return self._zk_db.subnet_is_addr_allocated(subnet, addr)
    # end subnet_is_addr_allocated

    def subnet_set_in_use(self, subnet, addr):
        return self._zk_db.subnet_set_in_use(subnet, addr)
    # end subnet_set_in_use

    def subnet_reset_in_use(self, subnet, addr):
        return self._zk_db.subnet_reset_in_use(subnet, addr)
    #end subnet_reset_in_use

    def subnet_alloc_count(self, subnet):
        return self._zk_db.subnet_alloc_count(subnet)
    # end subnet_alloc_count

    def subnet_alloc_req(self, subnet, value=None):
        return self._zk_db.subnet_alloc_req(subnet, value)
    # end subnet_alloc_req

    def subnet_reserve_req(self, subnet, addr=None, value=None):
        return self._zk_db.subnet_reserve_req(subnet, addr, value)
    # end subnet_reserve_req

    def subnet_free_req(self, subnet, addr):
        return self._zk_db.subnet_free_req(subnet, addr)
    # end subnet_free_req

    def subnet_create_allocator(self, subnet, subnet_alloc_list,
                                addr_from_start, should_persist,
                                start_subnet, size, alloc_unit):
        return self._zk_db.create_subnet_allocator(subnet,
                               subnet_alloc_list, addr_from_start,
                               should_persist, start_subnet, size, alloc_unit)
    # end subnet_create_allocator

    def subnet_delete_allocator(self, subnet):
        return self._zk_db.delete_subnet_allocator(subnet)
    # end subnet_delete_allocator

    def uuid_vnlist(self):
        return self._object_db.uuid_vnlist()
    # end uuid_vnlist

    def uuid_to_ifmap_id(self, res_type, id):
        fq_name = self.uuid_to_fq_name(id)
        return get_ifmap_id_from_fq_name(res_type, fq_name)
    # end uuid_to_ifmap_id

    def fq_name_to_uuid(self, obj_type, fq_name):
        obj_uuid = self._object_db.fq_name_to_uuid(obj_type, fq_name)
        return obj_uuid
    # end fq_name_to_uuid

    def uuid_to_fq_name(self, obj_uuid):
        return self._object_db.uuid_to_fq_name(obj_uuid)
    # end uuid_to_fq_name

    def uuid_to_obj_type(self, obj_uuid):
        return self._object_db.uuid_to_obj_type(obj_uuid)
    # end uuid_to_obj_type

    def uuid_to_obj_dict(self, obj_uuid):
        return self._object_db.uuid_to_obj_dict(obj_uuid)
    # end uuid_to_obj_dict

    def uuid_to_obj_perms(self, obj_uuid):
        return self._object_db.uuid_to_obj_perms(obj_uuid)
    # end uuid_to_obj_perms

    def prop_collection_get(self, obj_type, obj_uuid, obj_fields, position):
        (ok, cassandra_result) = self._object_db.prop_collection_read(
            obj_type, obj_uuid, obj_fields, position)
        return ok, cassandra_result
    # end prop_collection_get

    def prop_collection_update(self, obj_type, obj_uuid, updates):
        if not updates:
            return

        self._object_db.prop_collection_update(obj_type, obj_uuid, updates)
        self._msgbus.dbe_update_publish(obj_type, {'uuid': obj_uuid})
        return True, ''
    # end prop_collection_update

    def ref_update(self, obj_type, obj_uuid, ref_obj_type, ref_uuid, ref_data,
                   operation):
        self._object_db.ref_update(obj_type, obj_uuid, ref_obj_type,
                                      ref_uuid, ref_data, operation)
        self._msgbus.dbe_update_publish(obj_type, {'uuid': obj_uuid})
    # ref_update

    def ref_relax_for_delete(self, obj_uuid, ref_uuid):
        self._object_db.ref_relax_for_delete(obj_uuid, ref_uuid)
    # end ref_relax_for_delete

    def uuid_to_obj_perms2(self, obj_uuid):
        return self._object_db.uuid_to_obj_perms2(obj_uuid)
    # end uuid_to_obj_perms2


    def get_resource_class(self, type):
        return self._api_svr_mgr.get_resource_class(type)
    # end get_resource_class

    def get_default_perms2(self):
        return self._api_svr_mgr._get_default_perms2()

    # Helper routines for REST
    def generate_url(self, resource_type, obj_uuid):
        return self._api_svr_mgr.generate_url(resource_type, obj_uuid)
    # end generate_url

    def config_object_error(self, id, fq_name_str, obj_type,
                            operation, err_str):
        self._api_svr_mgr.config_object_error(
            id, fq_name_str, obj_type, operation, err_str)
    # end config_object_error

    def config_log(self, msg, level):
        self._api_svr_mgr.config_log(msg, level)
    # end config_log

    def get_server_port(self):
        return self._api_svr_mgr.get_server_port()
    # end get_server_port

    # return all objects shared with us (tenant)
    # useful for collections
    def get_shared_objects(self, obj_type, tenant_uuid, domain_uuid):
        shared = []
        # specifically shared with us
        if tenant_uuid:
            l1 = self._object_db.get_shared(obj_type, share_id = tenant_uuid, share_type = 'tenant')
            if l1:
                shared.extend(l1)

        # shared at domain level
        if domain_uuid:
            l1 = self._object_db.get_shared(obj_type, share_id = domain_uuid, share_type = 'domain')
            if l1:
                shared.extend(l1)

        # globally shared
        l2 = self._object_db.get_shared(obj_type)
        if l2:
            shared.extend(l2)

        return shared
    # end get_shared_objects

    def reset(self):
        if self.get_worker_id() == 0:
            self._ifmap_db.reset()
        self._msgbus.reset()
    # end reset

    def get_worker_id(self):
        return self._api_svr_mgr.get_worker_id()
    # end get_worker_id

    def get_autonomous_system(self):
        config_uuid = self.fq_name_to_uuid('global_system_config', ['default-global-system-config'])
        ok, config = self._object_db.object_read('global_system_config', [config_uuid])
        global_asn = config[0]['autonomous_system']
        return global_asn


# end class VncDbClient