Provide list apis in vnc_cassandra and vnc_db

Added new apis in vnc_cassandra and vnc_db to list objects
Use these apis in device manager, service monitor and schema transformer

Change-Id: I7ea68fb888ae71457cb68465463b9c9366615705
Closes-Bug: 1460550
Partial-Bug: 1470621
(cherry picked from commit 31df7f8)

src/config/api-server/vnc_cfg_ifmap.py

@@ -739,17 +739,6 @@ def create(self, method_name, *args, **kwargs):
         return method(*args, **kwargs)
     # end create
 
-    def read(self, method_name, *args, **kwargs):
-        method = getattr(self, '_cassandra_%s_read' % (method_name))
-        return method(*args, **kwargs)
-    # end read
-
-    def count_children(self, method_name, *args, **kwargs):
-        method = getattr(self, '_cassandra_%s_count_children' % (method_name))
-        return method(*args, **kwargs)
-    # end count_children
-
-
     def update(self, method_name, *args, **kwargs):
         method = getattr(self, '_cassandra_%s_update' % (method_name))
         return method(*args, **kwargs)
@@ -760,12 +749,6 @@ def delete(self, method_name, *args, **kwargs):
         return method(*args, **kwargs)
     # end delete
 
-    def list(self, method_name, *args, **kwargs):
-        method = getattr(self, '_cassandra_%s_list' % (method_name))
-        return method(*args, **kwargs)
-    # end list
-
-
     def ref_update(self, obj_type, obj_uuid, ref_type, ref_uuid, ref_data, operation):
         bch = self._obj_uuid_cf.batch()
         if operation == 'ADD':

src/config/api-server/vnc_cfg_types.py

@@ -984,6 +984,14 @@ def _check_policy_rules(entries, network_policy_rule=False):
         if network_policy_rule:
             if rule.get('action_list') is None:
                 return (False, (400, 'Action is required'))
+        else:
+            src_sg = [addr.get('security_group') for addr in
+                      rule.get('src_addresses', [])]
+            dst_sg = [addr.get('security_group') for addr in
+                      rule.get('dst_addresses', [])]
+            if ('local' not in src_sg and 'local' not in dst_sg):
+                return (False, (400, "At least one of source or destination"
+                                     " addresses must be 'local'"))
     return True, ""
 # end _check_policy_rules
 

src/config/common/tests/test_utils.py

@@ -501,7 +501,9 @@ def call(method, body):
 
             for (oper, result, info) in _items:
                 if oper == 'update':
-                    cls._published_messages.append((result, info))
+                    search_result = deepcopy(result)
+                    search_result.tag = 'searchResult'
+                    cls._published_messages.append((search_result, info))
                 else:
                     cls._published_messages = [
                         (r,i) for (r, i) in cls._published_messages

src/config/common/vnc_cassandra.py

@@ -709,6 +709,21 @@ def _object_delete(self, res_type, obj_uuid):
         return (True, '')
     # end _object_delete
 
+    def read(self, method_name, *args, **kwargs):
+        method = getattr(self, '_cassandra_%s_read' % (method_name))
+        return method(*args, **kwargs)
+    # end read
+
+    def count_children(self, method_name, *args, **kwargs):
+        method = getattr(self, '_cassandra_%s_count_children' % (method_name))
+        return method(*args, **kwargs)
+    # end count_children
+
+    def list(self, method_name, *args, **kwargs):
+        method = getattr(self, '_cassandra_%s_list' % (method_name))
+        return method(*args, **kwargs)
+    # end list
+
     def cache_uuid_to_fq_name_add(self, id, fq_name, obj_type):
         self._cache_uuid_to_fq_name[id] = (fq_name, obj_type)
     # end cache_uuid_to_fq_name_add

src/config/common/vnc_db.py

@@ -166,9 +166,7 @@ def update_multiple_refs(self, ref_type, obj):
     # end update_multiple_refs
 
     def read_obj(self, uuid, obj_type=None):
-        method_name = "_cassandra_%s_read" % (obj_type or self.obj_type)
-        method = getattr(self._cassandra, method_name)
-        ok, objs = method([uuid])
+        ok, objs = self._cassandra.read(obj_type or self.obj_type, [uuid])
         if not ok:
             self._logger.error(
                 'Cannot read %s %s, error %s' % (obj_type, uuid, objs))
@@ -191,6 +189,28 @@ def read_vnc_obj(self, uuid=None, fq_name=None, obj_type=None):
         return obj
     # end read_vnc_obj
 
+    @classmethod
+    def list_obj(cls, obj_type=None):
+        obj_type = obj_type or cls.obj_type
+        ok, result = cls._cassandra.list(obj_type)
+        if not ok:
+            return []
+        uuids = [uuid for _, uuid in result]
+        ok, objs = cls._cassandra.read(obj_type, uuids)
+        if not ok:
+            return []
+        return objs
+
+    @classmethod
+    def list_vnc_obj(cls, obj_type=None):
+        obj_type = obj_type or cls.obj_type
+        vnc_cls = obj_type_to_vnc_class(obj_type, __name__)
+        obj_dicts = cls.list_obj(obj_type)
+        for obj_dict in obj_dicts:
+            obj = vnc_cls.from_dict(**obj_dict)
+            obj.clear_pending_updates()
+            yield obj
+
     def get_parent_uuid(self, obj):
         if 'parent_uuid' in obj:
             return obj['parent_uuid']

src/config/device-manager/device_manager/device_manager.py

@@ -180,86 +180,48 @@ def __init__(self, args=None):
                                              self.config_log)
 
         DBBaseDM.init(self, self._sandesh.logger(), self._cassandra)
-        ok, global_system_config_list = self._cassandra._cassandra_global_system_config_list()
-        if not ok:
-            self.config_log('global system config list returned error: %s' %
-                            global_system_config_list)
-        else:
-            for fq_name, uuid in global_system_config_list:
-                GlobalSystemConfigDM.locate(uuid)
-
-        ok, global_vrouter_config_list = self._cassandra._cassandra_global_vrouter_config_list()
-        if not ok:
-            self.config_log('global vrouter config list returned error: %s' %
-                            global_vrouter_config_list)
-        else:
-            for fq_name, uuid in global_vrouter_config_list:
-                GlobalVRouterConfigDM.locate(uuid)
-
-        ok, vn_list = self._cassandra._cassandra_virtual_network_list()
-        if not ok:
-            self.config_log('virtual network list returned error: %s' %
-                            vn_list)
-        else:
-            for fq_name, uuid in vn_list:
-                vn = VirtualNetworkDM.locate(uuid)
-                if vn is not None and vn.routing_instances is not None:
-                    for ri_id in vn.routing_instances:
-                        ri_obj = RoutingInstanceDM.locate(ri_id)
-
-        ok, bgp_list = self._cassandra._cassandra_bgp_router_list()
-        if not ok:
-            self.config_log('bgp router list returned error: %s' %
-                            bgp_list)
-        else:
-            for fq_name, uuid in bgp_list:
-                BgpRouterDM.locate(uuid)
-
-        ok, pr_list = self._cassandra._cassandra_physical_router_list()
-        if not ok:
-            self.config_log('physical router list returned error: %s' %
-                            pr_list)
-        else:
-            for fq_name, uuid in pr_list:
-                pr = PhysicalRouterDM.locate(uuid)
-                if pr.bgp_router:
-                    BgpRouterDM.locate(pr.bgp_router)
-                li_set = pr.logical_interfaces
-                for pi_id in pr.physical_interfaces:
-                    pi = PhysicalInterfaceDM.locate(pi_id)
-                    if pi:
-                        li_set |= pi.logical_interfaces
-                vmi_set = set()
-                for li_id in li_set:
-                    li = LogicalInterfaceDM.locate(li_id)
-                    if li and li.virtual_machine_interface:
-                        vmi_set |= set([li.virtual_machine_interface])
-                for vmi_id in vmi_set:
-                    vmi = VirtualMachineInterfaceDM.locate(vmi_id)
-
-            ok, ip_list = self._cassandra._cassandra_instance_ip_list()
-            if not ok:
-                self.config_log('instance ip list returned error: %s' %
-                            ip_list)
-            else:
-                for fq_name, uuid in ip_list:
-                    InstanceIpDM.locate(uuid)
-
-            ok, fip_list = self._cassandra._cassandra_floating_ip_list()
-            if not ok:
-                self.config_log('floating ip list returned error: %s' %
-                            fip_list)
-            else:
-                for fq_name, uuid in fip_list:
-                    FloatingIpDM.locate(uuid)
+        for obj in GlobalSystemConfigDM.list_obj():
+            GlobalSystemConfigDM.locate(obj['uuid'], obj)
+
+        for obj in GlobalVRouterConfigDM.list_obj():
+            GlobalVRouterConfigDM.locate(obj['uuid'], obj)
+
+        for obj in VirtualNetworkDM.list_obj():
+            vn = VirtualNetworkDM.locate(obj['uuid'], obj)
+            if vn is not None and vn.routing_instances is not None:
+                for ri_id in vn.routing_instances:
+                    ri_obj = RoutingInstanceDM.locate(ri_id)
+
+        for obj in BgpRouterDM.list_obj():
+            BgpRouterDM.locate(obj['uuid'], obj)
+
+        for obj in PhysicalRouterDM.list_obj():
+            pr = PhysicalRouterDM.locate(obj['uuid'], obj)
+            li_set = pr.logical_interfaces
+            for pi_id in pr.physical_interfaces:
+                pi = PhysicalInterfaceDM.locate(pi_id)
+                if pi:
+                    li_set |= pi.logical_interfaces
+            vmi_set = set()
+            for li_id in li_set:
+                li = LogicalInterfaceDM.locate(li_id)
+                if li and li.virtual_machine_interface:
+                    vmi_set |= set([li.virtual_machine_interface])
+            for vmi_id in vmi_set:
+                vmi = VirtualMachineInterfaceDM.locate(vmi_id)
+
+        for obj in InstanceIpDM.list_obj():
+            InstanceIpDM.locate(obj['uuid'], obj)
+
+        for obj in FloatingIpDM.list_obj():
+            FloatingIpDM.locate(obj['uuid'], obj)
+
+        for vn in VirtualNetworkDM.values():
+            vn.update_instance_ip_map()
+
+        for pr in PhysicalRouterDM.values():
+            pr.set_config_state()
 
-            for fq_name, uuid in vn_list:
-                vn = VirtualNetworkDM.locate(uuid)
-                if vn is not None:
-                    vn.update_instance_ip_map()
-
-            for pr in PhysicalRouterDM.values():
-                pr.set_config_state()
         self._db_resync_done.set()
         while 1:
             # Just wait indefinitely

src/config/schema-transformer/to_bgp.py

@@ -218,7 +218,7 @@ def __init__(self, name, obj=None, acl_dict=None, ri_dict=None):
                     if remote_ri_fq_name[-1] == remote_ri_fq_name[-2]:
                         self.connections.add(':'.join(remote_ri_fq_name[0:-1] ))
 
-        for ri in self.obj.get_routing_instances() or []:
+        for ri in getattr(self.obj, 'routing_instances', []):
             ri_name = ri['to'][-1]
             if ri_name not in self.rinst:
                 sc_id = self._get_service_id_from_ri(ri_name)
@@ -2573,16 +2573,12 @@ def config_log(self, msg, level):
 
     # Clean up stale objects
     def reinit(self):
-        lr_list = _vnc_lib.logical_routers_list(detail=True)
-        for lr in lr_list:
+        for lr in LogicalRouterST.list_vnc_obj():
            LogicalRouterST.locate(lr.get_fq_name_str(), lr)
-        vn_list = _vnc_lib.virtual_networks_list(detail=True,
-                                                 fields=['routing_instances',
-                                                         'access_control_lists'])
+        vn_list = list(VirtualNetworkST.list_vnc_obj())
         vn_id_list = [vn.uuid for vn in vn_list]
-        ri_list = _vnc_lib.routing_instances_list(detail=True)
         ri_dict = {}
-        for ri in ri_list:
+        for ri in DBBaseST.list_vnc_obj('routing_instance'):
             delete = False
             if ri.parent_uuid not in vn_id_list:
                 delete = True
@@ -2607,13 +2603,11 @@ def reinit(self):
 
         # end for ri
 
-        sg_list = _vnc_lib.security_groups_list(detail=True,
-                                                fields=['access_control_lists'])
+        sg_list = list(SecurityGroupST.list_vnc_obj())
         sg_id_list = [sg.uuid for sg in sg_list]
-        acl_list = _vnc_lib.access_control_lists_list(detail=True)
         sg_acl_dict = {}
         vn_acl_dict = {}
-        for acl in acl_list or []:
+        for acl in DBBaseST.list_vnc_obj('access_control_list'):
             delete = False
             if acl.parent_type == 'virtual-network':
                 if acl.parent_uuid in vn_id_list:
@@ -2643,16 +2637,14 @@ def reinit(self):
         for sg in sg_list:
             SecurityGroupST.locate(sg.get_fq_name_str(), sg, sg_acl_dict)
         gevent.sleep(0.001)
-        rt_list = _vnc_lib.route_targets_list()['route-targets']
-        for rt in rt_list:
-            rt_name = ':'.join(rt['fq_name'])
+        for rt in RouteTargetST.list_vnc_obj():
+            rt_name = rt.get_fq_name_str()
             RouteTargetST.locate(rt_name, RouteTarget(rt_name))
         for vn in vn_list:
             VirtualNetworkST.locate(vn.get_fq_name_str(), vn, vn_acl_dict,
                                     ri_dict)
         gevent.sleep(0.001)
-        vmi_list = _vnc_lib.virtual_machine_interfaces_list(detail=True)
-        for vmi in vmi_list:
+        for vmi in VirtualMachineInterfaceST.list_vnc_obj():
             VirtualMachineInterfaceST.locate(vmi.get_fq_name_str(), vmi)
     # end reinit
 

src/config/svc-monitor/svc_monitor/config_db.py

@@ -783,7 +783,7 @@ def delete(cls, uuid):
 
 class ServiceApplianceSetSM(DBBaseSM):
     _dict = {}
-    obj_type = 'service_appliance'
+    obj_type = 'service_appliance_set'
 
     def __init__(self, uuid, obj_dict=None):
         self.uuid = uuid