Add support for diff calculation in dependency tracker

It is possible that processing an object can take a long time in schema
transformer. During this time, many updates to the same objects could
be received from rabbitmq. All those changes would be processed with the
one update only. We should have a way to ignore the updates where no
change is detected.

With this change, we are adding support for specifying ref/prop fields
for each object type and on update, we compare these fields from
previously cached values. If no change is detected, we terminate the
dependency tracker process.

Closes-Bug: 1674514

Change-Id: I0b1244e5473ded2f085874f2f6bcdade74a5fe63

src/api-lib/vnc_api.py

@@ -42,13 +42,6 @@ def wrapper(self, *args, **kwargs):
         return func(self, *args, **kwargs)
     return wrapper
 
-def compare_refs(old_refs, new_refs):
-    # compare refs in an object
-    old_ref_dict = dict((':'.join(ref['to']), ref['attr']) for ref in old_refs or [])
-    new_ref_dict = dict((':'.join(ref['to']), ref['attr']) for ref in new_refs or [])
-    return old_ref_dict == new_ref_dict
-# end compare_refs
-
 def get_object_class(res_type):
     cls_name = '%s' %(utils.CamelCase(res_type))
     return utils.str_to_class(cls_name, __name__)

src/config/common/utils.py

@@ -178,3 +178,9 @@ def shareinfo_from_perms2(field):
     return x
 # end
 
+def compare_refs(old_refs, new_refs):
+    # compare refs in an object
+    old_ref_dict = dict((':'.join(ref['to']), ref.get('attr')) for ref in old_refs or [])
+    new_ref_dict = dict((':'.join(ref['to']), ref.get('attr')) for ref in new_refs or [])
+    return old_ref_dict == new_ref_dict
+# end compare_refs

src/config/common/vnc_amqp.py

@@ -38,8 +38,8 @@ def msgbus_store_err_msg(self, msg):
         self.msg_tracer.error = msg
 
     def msgbus_trace_msg(self):
-            self.msg_tracer.trace_msg(name='MessageBusNotifyTraceBuf',
-                                      sandesh=self.logger._sandesh)
+        self.msg_tracer.trace_msg(name='MessageBusNotifyTraceBuf',
+                                  sandesh=self.logger._sandesh)
 
     def _vnc_subscribe_callback(self, oper_info):
         self._db_resync_done.wait()
@@ -137,7 +137,11 @@ def handle_update(self):
             return
 
         try:
-            self.obj.update()
+            if self.obj.update() == False:
+                # If update returns a False it indicates nothing has changed.
+                # If it returns True or None, then some change was detected.
+                # If no change, then terminate dependency tracker
+                return
         except NoIdError:
             obj_id = self.oper_info['uuid']
             self.logger.warning('%s uuid %s update caused NoIdError' %

src/config/common/vnc_db.py

@@ -7,7 +7,8 @@
 """
 from exceptions import NoIdError
 from vnc_api.gen.resource_client import *
-from utils import obj_type_to_vnc_class
+from utils import obj_type_to_vnc_class, compare_refs
+
 
 class DBBase(object):
     # This is the base class for all DB objects. All derived objects must
@@ -191,6 +192,7 @@ def get_single_ref_attr(self, ref_type, obj):
         return None
     # end get_single_ref_attr
 
+    # Update a single ref. Return True if any update was made
     def update_single_ref(self, ref_type, obj):
         if isinstance(obj, dict):
             refs = obj.get(ref_type+'_refs') or obj.get(ref_type+'_back_refs')
@@ -204,14 +206,17 @@ def update_single_ref(self, ref_type, obj):
             new_key = None
         old_key = getattr(self, ref_type, None)
         if old_key == new_key:
-            return
-        ref_obj = self.get_obj_type_map()[ref_type].get(old_key)
-        if ref_obj is not None:
-            ref_obj.delete_ref(self.obj_type, self.get_key())
-        ref_obj = self.get_obj_type_map()[ref_type].get(new_key)
-        if ref_obj is not None:
-            ref_obj.add_ref(self.obj_type, self.get_key())
+            return False
+        ref_cls = self.get_obj_type_map().get(ref_type)
+        if ref_cls:
+            ref_obj = ref_cls.get(old_key)
+            if ref_obj is not None:
+                ref_obj.delete_ref(self.obj_type, self.get_key())
+            ref_obj = ref_cls.get(new_key)
+            if ref_obj is not None:
+                ref_obj.add_ref(self.obj_type, self.get_key())
         setattr(self, ref_type, new_key)
+        return True
     # end update_single_ref
 
     def set_children(self, ref_type, obj):
@@ -226,6 +231,7 @@ def set_children(self, ref_type, obj):
         setattr(self, ref_type+'s', new_refs)
     # end set_children
 
+    # Update a multiple refs. Return True if any update was made
     def update_multiple_refs(self, ref_type, obj):
         if isinstance(obj, dict):
             refs = obj.get(ref_type+'_refs') or obj.get(ref_type+'_back_refs')
@@ -238,6 +244,8 @@ def update_multiple_refs(self, ref_type, obj):
             new_key = self._get_ref_key(ref, ref_type)
             new_refs.add(new_key)
         old_refs = getattr(self, ref_type+'s')
+        if old_refs == new_refs:
+            return False
         for ref_key in old_refs - new_refs:
             ref_obj = self.get_obj_type_map()[ref_type].get(ref_key)
             if ref_obj is not None:
@@ -247,8 +255,15 @@ def update_multiple_refs(self, ref_type, obj):
             if ref_obj is not None:
                 ref_obj.add_ref(self.obj_type, self.get_key())
         setattr(self, ref_type+'s', new_refs)
+        return True
     # end update_multiple_refs
 
+    def update_refs(self, ref_type, obj):
+        if hasattr(self, ref_type):
+            return self.update_single_ref(ref_type, obj)
+        elif isinstance(getattr(self, ref_type+'s', None), set):
+            return self.update_multiple_refs(ref_type, obj)
+
     def update_multiple_refs_with_attr(self, ref_type, obj):
         if isinstance(obj, dict):
             refs = obj.get(ref_type+'_refs') or obj.get(ref_type+'_back_refs')
@@ -261,17 +276,21 @@ def update_multiple_refs_with_attr(self, ref_type, obj):
             new_key = self._get_ref_key(ref, ref_type)
             new_refs[new_key] = ref.get('attr')
         old_refs = getattr(self, ref_type+'s')
+        update = False
         for ref_key in set(old_refs.keys()) - set(new_refs.keys()):
+            update = True
             ref_obj = self.get_obj_type_map()[ref_type].get(ref_key)
             if ref_obj is not None:
                 ref_obj.delete_ref(self.obj_type, self.get_key())
         for ref_key in new_refs:
             if ref_key in old_refs and new_refs[ref_key] == old_refs[ref_key]:
                 continue
+            update = True
             ref_obj = self.get_obj_type_map()[ref_type].get(ref_key)
             if ref_obj is not None:
                 ref_obj.add_ref(self.obj_type, self.get_key(), new_refs[ref_key])
         setattr(self, ref_type+'s', new_refs)
+        return update
     # end update_multiple_refs
 
     @classmethod
@@ -305,6 +324,36 @@ def read_vnc_obj(cls, uuid=None, fq_name=None, obj_type=None, fields=None):
         return obj
     # end read_vnc_obj
 
+    def update_vnc_obj(self, obj=None):
+        if obj:
+            old_obj = None
+            self.obj = obj
+        else:
+            old_obj = getattr(self, 'obj', None)
+            uuid = getattr(self, 'uuid', None)
+            if uuid:
+                self.obj = self.read_vnc_obj(uuid=uuid)
+            else:
+                self.obj = self.read_vnc_obj(fq_name=self.name)
+
+        changed = []
+        for field in self.ref_fields or []:
+            old_field = getattr(old_obj, field+'_refs', None)
+            new_field = getattr(self.obj, field+'_refs', None)
+            if compare_refs(old_field, new_field):
+                continue
+            self.update_refs(field, self.obj)
+            changed.append(field)
+        for field in self.prop_fields or []:
+            old_field = getattr(old_obj, field, None)
+            new_field = getattr(self.obj, field, None)
+            if old_field == new_field:
+                continue
+            if hasattr(self, field):
+                setattr(self, field, new_field)
+            changed.append(field)
+        return changed
+
     @classmethod
     def list_obj(cls, obj_type=None, fields=None):
         obj_type = obj_type or cls.obj_type