Scaled service delete issue with static routes

Currently there is an issue when static routes are configured on an interface
of a service instance. The deletion is not handled properly when a service
instance is deleted. With scaling there are multiple service instance interfaces
referring to the same static route table when scaling is enabled for the
service instance. The delete of a neutron port returns an error because the
interface route table still has references to other ports of the scaled
instances. Hence heat stack delete for scaled service instances with static
routing showed stale VMs in error state.

The fix is to ignore the reference error since the last neutron port cleanup
will remove the static route table. Hence neutron port deletes should return
no error even if static route table is not deleted for shared instances.

Change-Id: Iaa49aaefe297f027ae0c78da2ac9365ed5cf9037
Closes-Bug: #1512556

src/config/svc-monitor/svc_monitor/instance_manager.py

@@ -322,11 +322,11 @@ def validate_network_config(self, st, si):
 
         return config_complete
 
-    def cleanup_svc_vm_ports(self, vmi_list, port_delete=True):
+    def cleanup_svc_vm_ports(self, vmi_list):
         for vmi_id in vmi_list:
             try:
                 vmi_obj = self._vnc_lib.virtual_machine_interface_read(
-                    id=vmi_id, fields=['instance_ip_back_refs'])
+                    id=vmi_id, fields = ['instance_ip_back_refs', 'interface_route_table_refs'])
             except NoIdError:
                 continue
 
@@ -337,11 +337,17 @@ def cleanup_svc_vm_ports(self, vmi_list, port_delete=True):
                 except NoIdError:
                     pass
 
-            if port_delete:
+            try:
+                self._vnc_lib.virtual_machine_interface_delete(id=vmi_id)
+                VirtualMachineInterfaceSM.delete(vmi_id)
+            except (NoIdError, RefsExistError) as e:
+                pass
+
+            for irt in vmi_obj.get_interface_route_table_refs() or []:
                 try:
-                    self._vnc_lib.virtual_machine_interface_delete(id=vmi_id)
-                    VirtualMachineInterfaceSM.delete(vmi_id)
-                except (NoIdError, RefsExistError):
+                    self._vnc_lib.interface_route_table_delete(id=irt['uuid'])
+                    InterfaceRouteTableSM.delete(irt['uuid'])
+                except (NoIdError, RefsExistError) as e:
                     pass
 
     def _check_create_service_vn(self, itf_type, si):

src/config/svc-monitor/svc_monitor/virtual_machine_manager.py

@@ -140,13 +140,13 @@ def delete_service(self, vm):
             except Exception as e:
                 self.logger.log_error("%s nova delete failed with error %s" %
                     (vm.uuid, str(e)))
-        else:
-            try:
-                self._vnc_lib.virtual_machine_delete(id=vm.uuid)
-            except NoIdError:
-                pass
-            except RefsExistError:
-                self.logger.log_error("%s vm delete RefsExist" % (vm.uuid))
+
+        try:
+            self._vnc_lib.virtual_machine_delete(id=vm.uuid)
+        except NoIdError:
+            pass
+        except RefsExistError:
+            self.logger.log_error("%s vm delete RefsExist" % (vm.uuid))
 
     def check_service(self, si):
         vm_id_list = list(si.virtual_machines)

src/config/vnc_openstack/vnc_openstack/neutron_plugin_db.py

@@ -2207,7 +2207,7 @@ def _port_remove_iface_route_table(self, ipobj, subnet_id):
                     port_obj.del_interface_route_table(intf_route_table_obj)
                     self._vnc_lib.virtual_machine_interface_update(port_obj)
                     self._vnc_lib.interface_route_table_delete(id=rt_ref['uuid'])
-                except vnc_exc.NoIdError:
+                except (NoIdError, RefsExistError) as e:
                     pass
 
     # public methods
@@ -3499,7 +3499,7 @@ def port_delete(self, port_id):
         for rt_ref in port_obj.get_interface_route_table_refs() or []:
             try:
                 self._vnc_lib.interface_route_table_delete(id=rt_ref['uuid'])
-            except vnc_exc.NoIdError:
+            except (NoIdError, RefsExistError) as e:
                 pass
 
         # delete instance if this was the last port