diff --git a/src/api-lib/tools/install_venv_common.py b/src/api-lib/tools/install_venv_common.py index 87c8de2e4ed..a3183784b2f 100644 --- a/src/api-lib/tools/install_venv_common.py +++ b/src/api-lib/tools/install_venv_common.py @@ -30,8 +30,18 @@ import os import subprocess import sys +import fnmatch +import re +def find_files(directory, file_pattern): + for root, dirs, files in os.walk(directory): + for afile in files: + if fnmatch.fnmatch(afile, file_pattern): + yield os.path.join(root, afile) + for adir in dirs: + find_files(adir, file_pattern) + class InstallVenv(object): def __init__(self, root, venv, requirements, @@ -115,6 +125,32 @@ def pip_install(self, find_links, *args): self.run_command(cmd_array + list(args), redirect_output=False) + def get_requirements(self): + with open(self.requirements, 'r+') as fd: + reqs = [req.strip() for req in fd.readlines()] + gteq_or_lteq_or_eqeq = re.compile('(.*)[>=<]+[>=<]+') + gt_or_lt_or_eq = re.compile('(.*)[>=<]+') + requirements = [] + for req in reqs: + for regexp in [gteq_or_lteq_or_eqeq, gt_or_lt_or_eq]: + match = regexp.match(req) + if match: + requirements.append(match.group(1)) + continue + else: + requirements.append(req) + return requirements + + def pip_clear_cache(self, find_links): + contrail_reqs = [] + cache_dir = os.path.expanduser('~/.cache/pip') + for req in self.get_requirements(): + req_pattern = '*' + req + '*' + for find_link in find_links: + for contrail_req in find_files(find_link, req_pattern): + for cachefile in find_files(cache_dir, req_pattern): + os.remove(cachefile) + def install_dependencies(self, find_links): print('Installing dependencies with pip (this can take a while)...') @@ -124,6 +160,7 @@ def install_dependencies(self, find_links): self.pip_install(find_links, 'setuptools') self.pip_install(find_links, 'pbr') + self.pip_clear_cache(find_links) self.pip_install(find_links, '-r', self.requirements, '-r', self.test_requirements, '--pre') def parse_args(self, argv): diff --git a/src/config/common/tests/tools/install_venv_common.py b/src/config/common/tests/tools/install_venv_common.py index 209b891febf..6c77c839664 100644 --- a/src/config/common/tests/tools/install_venv_common.py +++ b/src/config/common/tests/tools/install_venv_common.py @@ -39,6 +39,8 @@ def find_files(directory, file_pattern): for afile in files: if fnmatch.fnmatch(afile, file_pattern): yield os.path.join(root, afile) + for adir in dirs: + find_files(adir, file_pattern) class InstallVenv(object): @@ -142,14 +144,13 @@ def get_requirements(self): def pip_clear_cache(self, find_links): contrail_reqs = [] + cache_dir = os.path.expanduser('~/.cache/pip') for req in self.get_requirements(): + req_pattern = '*' + req + '*' for find_link in find_links: - if find_files(find_link, req): - contrail_reqs.append(req) - for contrail_req in contrail_reqs: - cache_dir = os.path.expanduser('~/.cache/pip') - for cachefile in find_files(cache_dir, contrail_req): - os.remove(cachefile) + for contrail_req in find_files(find_link, req_pattern): + for cachefile in find_files(cache_dir, req_pattern): + os.remove(cachefile) def install_dependencies(self, find_links): print('Installing dependencies with pip (this can take a while)...') diff --git a/src/config/schema-transformer/config_db.py b/src/config/schema-transformer/config_db.py index 1bbff413de3..9c4ae2de168 100644 --- a/src/config/schema-transformer/config_db.py +++ b/src/config/schema-transformer/config_db.py @@ -1764,6 +1764,7 @@ class RoutingInstanceST(DBBaseST): def __init__(self, name, obj=None): self.name = name self.obj = obj or self.read_vnc_obj(fq_name=name) + self.stale_route_targets = [] self.service_chain = self._get_service_id_from_ri(self.name) self.connections = set() self.virtual_network = None @@ -1890,35 +1891,42 @@ def update_routing_policy_and_aggregates(self): # end update_routing_policy_and_aggregates def import_default_ri_route_target_to_service_ri(self): + update_ri = False if not self.service_chain: - return + return update_ri sc = ServiceChain.get(self.service_chain) if sc is None or not sc.created: - return + return update_ri left_vn = VirtualNetworkST.get(sc.left_vn) right_vn = VirtualNetworkST.get(sc.right_vn) if left_vn is None or right_vn is None: self._logger.debug("left or right vn not found for RI " + self.name) - return + return update_ri multi_policy_enabled = ( left_vn.multi_policy_service_chains_enabled and right_vn.multi_policy_service_chains_enabled) if not multi_policy_enabled: - return + return update_ri vn = VirtualNetworkST.get(self.virtual_network) if sc.left_vn == vn.name: si_name = sc.service_list[0] elif sc.right_vn == vn.name: si_name = sc.service_list[-1] else: - return + return update_ri service_ri_name = vn.get_service_name(sc.name, si_name) if service_ri_name == self.name: - rt_obj = RouteTargetST.get(vn.get_route_target()) - self.obj.add_route_target(rt_obj.obj, - InstanceTargetType('import')) - # end locate_imported_service_chain_route_target + rt = vn.get_route_target() + if rt not in self.stale_route_targets: + rt_obj = RouteTargetST.get(rt) + self.obj.add_route_target(rt_obj.obj, + InstanceTargetType('import')) + update_ri = True + else: + self.stale_route_targets.remove(rt) + return update_ri + # end import_default_ri_route_target_to_service_ri def locate_route_target(self): old_rtgt = self._cassandra.get_route_target(self.name) @@ -1945,27 +1953,48 @@ def locate_route_target(self): self._vnc_lib.routing_instance_delete(id=self.obj.uuid) self.obj = None else: - old_rt_refs = copy.deepcopy(self.obj.get_route_target_refs()) - self.obj.set_route_target(rtgt_obj, InstanceTargetType()) + update_ri = False + self.stale_route_targets = [':'.join(rt_ref['to']) + for rt_ref in self.obj.get_route_target_refs() or []] + if rt_key not in self.stale_route_targets: + self.obj.set_route_target(rtgt_obj, InstanceTargetType()) + update_ri = True + else: + self.stale_route_targets.remove(rt_key) if inst_tgt_data: for rt in vn.rt_list: - rtgt_obj = RouteTargetST.locate(rt) - self.obj.add_route_target(rtgt_obj.obj, inst_tgt_data) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt) + self.obj.add_route_target(rtgt_obj.obj, inst_tgt_data) + update_ri = True + else: + self.stale_route_targets.remove(rt) if self.is_default: for rt in vn.export_rt_list: - rtgt_obj = RouteTargetST.locate(rt) - self.obj.add_route_target( - rtgt_obj.obj, InstanceTargetType('export')) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt) + self.obj.add_route_target( + rtgt_obj.obj, InstanceTargetType('export')) + update_ri = True + else: + self.stale_route_targets.remove(rt) for rt in vn.import_rt_list: - rtgt_obj = RouteTargetST.locate(rt) - self.obj.add_route_target( - rtgt_obj.obj, InstanceTargetType('import')) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt) + self.obj.add_route_target( + rtgt_obj.obj, InstanceTargetType('import')) + update_ri = True + else: + self.stale_route_targets.remove(rt) elif vn.allow_transit: - rtgt_obj = RouteTarget(vn._route_target) - self.obj.add_route_target(rtgt_obj, inst_tgt_data) - self.import_default_ri_route_target_to_service_ri() - if not compare_refs(self.obj.get_route_target_refs(), - old_rt_refs): + if vn._route_target not in self.stale_route_targets: + rtgt_obj = RouteTarget(vn._route_target) + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update_ri = True + else: + self.stale_route_targets.remove(vn._route_target) + update_ri |= self.import_default_ri_route_target_to_service_ri() + if update_ri: self._vnc_lib.routing_instance_update(self.obj) except NoIdError as e: self._logger.error( @@ -2052,22 +2081,38 @@ def add_service_info(self, remote_vn, service_instance=None, def update_route_target_list(self, rt_add=None, rt_add_import=None, rt_add_export=None, rt_del=None): + update = False for rt in rt_del or []: + if rt in self.stale_route_targets: + self.stale_route_targets.remove(rt) rtgt_obj = RouteTarget(rt) self.obj.del_route_target(rtgt_obj) + update = True for rt in rt_add or []: - rtgt_obj = RouteTargetST.locate(rt).obj - inst_tgt_data = InstanceTargetType(import_export=None) - self.obj.add_route_target(rtgt_obj, inst_tgt_data) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt).obj + inst_tgt_data = InstanceTargetType(import_export=None) + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update = True + else: + self.stale_route_targets.remove(rt) for rt in rt_add_import or []: - rtgt_obj = RouteTargetST.locate(rt).obj - inst_tgt_data = InstanceTargetType(import_export='import') - self.obj.add_route_target(rtgt_obj, inst_tgt_data) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt).obj + inst_tgt_data = InstanceTargetType(import_export='import') + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update = True + else: + self.stale_route_targets.remove(rt) for rt in rt_add_export or []: - rtgt_obj = RouteTargetST.locate(rt).obj - inst_tgt_data = InstanceTargetType(import_export='export') - self.obj.add_route_target(rtgt_obj, inst_tgt_data) - if rt_add or rt_add_export or rt_add_import or rt_del: + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt).obj + inst_tgt_data = InstanceTargetType(import_export='export') + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update = True + else: + self.stale_route_targets.remove(rt) + if update: try: self._vnc_lib.routing_instance_update(self.obj) except NoIdError: diff --git a/src/config/schema-transformer/to_bgp.py b/src/config/schema-transformer/to_bgp.py index 6eb0bf7c0ed..3b113065c19 100644 --- a/src/config/schema-transformer/to_bgp.py +++ b/src/config/schema-transformer/to_bgp.py @@ -492,6 +492,10 @@ def process_stale_objects(self): sc.destroy() if sc.present_stale: sc.delete() + for rinst in RoutingInstanceST.values(): + if rinst.stale_route_targets: + rinst.update_route_target_list( + rt_del=rinst.stale_route_targets) # end process_stale_objects def sandesh_ri_build(self, vn_name, ri_name):