From 0e019b2cc9e09edbe38e5b8b69c5f385ddc13c05 Mon Sep 17 00:00:00 2001 From: Ignatious Johnson Christopher Date: Wed, 22 Jun 2016 07:50:38 +0000 Subject: [PATCH] Route targets of the connected logical routers are not added to routing instances during reinit, when schema transformer restarts. They are added later during the Logical Router objects are evaluated. Fix is to update only the new route targets and remove all stale route targets after reint. Closes-Bug: 1588204 Also contrail python packages are cleared from the pip cache before creation of test virtual enviroiment to make sure the virtual enviroinment is installed with the newly built python package. Change-Id: Iaf7aa002020db7b7de01c32d44c3c74f2215e90f --- src/api-lib/tools/install_venv_common.py | 37 ++++++ .../common/tests/tools/install_venv_common.py | 13 +- src/config/schema-transformer/config_db.py | 113 ++++++++++++------ src/config/schema-transformer/to_bgp.py | 4 + 4 files changed, 127 insertions(+), 40 deletions(-) diff --git a/src/api-lib/tools/install_venv_common.py b/src/api-lib/tools/install_venv_common.py index 87c8de2e4ed..a3183784b2f 100644 --- a/src/api-lib/tools/install_venv_common.py +++ b/src/api-lib/tools/install_venv_common.py @@ -30,8 +30,18 @@ import os import subprocess import sys +import fnmatch +import re +def find_files(directory, file_pattern): + for root, dirs, files in os.walk(directory): + for afile in files: + if fnmatch.fnmatch(afile, file_pattern): + yield os.path.join(root, afile) + for adir in dirs: + find_files(adir, file_pattern) + class InstallVenv(object): def __init__(self, root, venv, requirements, @@ -115,6 +125,32 @@ def pip_install(self, find_links, *args): self.run_command(cmd_array + list(args), redirect_output=False) + def get_requirements(self): + with open(self.requirements, 'r+') as fd: + reqs = [req.strip() for req in fd.readlines()] + gteq_or_lteq_or_eqeq = re.compile('(.*)[>=<]+[>=<]+') + gt_or_lt_or_eq = re.compile('(.*)[>=<]+') + requirements = [] + for req in reqs: + for regexp in [gteq_or_lteq_or_eqeq, gt_or_lt_or_eq]: + match = regexp.match(req) + if match: + requirements.append(match.group(1)) + continue + else: + requirements.append(req) + return requirements + + def pip_clear_cache(self, find_links): + contrail_reqs = [] + cache_dir = os.path.expanduser('~/.cache/pip') + for req in self.get_requirements(): + req_pattern = '*' + req + '*' + for find_link in find_links: + for contrail_req in find_files(find_link, req_pattern): + for cachefile in find_files(cache_dir, req_pattern): + os.remove(cachefile) + def install_dependencies(self, find_links): print('Installing dependencies with pip (this can take a while)...') @@ -124,6 +160,7 @@ def install_dependencies(self, find_links): self.pip_install(find_links, 'setuptools') self.pip_install(find_links, 'pbr') + self.pip_clear_cache(find_links) self.pip_install(find_links, '-r', self.requirements, '-r', self.test_requirements, '--pre') def parse_args(self, argv): diff --git a/src/config/common/tests/tools/install_venv_common.py b/src/config/common/tests/tools/install_venv_common.py index 209b891febf..6c77c839664 100644 --- a/src/config/common/tests/tools/install_venv_common.py +++ b/src/config/common/tests/tools/install_venv_common.py @@ -39,6 +39,8 @@ def find_files(directory, file_pattern): for afile in files: if fnmatch.fnmatch(afile, file_pattern): yield os.path.join(root, afile) + for adir in dirs: + find_files(adir, file_pattern) class InstallVenv(object): @@ -142,14 +144,13 @@ def get_requirements(self): def pip_clear_cache(self, find_links): contrail_reqs = [] + cache_dir = os.path.expanduser('~/.cache/pip') for req in self.get_requirements(): + req_pattern = '*' + req + '*' for find_link in find_links: - if find_files(find_link, req): - contrail_reqs.append(req) - for contrail_req in contrail_reqs: - cache_dir = os.path.expanduser('~/.cache/pip') - for cachefile in find_files(cache_dir, contrail_req): - os.remove(cachefile) + for contrail_req in find_files(find_link, req_pattern): + for cachefile in find_files(cache_dir, req_pattern): + os.remove(cachefile) def install_dependencies(self, find_links): print('Installing dependencies with pip (this can take a while)...') diff --git a/src/config/schema-transformer/config_db.py b/src/config/schema-transformer/config_db.py index 1bbff413de3..9c4ae2de168 100644 --- a/src/config/schema-transformer/config_db.py +++ b/src/config/schema-transformer/config_db.py @@ -1764,6 +1764,7 @@ class RoutingInstanceST(DBBaseST): def __init__(self, name, obj=None): self.name = name self.obj = obj or self.read_vnc_obj(fq_name=name) + self.stale_route_targets = [] self.service_chain = self._get_service_id_from_ri(self.name) self.connections = set() self.virtual_network = None @@ -1890,35 +1891,42 @@ def update_routing_policy_and_aggregates(self): # end update_routing_policy_and_aggregates def import_default_ri_route_target_to_service_ri(self): + update_ri = False if not self.service_chain: - return + return update_ri sc = ServiceChain.get(self.service_chain) if sc is None or not sc.created: - return + return update_ri left_vn = VirtualNetworkST.get(sc.left_vn) right_vn = VirtualNetworkST.get(sc.right_vn) if left_vn is None or right_vn is None: self._logger.debug("left or right vn not found for RI " + self.name) - return + return update_ri multi_policy_enabled = ( left_vn.multi_policy_service_chains_enabled and right_vn.multi_policy_service_chains_enabled) if not multi_policy_enabled: - return + return update_ri vn = VirtualNetworkST.get(self.virtual_network) if sc.left_vn == vn.name: si_name = sc.service_list[0] elif sc.right_vn == vn.name: si_name = sc.service_list[-1] else: - return + return update_ri service_ri_name = vn.get_service_name(sc.name, si_name) if service_ri_name == self.name: - rt_obj = RouteTargetST.get(vn.get_route_target()) - self.obj.add_route_target(rt_obj.obj, - InstanceTargetType('import')) - # end locate_imported_service_chain_route_target + rt = vn.get_route_target() + if rt not in self.stale_route_targets: + rt_obj = RouteTargetST.get(rt) + self.obj.add_route_target(rt_obj.obj, + InstanceTargetType('import')) + update_ri = True + else: + self.stale_route_targets.remove(rt) + return update_ri + # end import_default_ri_route_target_to_service_ri def locate_route_target(self): old_rtgt = self._cassandra.get_route_target(self.name) @@ -1945,27 +1953,48 @@ def locate_route_target(self): self._vnc_lib.routing_instance_delete(id=self.obj.uuid) self.obj = None else: - old_rt_refs = copy.deepcopy(self.obj.get_route_target_refs()) - self.obj.set_route_target(rtgt_obj, InstanceTargetType()) + update_ri = False + self.stale_route_targets = [':'.join(rt_ref['to']) + for rt_ref in self.obj.get_route_target_refs() or []] + if rt_key not in self.stale_route_targets: + self.obj.set_route_target(rtgt_obj, InstanceTargetType()) + update_ri = True + else: + self.stale_route_targets.remove(rt_key) if inst_tgt_data: for rt in vn.rt_list: - rtgt_obj = RouteTargetST.locate(rt) - self.obj.add_route_target(rtgt_obj.obj, inst_tgt_data) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt) + self.obj.add_route_target(rtgt_obj.obj, inst_tgt_data) + update_ri = True + else: + self.stale_route_targets.remove(rt) if self.is_default: for rt in vn.export_rt_list: - rtgt_obj = RouteTargetST.locate(rt) - self.obj.add_route_target( - rtgt_obj.obj, InstanceTargetType('export')) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt) + self.obj.add_route_target( + rtgt_obj.obj, InstanceTargetType('export')) + update_ri = True + else: + self.stale_route_targets.remove(rt) for rt in vn.import_rt_list: - rtgt_obj = RouteTargetST.locate(rt) - self.obj.add_route_target( - rtgt_obj.obj, InstanceTargetType('import')) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt) + self.obj.add_route_target( + rtgt_obj.obj, InstanceTargetType('import')) + update_ri = True + else: + self.stale_route_targets.remove(rt) elif vn.allow_transit: - rtgt_obj = RouteTarget(vn._route_target) - self.obj.add_route_target(rtgt_obj, inst_tgt_data) - self.import_default_ri_route_target_to_service_ri() - if not compare_refs(self.obj.get_route_target_refs(), - old_rt_refs): + if vn._route_target not in self.stale_route_targets: + rtgt_obj = RouteTarget(vn._route_target) + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update_ri = True + else: + self.stale_route_targets.remove(vn._route_target) + update_ri |= self.import_default_ri_route_target_to_service_ri() + if update_ri: self._vnc_lib.routing_instance_update(self.obj) except NoIdError as e: self._logger.error( @@ -2052,22 +2081,38 @@ def add_service_info(self, remote_vn, service_instance=None, def update_route_target_list(self, rt_add=None, rt_add_import=None, rt_add_export=None, rt_del=None): + update = False for rt in rt_del or []: + if rt in self.stale_route_targets: + self.stale_route_targets.remove(rt) rtgt_obj = RouteTarget(rt) self.obj.del_route_target(rtgt_obj) + update = True for rt in rt_add or []: - rtgt_obj = RouteTargetST.locate(rt).obj - inst_tgt_data = InstanceTargetType(import_export=None) - self.obj.add_route_target(rtgt_obj, inst_tgt_data) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt).obj + inst_tgt_data = InstanceTargetType(import_export=None) + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update = True + else: + self.stale_route_targets.remove(rt) for rt in rt_add_import or []: - rtgt_obj = RouteTargetST.locate(rt).obj - inst_tgt_data = InstanceTargetType(import_export='import') - self.obj.add_route_target(rtgt_obj, inst_tgt_data) + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt).obj + inst_tgt_data = InstanceTargetType(import_export='import') + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update = True + else: + self.stale_route_targets.remove(rt) for rt in rt_add_export or []: - rtgt_obj = RouteTargetST.locate(rt).obj - inst_tgt_data = InstanceTargetType(import_export='export') - self.obj.add_route_target(rtgt_obj, inst_tgt_data) - if rt_add or rt_add_export or rt_add_import or rt_del: + if rt not in self.stale_route_targets: + rtgt_obj = RouteTargetST.locate(rt).obj + inst_tgt_data = InstanceTargetType(import_export='export') + self.obj.add_route_target(rtgt_obj, inst_tgt_data) + update = True + else: + self.stale_route_targets.remove(rt) + if update: try: self._vnc_lib.routing_instance_update(self.obj) except NoIdError: diff --git a/src/config/schema-transformer/to_bgp.py b/src/config/schema-transformer/to_bgp.py index 6eb0bf7c0ed..3b113065c19 100644 --- a/src/config/schema-transformer/to_bgp.py +++ b/src/config/schema-transformer/to_bgp.py @@ -492,6 +492,10 @@ def process_stale_objects(self): sc.destroy() if sc.present_stale: sc.delete() + for rinst in RoutingInstanceST.values(): + if rinst.stale_route_targets: + rinst.update_route_target_list( + rt_del=rinst.stale_route_targets) # end process_stale_objects def sandesh_ri_build(self, vn_name, ri_name):