From a1cadd44984f1d23e67cd7c348bbbd5120079760 Mon Sep 17 00:00:00 2001 From: Rudra Rugge Date: Thu, 21 Jan 2016 14:55:30 -0800 Subject: [PATCH] Create vrouter state cache from polling uve Currently every launch of a netns VM causes fetch of vrouter UVEs. This is inefficient and the plan is to maintain a local cache for the same. Also need to add debouncing logic for marking the vrouter as down as the collectors might return inconsistent data when there is flux in the system. Also added fix to pick analytics api client from discovery and not from configuration as before. Change-Id: I81e77d9c975a0599d7036b19fd73e4042f3604a3 Closes-Bug: #1536821 Closes-Bug: #1409073 --- .../svc-monitor/svc_monitor/config_db.py | 15 + .../svc_monitor/instance_manager.py | 13 +- .../scheduler/vrouter_scheduler.py | 217 +++++---- .../svc-monitor/svc_monitor/svc_monitor.py | 6 +- .../scheduler/test_vrouter_schedulers.py | 420 ++++-------------- .../svc_monitor/tests/test_common_utils.py | 10 + .../svc_monitor/tests/test_snat.py | 6 +- 7 files changed, 223 insertions(+), 464 deletions(-) diff --git a/src/config/svc-monitor/svc_monitor/config_db.py b/src/config/svc-monitor/svc_monitor/config_db.py index 8ce29253d60..6920b320960 100644 --- a/src/config/svc-monitor/svc_monitor/config_db.py +++ b/src/config/svc-monitor/svc_monitor/config_db.py @@ -331,6 +331,8 @@ class VirtualRouterSM(DBBaseSM): def __init__(self, uuid, obj_dict=None): self.uuid = uuid + self.agent_state = False + self.agent_down_count = 0 self.virtual_machines = set() self.update(obj_dict) # end __init__ @@ -351,6 +353,19 @@ def delete(cls, uuid): obj.update_multiple_refs('virtual_machine', {}) del cls._dict[uuid] # end delete + + def set_agent_state(self, up): + if up: + self.agent_down_count = 0 + self.agent_state = True + else: + self.agent_down_count += 1 + if not (self.agent_down_count % 3): + self.agent_state = False + + def set_netns_version(self, netns_version): + self.netns_version = netns_version + # end VirtualRouterSM diff --git a/src/config/svc-monitor/svc_monitor/instance_manager.py b/src/config/svc-monitor/svc_monitor/instance_manager.py index 23c9d24a500..fed02e1a680 100644 --- a/src/config/svc-monitor/svc_monitor/instance_manager.py +++ b/src/config/svc-monitor/svc_monitor/instance_manager.py @@ -593,13 +593,12 @@ def _create_svc_vm_port(self, nic, instance_name, si, st, def _associate_vrouter(self, si, vm): vrouter_name = None if not vm.virtual_router: - chosen_vr_fq_name = None - chosen_vr_fq_name = self.vrouter_scheduler.schedule( - si.uuid, vm.uuid) - if chosen_vr_fq_name: - vrouter_name = chosen_vr_fq_name[-1] + chosen_vr = self.vrouter_scheduler.schedule(si, vm) + if chosen_vr: + vr = VirtualRouterSM.get(chosen_vr) + vrouter_name = vr.name self.logger.log_notice("vrouter %s updated with vm %s" % - (':'.join(chosen_vr_fq_name), vm.name)) + (':'.join(vr.fq_name), vm.name)) vm.update() else: vr = VirtualRouterSM.get(vm.virtual_router) @@ -660,7 +659,7 @@ def check_service(self, si): service_up = False else: vr = VirtualRouterSM.get(vm.virtual_router) - if self.vrouter_scheduler.vrouter_running(vr.name): + if vr.agent_state: continue self._vnc_lib.ref_update('virtual-router', vr.uuid, 'virtual-machine', vm.uuid, None, 'DELETE') diff --git a/src/config/svc-monitor/svc_monitor/scheduler/vrouter_scheduler.py b/src/config/svc-monitor/svc_monitor/scheduler/vrouter_scheduler.py index a4013f3ad69..1231f35e7d5 100644 --- a/src/config/svc-monitor/svc_monitor/scheduler/vrouter_scheduler.py +++ b/src/config/svc-monitor/svc_monitor/scheduler/vrouter_scheduler.py @@ -28,18 +28,20 @@ from sandesh_common.vns import constants from vnc_api.vnc_api import NoIdError +from svc_monitor.config_db import * + +from sandesh_common.vns.constants import \ + ANALYTICS_API_SERVER_DISCOVERY_SERVICE_NAME as analytics_svc_name + @six.add_metaclass(abc.ABCMeta) class VRouterScheduler(object): - def __init__(self, vnc_lib, nova_client, args): + def __init__(self, vnc_lib, nova_client, disc, logger, args): self._vnc_lib = vnc_lib self._args = args self._nc = nova_client - - # initialize analytics client - endpoint = "http://%s:%s" % (self._args.analytics_server_ip, - self._args.analytics_server_port) - self._analytics = analytics_client.Client(endpoint) + self._disc = disc + self._logger = logger @abc.abstractmethod def schedule(self, plugin, context, router_id, candidates=None): @@ -51,130 +53,119 @@ def schedule(self, plugin, context, router_id, candidates=None): """ pass - def _get_candidates(self, si_uuid, vm_uuid): - """Return vrouter agents where a service instance virtual machine - could be scheduled. - - If a VM of a same service instance is already scheduled on the vrouter, - that vrouter is exclude from the candidates list. - If the VM is already scheduled on a running vrouter, only that vrouter - is return in the candidates list. - """ - - # if availability zone configured then use it - vr_list = [] - if self._args.netns_availability_zone: - az_list = self._nc.oper('availability_zones', 'list', - 'admin', detailed=True) - for az in az_list: - if self._args.netns_availability_zone in str(az): - for host in az.hosts: - vr_list.append({'fq_name': - ['default-global-system-config', host]}) - else: - vr_list = self._vnc_lib.virtual_routers_list()['virtual-routers'] - - # check if vrouters are functional and support service instance - vrs_fq_name = [vr['fq_name'] for vr in vr_list - if self.vrouter_running(vr['fq_name'][-1])] - vrs_fq_name = [vr_fq_name for vr_fq_name in vrs_fq_name - if self.vrouter_check_version( - vr_fq_name[-1], - svc_info._VROUTER_NETNS_SUPPORTED_VERSION)] - - for vr_fq_name in vrs_fq_name: - try: - vr_obj = self._vnc_lib.virtual_router_read(fq_name=vr_fq_name) - except NoIdError: - vrs_fq_name.remove(vr_fq_name) + def _get_az_vrouter_list(self): + if not self._args.netns_availability_zone: + return None + az_list = self._nc.oper('availability_zones', 'list', + 'admin', detailed=True) + az_vr_list = [] + for az in az_list: + if self._args.netns_availability_zone not in str(az): continue - for vm_ref in vr_obj.get_virtual_machine_refs() or []: - if vm_uuid == vm_ref['uuid']: - return [vr_fq_name] - try: - vm_obj = self._vnc_lib.virtual_machine_read( - id=vm_ref['uuid']) - except NoIdError: - continue - if si_uuid in [si['uuid'] for si in - vm_obj.get_service_instance_refs()]: - vrs_fq_name.remove(vr_fq_name) - continue - return vrs_fq_name - - def vrouter_running(self, vrouter_name): - """Check if a vrouter agent is up and running.""" + az_vr_list.extend(az.hosts.keys()) + + return az_vr_list + + def get_analytics_client(self): + try: + sub_obj = self._disc.subscribe(analytics_svc_name, 0) + slist= sub_obj.info + except Exception as ex: + self._logger.log_error('Failed to get analytics api from discovery') + return None + else: + if not slist: + self._logger.log_error('No analytics api client in discovery') + return None + analytics_api = random.choice(slist) + endpoint = "http://%s:%s" % (analytics_api['ip-address'], + str(analytics_api['port'])) + return analytics_client.Client(endpoint) + + def query_uve(self, filter_string): path = "/analytics/uves/vrouter/" - - fqdn_uuid = "%s?cfilt=VrouterAgent" % vrouter_name - try: - vrouter_agent = self._analytics.request(path, fqdn_uuid) - except analytics_client.OpenContrailAPIFailed: - return False - - if 'VrouterAgent' not in vrouter_agent or \ - ('mode' in vrouter_agent['VrouterAgent'] and \ - vrouter_agent['VrouterAgent']['mode'] != \ - constants.VrouterAgentTypeMap[ - constants.VrouterAgentType.VROUTER_AGENT_EMBEDDED]): - return False - - fqdn_uuid = "%s?cfilt=NodeStatus" % vrouter_name + response_dict = {} try: - node_status = self._analytics.request(path, fqdn_uuid) + response = self._analytics.request(path, filter_string) + for values in response['value']: + response_dict[values['name']] = values['value'] except analytics_client.OpenContrailAPIFailed: - return False + pass + return response_dict - if not node_status or 'NodeStatus' not in node_status or \ - 'process_status' not in node_status['NodeStatus']: - return False + def vrouters_running(self): + # get az host list + az_vrs = self._get_az_vrouter_list() - for process in node_status['NodeStatus']['process_status']: - if (process['module_id'] == constants.MODULE_VROUTER_AGENT_NAME and - int(process['instance_id']) == 0 and - process['state'] == 'Functional'): - return True - return False + # read all vrouter information + self._analytics = self.get_analytics_client() + if not self._analytics: + return + agents_status = self.query_uve("*?cfilt=NodeStatus:process_status") + vrouters_mode = self.query_uve("*?cfilt=VrouterAgent:mode") - def vrouter_check_version(self, vrouter_name, version): - """Check the vrouter version is upper or equal to a desired version.""" - path = "/analytics/uves/vrouter/" - fqdn_uuid = "%s?cfilt=VrouterAgent" % vrouter_name + for vr in VirtualRouterSM.values(): + if az_vrs and vr.name not in az_vrs: + vr.set_agent_state(False) + continue - try: - vrouter_agent = self._analytics.request(path, fqdn_uuid) - except analytics_client.OpenContrailAPIFailed: - return False + if vr.name not in vrouters_mode or vr.name not in agents_status: + vr.set_agent_state(False) + continue - if not vrouter_agent: - return False + try: + vr_mode = vrouters_mode[vr.name]['VrouterAgent'] + if (vr_mode['mode'] != constants.VrouterAgentTypeMap[ + constants.VrouterAgentType.VROUTER_AGENT_EMBEDDED]): + vr.set_agent_state(False) + continue + except Exception as e: + vr.set_agent_state(False) + continue - try: - build_info = ast.literal_eval( - vrouter_agent['VrouterAgent']['build_info']) - vrouter_version = V(build_info['build-info'][0]['build-version']) - requested_version = V(version) - except KeyError, ValueError: - return False + try: + vr_status = agents_status[vr.name]['NodeStatus']['process_status'][0] + if (vr_status['module_id'] == constants.MODULE_VROUTER_AGENT_NAME and + int(vr_status['instance_id']) == 0 and + vr_status['state'] == 'Functional'): + vr.set_agent_state(True) + else: + vr.set_agent_state(False) + except Exception as e: + vr.set_agent_state(False) + continue - return vrouter_version >= requested_version + def _get_candidates(self, si, vm): + if vm.virtual_router: + return [vm.virtual_router] - def _bind_vrouter(self, vm_uuid, vr_fq_name): - """Bind the virtual machine to the vrouter which has been chosen.""" - vm_obj = self._vnc_lib.virtual_machine_read(id=vm_uuid) - vr_obj = self._vnc_lib.virtual_router_read(fq_name=vr_fq_name) - vr_obj.add_virtual_machine(vm_obj) - self._vnc_lib.virtual_router_update(vr_obj) + vr_list = VirtualRouterSM._dict.keys() + for vm_id in si.virtual_machines: + if vm_id == vm.uuid: + continue + anti_affinity_vm = VirtualMachineSM.get(vm_id) + if anti_affinity_vm: + try: + vr_list.remove(anti_affinity_vm.virtual_router) + except ValueError: + pass + for vr in VirtualRouterSM.values(): + if not vr.agent_state: + try: + vr_list.remove(vr.uuid) + except ValueError: + pass + return vr_list class RandomScheduler(VRouterScheduler): """Randomly allocate a vrouter agent for virtual machine of a service instance.""" - - def schedule(self, si_uuid, vm_uuid): - candidates = self._get_candidates(si_uuid, vm_uuid) + def schedule(self, si, vm): + candidates = self._get_candidates(si, vm) if not candidates: - return + return None chosen_vrouter = random.choice(candidates) - self._bind_vrouter(vm_uuid, chosen_vrouter) + self._vnc_lib.ref_update('virtual-router', chosen_vrouter, + 'virtual-machine', vm.uuid, None, 'ADD') return chosen_vrouter diff --git a/src/config/svc-monitor/svc_monitor/svc_monitor.py b/src/config/svc-monitor/svc_monitor/svc_monitor.py index 0d7ee3e54f0..6ba4b9ceb84 100644 --- a/src/config/svc-monitor/svc_monitor/svc_monitor.py +++ b/src/config/svc-monitor/svc_monitor/svc_monitor.py @@ -347,7 +347,7 @@ def post_init(self, vnc_lib, args=None): self.vrouter_scheduler = importutils.import_object( self._args.si_netns_scheduler_driver, self._vnc_lib, self._nova_client, - self._args) + self._disc, self.logger, self._args) # load virtual machine instance manager self.vm_manager = importutils.import_object( @@ -429,6 +429,7 @@ def post_init(self, vnc_lib, args=None): self.upgrade() # check services + self.vrouter_scheduler.vrouters_running() self.launch_services() self._db_resync_done.set() @@ -765,6 +766,9 @@ def timer_callback(monitor): if len(vmi_delete_list): monitor.vm_manager.cleanup_svc_vm_ports(vmi_delete_list) + # check vrouter agent status + monitor.vrouter_scheduler.vrouters_running() + # check status of service si_list = list(ServiceInstanceSM.values()) for si in si_list: diff --git a/src/config/svc-monitor/svc_monitor/tests/scheduler/test_vrouter_schedulers.py b/src/config/svc-monitor/svc_monitor/tests/scheduler/test_vrouter_schedulers.py index 391ce5b211e..1929ba21a98 100644 --- a/src/config/svc-monitor/svc_monitor/tests/scheduler/test_vrouter_schedulers.py +++ b/src/config/svc-monitor/svc_monitor/tests/scheduler/test_vrouter_schedulers.py @@ -26,255 +26,21 @@ import svc_monitor.scheduler.vrouter_scheduler as scheduler from vnc_api.vnc_api import VirtualRouter, VirtualMachine - -RUNNING_VROUTER_UVES_STATUS = { - "NodeStatus": { - "process_status": [ - { - "instance_id": "0", - "module_id": constants.MODULE_VROUTER_AGENT_NAME, - "state": "Functional", - "connection_infos": [ - { - "server_addrs": [ - "10.0.1.2:0" - ], - "status": "Up", - "type": "XMPP", - "name": "control-node:10.0.1.2", - "description": "OpenSent" - }, - { - "server_addrs": [ - "127.0.0.1:8086" - ], - "status": "Up", - "type": "Collector", - "name": "null", - "description": "Established" - } - ] - } - ], - "description": "null" - }, - "VrouterAgent": { - "build_info": "{\"build-info\":[{\"build-time\":\"2015-01-12 11:13:42.160435\",\"build-hostname\":\"vrouter1\",\"build-git-ver\":\"bdcb043\",\"build-user\":\"root\",\"build-version\":\"2.0\",\"build-id\":\"unknown\",\"build-number\":\"unknown\"}]}" - } -} - -NON_RUNNING_VROUTER_UVES_STATUS_1 = { - "NodeStatus": { - "process_status": [ - { - "instance_id": "2", - "module_id": constants.MODULE_VROUTER_AGENT_NAME, - "state": "Functional", - "connection_infos": [ - { - "server_addrs": [ - "10.0.1.2:0" - ], - "status": "Up", - "type": "XMPP", - "name": "control-node:10.0.1.2", - "description": "OpenSent" - }, - { - "server_addrs": [ - "127.0.0.1:8086" - ], - "status": "Up", - "type": "Collector", - "name": "null", - "description": "Established" - } - ] - } - ], - "description": "null" - }, - "VrouterAgent": { - "build_info": "{\"build-info\":[{\"build-time\":\"2015-01-12 11:13:42.160435\",\"build-hostname\":\"vrouter1\",\"build-git-ver\":\"bdcb043\",\"build-user\":\"root\",\"build-version\":\"1.06\",\"build-id\":\"unknown\",\"build-number\":\"unknown\"}]}" - } +from svc_monitor.config_db import ServiceInstanceSM, VirtualRouterSM +import svc_monitor.tests.test_common_utils as test_utils + +AGENTS_STATUS = \ +{ + u'vrouter1': {u'NodeStatus': {u'process_status': + [{u'module_id': u'contrail-vrouter-agent', u'state': u'Functional'}]}}, + u'vrouter2': {u'NodeStatus': {u'process_status': + [{u'module_id': u'contrail-vrouter-agent', u'state': u'Functional'}]}} } -NON_RUNNING_VROUTER_UVES_STATUS_2 = { - "NodeStatus": { - "process_status": [ - { - "instance_id": "0", - "module_id": "FakeModuleID", - "state": "Functional", - "connection_infos": [ - { - "server_addrs": [ - "10.0.1.2:0" - ], - "status": "Up", - "type": "XMPP", - "name": "control-node:10.0.1.2", - "description": "OpenSent" - }, - { - "server_addrs": [ - "127.0.0.1:8086" - ], - "status": "Up", - "type": "Collector", - "name": "null", - "description": "Established" - } - ] - } - ], - "description": "null" - }, - "VrouterAgent": { - "build_info": "{\"build-info\":[{\"build-time\":\"2015-01-12 11:13:42.160435\",\"build-hostname\":\"vrouter1\",\"build-git-ver\":\"bdcb043\",\"build-user\":\"root\",\"build-version\":\"1.10\",\"build-id\":\"unknown\",\"build-number\":\"unknown\"}]}" - } -} - -NON_RUNNING_VROUTER_UVES_STATUS_3 = { - "NodeStatus": { - "process_status": [ - { - "instance_id": "0", - "module_id": constants.MODULE_VROUTER_AGENT_NAME, - "state": "Non-functional", - "connection_infos": [ - { - "server_addrs": [ - "10.0.1.2:0" - ], - "status": "Up", - "type": "XMPP", - "name": "control-node:10.0.1.2", - "description": "OpenSent" - }, - { - "server_addrs": [ - "127.0.0.1:8086" - ], - "status": "Up", - "type": "Collector", - "name": "null", - "description": "Established" - } - ] - } - ], - "description": "null" - }, - "VrouterAgent": { - "build_info": "{\"build-info\":[{\"build-time\":\"2015-01-12 11:13:42.160435\",\"build-hostname\":\"vrouter1\",\"build-git-ver\":\"bdcb043\",\"build-user\":\"root\",\"build-version\":\"2.0\",\"build-id\":\"unknown\",\"build-number\":\"unknown\"}]}" - } -} - -NON_RUNNING_VROUTER_UVES_STATUS_4 = {} - -NON_RUNNING_VROUTER_UVES_STATUS_5 = { - "NodeStatus": { - "process_status": [ - { - "instance_id": "0", - "module_id": constants.MODULE_VROUTER_AGENT_NAME, - "state": "Functional", - "connection_infos": [ - { - "server_addrs": [ - "10.0.1.2:0" - ], - "status": "Up", - "type": "XMPP", - "name": "control-node:10.0.1.2", - "description": "OpenSent" - }, - { - "server_addrs": [ - "127.0.0.1:8086" - ], - "status": "Up", - "type": "Collector", - "name": "null", - "description": "Established" - } - ] - } - ], - "description": "null" - }, - "VrouterAgent": { - "mode": "TOR", - "build_info": "{\"build-info\":[{\"build-time\":\"2015-01-12 11:13:42.160435\",\"build-hostname\":\"vrouter1\",\"build-git-ver\":\"bdcb043\",\"build-user\":\"root\",\"build-version\":\"2.0\",\"build-id\":\"unknown\",\"build-number\":\"unknown\"}]}" - } -} - -NON_RUNNING_VROUTER_UVES_STATUS_6 = { - "NodeStatus": { - "process_status": [ - { - "instance_id": "0", - "module_id": constants.MODULE_VROUTER_AGENT_NAME, - "state": "Functional", - "connection_infos": [ - { - "server_addrs": [ - "10.0.1.2:0" - ], - "status": "Up", - "type": "XMPP", - "name": "control-node:10.0.1.2", - "description": "OpenSent" - }, - { - "server_addrs": [ - "127.0.0.1:8086" - ], - "status": "Up", - "type": "Collector", - "name": "null", - "description": "Established" - } - ] - } - ], - "description": "null" - }, - "VrouterAgent": { - "mode": "TSN", - "build_info": "{\"build-info\":[{\"build-time\":\"2015-01-12 11:13:42.160435\",\"build-hostname\":\"vrouter1\",\"build-git-ver\":\"bdcb043\",\"build-user\":\"root\",\"build-version\":\"2.0\",\"build-id\":\"unknown\",\"build-number\":\"unknown\"}]}" - } -} - - -VROUTER_LIST = { - "virtual-routers": [ - { - "href": "http://127.0.0.1:8082/virtual-router/uuid1", - "fq_name": [ - "default-global-system-config", - "vrouter1" - ], - "uuid": "uuid1" - }, - { - "href": "http://127.0.0.1:8082/virtual-router/uuid2", - "fq_name": [ - "default-global-system-config", - "vrouter2" - ], - "uuid": "uuid2" - }, - { - "href": "http://127.0.0.1:8082/virtual-router/uuid3", - "fq_name": [ - "default-global-system-config", - "vrouter3" - ], - "uuid": "uuid3" - } - ] +VROUTERS_MODE = \ +{ + u'vrouter1': {u'VrouterAgent': {u'mode': u'VROUTER'}}, + u'vrouter2': {u'VrouterAgent': {u'mode': u'VROUTER'}} } @@ -291,106 +57,73 @@ def setUp(self): self.scheduler = \ scheduler.RandomScheduler(self.vnc_mock, mock.MagicMock(), + mock.MagicMock(), mock.MagicMock(), mock.MagicMock(netns_availability_zone=False)) def tearDown(self): self.analytics_patch.stop() + VirtualRouterSM.reset() + ServiceInstanceSM.reset() super(TestRandomScheduler, self).tearDown() def test_get_candidates(self): - self.vnc_mock.virtual_routers_list.return_value = VROUTER_LIST - - self.analytics_mock.side_effect = [RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - NON_RUNNING_VROUTER_UVES_STATUS_3, - NON_RUNNING_VROUTER_UVES_STATUS_3, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS] - - vr_obj = VirtualRouter() - vr_obj.get_virtual_machine_refs = mock.MagicMock( - return_value=[{'uuid': 'fake_vm_uuid1'}]) - self.vnc_mock.virtual_router_read.return_value = vr_obj - - vm_obj_1 = VirtualMachine() - vm_obj_1.get_service_instance_refs = mock.MagicMock( - return_value=[{'uuid': 'fake_si_uuid1'}]) - vm_obj_2 = VirtualMachine() - vm_obj_2.get_service_instance_refs = mock.MagicMock( - return_value=[{'uuid': 'fake_si_uuid2'}]) - self.vnc_mock.virtual_machine_read.side_effect = [vm_obj_1, vm_obj_2] - - # Test the vrouters seected does not already have a VM of the same SI - # schedule on it. - expected_result = [["default-global-system-config", "vrouter3"]] - self.assertEqual(self.scheduler._get_candidates('fake_si_uuid1', - 'fake_vm_uuid2'), - expected_result) - - self.analytics_mock.side_effect = [RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - NON_RUNNING_VROUTER_UVES_STATUS_3, - NON_RUNNING_VROUTER_UVES_STATUS_3, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS] - self.vnc_mock.virtual_machine_read.side_effect = [vm_obj_1, vm_obj_2] - - # Test the same vrouter is return if the VM is already scheduled on - # a running vrouter - expected_result = [["default-global-system-config", "vrouter1"]] - self.assertEqual(self.scheduler._get_candidates('fake_si_uuid1', - 'fake_vm_uuid1'), - expected_result) + test_utils.create_test_project('fake-domain:fake-project') + test_utils.create_test_virtual_network('fake-domain:fake-project:vn1') + test_utils.create_test_virtual_network('fake-domain:fake-project:vn2') + test_utils.create_test_security_group('fake-domain:fake-project:default') + st = test_utils.create_test_st(name='test-template', + virt_type='network-namespace', + intf_list=[['right', True], ['left', True]]) + si = test_utils.create_test_si(name='test-instance', count=2, + intf_list=['vn1', 'vn2']) + + # test anti-affinity + vr1 = test_utils.create_test_virtual_router('vr-candidate1') + vr2 = test_utils.create_test_virtual_router('vr-candidate2') + vm1 = test_utils.create_test_virtual_machine('vm1') + vm2 = test_utils.create_test_virtual_machine('vm2') + si.virtual_machines.add(vm1.uuid) + si.virtual_machines.add(vm2.uuid) + vm1.virtual_router = vr1.uuid + candidates = self.scheduler._get_candidates(si, vm2) + self.assertEqual(candidates, [vr2.uuid]) + + # test same vrouter returned if already scheduled + candidates = self.scheduler._get_candidates(si, vm1) + self.assertEqual(len(candidates), 1) + self.assertEqual(candidates, [vr1.uuid]) + + # test all candidates returned + vm1.virtual_router = None + candidates = self.scheduler._get_candidates(si, vm1) + self.assertEqual(len(candidates), 2) + + # test non running candidates returned + vr1.agent_state = False + candidates = self.scheduler._get_candidates(si, vm1) + self.assertEqual(len(candidates), 1) + self.assertEqual(candidates, [vr2.uuid]) + + # test no candidates + vr1.agent_state = False + vr2.agent_state = False + candidates = self.scheduler._get_candidates(si, vm1) + self.assertEqual(len(candidates), 0) def test_vrouter_running(self): - self.analytics_mock.side_effect = [analytics.OpenContrailAPIFailed, - NON_RUNNING_VROUTER_UVES_STATUS_1, - NON_RUNNING_VROUTER_UVES_STATUS_1, - NON_RUNNING_VROUTER_UVES_STATUS_2, - NON_RUNNING_VROUTER_UVES_STATUS_2, - NON_RUNNING_VROUTER_UVES_STATUS_3, - NON_RUNNING_VROUTER_UVES_STATUS_3, - NON_RUNNING_VROUTER_UVES_STATUS_4, - NON_RUNNING_VROUTER_UVES_STATUS_5, - NON_RUNNING_VROUTER_UVES_STATUS_6, - RUNNING_VROUTER_UVES_STATUS, - RUNNING_VROUTER_UVES_STATUS] - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertFalse(self.scheduler.vrouter_running('fake_vrouter_name')) - self.assertTrue(self.scheduler.vrouter_running('fake_vrouter_name')) - - def test_vrouter_check_version(self): - self.analytics_mock.side_effect = [analytics.OpenContrailAPIFailed, - NON_RUNNING_VROUTER_UVES_STATUS_1, - NON_RUNNING_VROUTER_UVES_STATUS_2, - NON_RUNNING_VROUTER_UVES_STATUS_3, - NON_RUNNING_VROUTER_UVES_STATUS_4, - RUNNING_VROUTER_UVES_STATUS] - self.assertFalse(self.scheduler.vrouter_check_version( - 'fake_vrouter_name', svc_info._VROUTER_NETNS_SUPPORTED_VERSION)) - self.assertFalse(self.scheduler.vrouter_check_version( - 'fake_vrouter_name', svc_info._VROUTER_NETNS_SUPPORTED_VERSION)) - self.assertTrue(self.scheduler.vrouter_check_version( - 'fake_vrouter_name', svc_info._VROUTER_NETNS_SUPPORTED_VERSION)) - self.assertTrue(self.scheduler.vrouter_check_version( - 'fake_vrouter_name', svc_info._VROUTER_NETNS_SUPPORTED_VERSION)) - self.assertFalse(self.scheduler.vrouter_check_version( - 'fake_vrouter_name', svc_info._VROUTER_NETNS_SUPPORTED_VERSION)) - self.assertTrue(self.scheduler.vrouter_check_version( - 'fake_vrouter_name', svc_info._VROUTER_NETNS_SUPPORTED_VERSION)) + vr1 = test_utils.create_test_virtual_router('vrouter1') + vr2 = test_utils.create_test_virtual_router('vrouter2') + self.scheduler.get_analytics_client = mock.MagicMock() + def query_uve_side_effect(query_str): + if 'NodeStatus' in query_str: + return AGENTS_STATUS + elif 'VrouterAgent' in query_str: + return VROUTERS_MODE + else: + return {} + self.scheduler.query_uve = query_uve_side_effect + self.assertTrue(VirtualRouterSM.get('vrouter1').agent_state) + self.assertTrue(VirtualRouterSM.get('vrouter2').agent_state) def test_random_scheduling(self): random_patch = mock.patch('random.choice') @@ -400,11 +133,14 @@ def side_effect(seq): return seq[0] random_mock.side_effect = side_effect + si = test_utils.create_test_si(name='test-instance', count=2, + intf_list=['vn1', 'vn2']) + vm = test_utils.create_test_virtual_machine('vm') + with mock.patch.object(scheduler.RandomScheduler, '_get_candidates', - return_value=[["default-global-system-config", "vrouter1"], - ["default-global-system-config", "vrouter2"]]): - self.assertEqual(self.scheduler.schedule('fake_uuid', 'fake_uuid'), - ["default-global-system-config", "vrouter1"]) + return_value=['vrouter1', 'vrouter2']): + chosen_vrouter = self.scheduler.schedule(si, vm) self.assertEqual(random_mock.call_count, 1) + self.assertEqual(chosen_vrouter, 'vrouter1') random_patch.stop() diff --git a/src/config/svc-monitor/svc_monitor/tests/test_common_utils.py b/src/config/svc-monitor/svc_monitor/tests/test_common_utils.py index b22de86381c..5dc57d37e75 100644 --- a/src/config/svc-monitor/svc_monitor/tests/test_common_utils.py +++ b/src/config/svc-monitor/svc_monitor/tests/test_common_utils.py @@ -130,6 +130,16 @@ def create_test_virtual_machine(fq_name_str): vm.proj_fq_name = ['fake-domain', 'fake-project'] return vm +def create_test_virtual_router(fq_name_str): + vr_obj = {} + vr_obj['fq_name'] = fq_name_str.split(':') + vr_obj['name'] = fq_name_str.split(':')[0] + vr_obj['uuid'] = fq_name_str + vr_obj['display_name'] = fq_name_str + vr = VirtualRouterSM.locate(vr_obj['uuid'], vr_obj) + vr.agent_state = True + return vr + def create_test_security_group(fq_name_str): sg_obj = {} sg_obj['fq_name'] = fq_name_str.split(':') diff --git a/src/config/svc-monitor/svc_monitor/tests/test_snat.py b/src/config/svc-monitor/svc_monitor/tests/test_snat.py index e8d08dce69a..10bf429a4f4 100644 --- a/src/config/svc-monitor/svc_monitor/tests/test_snat.py +++ b/src/config/svc-monitor/svc_monitor/tests/test_snat.py @@ -25,6 +25,9 @@ def setUp(self): self.mocked_vnc.virtual_network_create = test_utils.vn_create self.mocked_vnc.instance_ip_create = test_utils.iip_create + self.mocked_scheduler = mock.MagicMock() + self.mocked_scheduler.schedule = mock.Mock(return_value=('fake-virtual-router')) + self.nova_mock = mock.MagicMock() self.mocked_db = mock.MagicMock() @@ -35,7 +38,7 @@ def setUp(self): self.netns_manager = NetworkNamespaceManager( db=self.mocked_db, logger=mock.MagicMock(), - vnc_lib=self.mocked_vnc, vrouter_scheduler=mock.MagicMock(), + vnc_lib=self.mocked_vnc, vrouter_scheduler=self.mocked_scheduler, nova_client=self.nova_mock, agent_manager=self.mocked_manager, args=self.mocked_args) @@ -58,6 +61,7 @@ def test_snat_instance_create(self): test_utils.create_test_virtual_network('fake-domain:fake-project:public-vn') test_utils.create_test_virtual_network('fake-domain:fake-project:fake-vn-uuid') test_utils.create_test_security_group('fake-domain:fake-project:default') + test_utils.create_test_virtual_router('fake-virtual-router') st = test_utils.create_test_st(name='snat-template', virt_type='network-namespace',