From 7fc2d85894d9756dd23e8a62ebdd4e3e3ea095c7 Mon Sep 17 00:00:00 2001 From: Hampapur Ajay Date: Wed, 2 Nov 2016 16:53:44 -0700 Subject: [PATCH] Update connection to keystone status in the keystone _polling_ (for projects/domains) context. The status reported will be effective as of the last poll (default 60sec). Update status on edge (up->down, down->up) instead of at every poll. Closes-Bug: 1636343 Change-Id: I5c5e2f0cff004f0c25ce529fe519a92a57c5c840 --- src/base/sandesh/process_info.sandesh | 4 +- src/config/api-server/vnc_cfg_ifmap.py | 2 +- src/config/common/tests/test_common.py | 16 -- .../vnc_openstack/vnc_openstack/__init__.py | 229 +++++++++++------- .../vnc_openstack/tests/test_case.py | 14 +- .../vnc_openstack/tests/test_naming.py | 61 +++++ 6 files changed, 209 insertions(+), 117 deletions(-) diff --git a/src/base/sandesh/process_info.sandesh b/src/base/sandesh/process_info.sandesh index 3393fddb3b4..4ef7cae51ed 100644 --- a/src/base/sandesh/process_info.sandesh +++ b/src/base/sandesh/process_info.sandesh @@ -24,6 +24,7 @@ enum ConnectionType { REDIS_UVE, UVEPARTITIONS, KAFKA_PUB, + OTHER, } const map ConnectionTypeNames = { @@ -39,7 +40,8 @@ const map ConnectionTypeNames = { ConnectionType.TOR : "ToR", ConnectionType.REDIS_UVE: "Redis-UVE", ConnectionType.UVEPARTITIONS : "UvePartitions", - ConnectionType.KAFKA_PUB : "KafkaPub" + ConnectionType.KAFKA_PUB : "KafkaPub", + ConnectionType.OTHER: "Generic Connection" } enum ConnectionStatus { diff --git a/src/config/api-server/vnc_cfg_ifmap.py b/src/config/api-server/vnc_cfg_ifmap.py index fdc50476de6..a2207eae60b 100644 --- a/src/config/api-server/vnc_cfg_ifmap.py +++ b/src/config/api-server/vnc_cfg_ifmap.py @@ -1055,7 +1055,7 @@ def uuid_to_fq_name(self, uuid): def dbe_uve_trace(self, oper, typ, uuid, body): self._db_client_mgr.dbe_uve_trace(oper, typ, uuid, body) - # end uuid_to_fq_name + # end dbe_uve_trace def dbe_oper_publish_pending(self): return self.num_pending_messages() diff --git a/src/config/common/tests/test_common.py b/src/config/common/tests/test_common.py index 7a213e437ad..22f5e77ca31 100644 --- a/src/config/common/tests/test_common.py +++ b/src/config/common/tests/test_common.py @@ -394,21 +394,6 @@ def flexmocks(mocks): setattr(cls, method_name, method) # end flexmocks -@contextlib.contextmanager -def flexmocks(mocks): - orig_values = {} - try: - for cls, method_name, val in mocks: - kwargs = {method_name: val} - # save orig cls.method_name - orig_values[(cls, method_name)] = getattr(cls, method_name) - flexmock(cls, **kwargs) - yield - finally: - for (cls, method_name), method in orig_values.items(): - setattr(cls, method_name, method) -# end flexmocks - def setup_extra_flexmock(mocks): for (cls, method_name, val) in mocks: kwargs = {method_name: val} @@ -651,7 +636,6 @@ def setUpClass(cls, extra_mocks=None, extra_config_knobs=None, db=None): cfgm_common.zkclient.LOG_DIR = './' gevent.wsgi.WSGIServer.handler_class = FakeWSGIHandler - # end setUp cls.orig_mocked_values = setup_mocks(cls.mocks + (extra_mocks or [])) diff --git a/src/config/vnc_openstack/vnc_openstack/__init__.py b/src/config/vnc_openstack/vnc_openstack/__init__.py index d29830a7700..fff69aa180d 100644 --- a/src/config/vnc_openstack/vnc_openstack/__init__.py +++ b/src/config/vnc_openstack/vnc_openstack/__init__.py @@ -13,6 +13,7 @@ import bottle import logging import logging.handlers +from datetime import datetime import Queue import ConfigParser import keystoneclient.v2_0.client as keystone @@ -27,6 +28,9 @@ from cfgm_common.utils import cgitb_hook from pysandesh.sandesh_base import * from pysandesh.sandesh_logger import * +from pysandesh.connection_info import ConnectionState +from pysandesh.gen_py.process_info.ttypes import ConnectionStatus +from pysandesh.gen_py.process_info.ttypes import ConnectionType as ConnType from vnc_api import vnc_api from vnc_api.gen.resource_xsd import * from vnc_api.gen.resource_common import * @@ -103,7 +107,7 @@ def fill_keystone_opts(obj, conf_sections): 'keystone_resync_interval_secs') except ConfigParser.NoOptionError: resync_interval = '60' - obj._resync_interval_secs = int(resync_interval) + obj._resync_interval_secs = float(resync_interval) try: # Number of workers used to process keystone project resyncing @@ -222,7 +226,6 @@ def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh): fill_keystone_opts(self, conf_sections) if 'v3' in self._auth_url.split('/')[-1]: - self._get_keystone_conn = self._ksv3_get_conn self._ks_domains_list = self._ksv3_domains_list self._ks_domain_get = self._ksv3_domain_get self._ks_projects_list = self._ksv3_projects_list @@ -232,7 +235,6 @@ def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh): self._del_project_from_vnc = self._ksv3_del_project_from_vnc self._vnc_default_domain_id = None else: - self._get_keystone_conn = self._ksv2_get_conn self._ks_domains_list = None self._ks_domain_get = None self._ks_projects_list = self._ksv2_projects_list @@ -242,6 +244,9 @@ def __init__(self, api_server_ip, api_server_port, conf_sections, sandesh): self._del_project_from_vnc = self._ksv2_del_project_from_vnc self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.INIT, message='', + server_addrs=[self._auth_url]) self._vnc_lib = None # resync failures, don't retry forever @@ -290,38 +295,56 @@ def _get_vnc_conn(self): tenant_name=self._admin_tenant) # end _get_vnc_conn + def _get_keystone_conn(self): + if self._ks: + return + + if 'v3' in self._auth_url.split('/')[-1]: + self._ks = self._ksv3_get_conn() + if self._endpoint_type and self._ks.service_catalog: + self._ks.management_url = \ + self._ks.service_catalog.get_urls( + service_type='identity', + endpoint_type=self._endpoint_type)[0] + else: + self._ks = self._ksv2_get_conn() + + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.UP, message='', + server_addrs=[self._auth_url]) + # end _get_keystone_conn + def _ksv2_get_conn(self): - if not self._ks: - if self._admin_token: - if self._insecure: - self._ks = keystone.Client(token=self._admin_token, - endpoint=self._auth_url, - insecure=self._insecure) - elif not self._insecure and self._use_certs: - self._ks = keystone.Client(token=self._admin_token, - endpoint=self._auth_url, - cacert=self._kscertbundle) - else: - self._ks = keystone.Client(token=self._admin_token, - endpoint=self._auth_url) + if self._admin_token: + if self._insecure: + return keystone.Client(token=self._admin_token, + endpoint=self._auth_url, + insecure=self._insecure) + elif not self._insecure and self._use_certs: + return keystone.Client(token=self._admin_token, + endpoint=self._auth_url, + cacert=self._kscertbundle) else: - if self._insecure: - self._ks = keystone.Client(username=self._auth_user, - password=self._auth_passwd, - tenant_name=self._admin_tenant, - auth_url=self._auth_url, - insecure=self._insecure) - elif not self._insecure and self._use_certs: - self._ks = keystone.Client(username=self._auth_user, - password=self._auth_passwd, - tenant_name=self._admin_tenant, - auth_url=self._auth_url, - cacert=self._kscertbundle) - else: - self._ks = keystone.Client(username=self._auth_user, - password=self._auth_passwd, - tenant_name=self._admin_tenant, - auth_url=self._auth_url) + return keystone.Client(token=self._admin_token, + endpoint=self._auth_url) + else: + if self._insecure: + return keystone.Client(username=self._auth_user, + password=self._auth_passwd, + tenant_name=self._admin_tenant, + auth_url=self._auth_url, + insecure=self._insecure) + elif not self._insecure and self._use_certs: + return keystone.Client(username=self._auth_user, + password=self._auth_passwd, + tenant_name=self._admin_tenant, + auth_url=self._auth_url, + cacert=self._kscertbundle) + else: + return keystone.Client(username=self._auth_user, + password=self._auth_passwd, + tenant_name=self._admin_tenant, + auth_url=self._auth_url) # end _ksv2_get_conn def _ksv2_projects_list(self): @@ -336,7 +359,12 @@ def _ksv2_project_get(self, id): try: return {'name': self._ks.tenants.get(id).name} except Exception as e: - self._ks = None + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) self._get_keystone_conn() return {'name': self._ks.tenants.get(id).name} # end _ksv2_project_get @@ -408,47 +436,38 @@ def _ksv2_del_project_from_vnc(self, project_id): # _ksv2_del_project_from_vnc def _ksv3_get_conn(self): - if not self._ks: - if self._admin_token: - if not self._insecure and self._use_certs: - self._ks = keystonev3.Client(token=self._admin_token, - endpoint=self._auth_url, - verify=self._kscertbundle) - else: - self._ks = keystonev3.Client(token=self._admin_token, - endpoint=self._auth_url, - insecure=self._insecure) - elif self._project_domain_name: - self._ks = keystonev3.Client(user_domain_name=self._user_domain_name, - username=self._auth_user, - password=self._auth_passwd, - project_domain_name=self._project_domain_name, - project_name=self._project_name, - auth_url=self._auth_url, - insecure=self._insecure) - else: - if not self._insecure and self._use_certs: - self._ks = keystonev3.Client(user_domain_name=self._user_domain_name, - username=self._auth_user, - password=self._auth_passwd, - domain_id=self._domain_id, - auth_url=self._auth_url, - verify=self._kscertbundle) - else: - self._ks = keystonev3.Client(user_domain_name=self._user_domain_name, - username=self._auth_user, - password=self._auth_passwd, - domain_id=self._domain_id, - auth_url=self._auth_url, - insecure=self._insecure) - - - if self._endpoint_type and self._ks.service_catalog: - self._ks.management_url = \ - self._ks.service_catalog.get_urls( - service_type='identity', - endpoint_type=self._endpoint_type)[0] - + if self._admin_token: + if not self._insecure and self._use_certs: + return keystonev3.Client(token=self._admin_token, + endpoint=self._auth_url, + verify=self._kscertbundle) + else: + return keystonev3.Client(token=self._admin_token, + endpoint=self._auth_url, + insecure=self._insecure) + elif self._project_domain_name: + return keystonev3.Client(user_domain_name=self._user_domain_name, + username=self._auth_user, + password=self._auth_passwd, + project_domain_name=self._project_domain_name, + project_name=self._project_name, + auth_url=self._auth_url, + insecure=self._insecure) + else: + if not self._insecure and self._use_certs: + return keystonev3.Client(user_domain_name=self._user_domain_name, + username=self._auth_user, + password=self._auth_passwd, + domain_id=self._domain_id, + auth_url=self._auth_url, + verify=self._kscertbundle) + else: + return keystonev3.Client(user_domain_name=self._user_domain_name, + username=self._auth_user, + password=self._auth_passwd, + domain_id=self._domain_id, + auth_url=self._auth_url, + insecure=self._insecure) # end _ksv3_get_conn def _ksv3_domains_list(self): @@ -465,8 +484,13 @@ def _ksv3_domain_id_to_uuid(self, domain_id): def _ksv3_domain_get(self, id=None): try: return {'name': self._ks.domains.get(id).name} - except: - self._ks = None + except Exception as e: + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) self._get_keystone_conn() return {'name': self._ks.domains.get(id).name} # end _ksv3_domain_get @@ -480,7 +504,12 @@ def _ksv3_project_get(self, id=None): project = self._ks.projects.get(id) return {'id': project.id, 'name': project.name, 'domain_id': project.domain_id} except Exception as e: - self._ks = None + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) self._get_keystone_conn() project = self._ks.projects.get(id) return {'id': project.id, 'name': project.name, 'domain_id': project.domain_id} @@ -650,7 +679,12 @@ def _resync_all_domains(self): for dom in self._ks_domains_list() if dom['id'] != 'default']) ks_domain_ids.add(self._vnc_default_domain_id) except Exception as e: - self._ks = None + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) return True # retry vnc_domain_ids = self._vnc_domain_ids @@ -688,7 +722,12 @@ def _resync_all_projects(self): [str(uuid.UUID(proj['id'])) for proj in self._ks_projects_list()]) except Exception as e: - self._ks = None + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) return True # retry vnc_project_ids = self._vnc_project_ids @@ -745,24 +784,34 @@ def _resync_domains_projects_forever(self): try: retry = self._resync_all_domains() if retry: - gevent.sleep(60) + gevent.sleep(self._resync_interval_secs) continue except Exception as e: - self._ks = None - self._cgitb_error_log() - self._sandesh_logger.error( - "Failed to resync domains: %s" % e) + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) + self._cgitb_error_log() + self._sandesh_logger.error( + "Failed to resync domains: %s" % e) try: retry = self._resync_all_projects() if retry: - gevent.sleep(60) + gevent.sleep(self._resync_interval_secs) continue except Exception as e: - self._ks = None - self._cgitb_error_log() - self._sandesh_logger.error( - "Failed to resync projects: %s" % e) + if self._ks is not None: + self._ks = None + ConnectionState.update(conn_type=ConnType.OTHER, + name='Keystone', status=ConnectionStatus.DOWN, + message='Error: %s at UTC %s' %(e, datetime.utcnow()), + server_addrs=[self._auth_url]) + self._cgitb_error_log() + self._sandesh_logger.error( + "Failed to resync projects: %s" % e) gevent.sleep(self._resync_interval_secs) diff --git a/src/config/vnc_openstack/vnc_openstack/tests/test_case.py b/src/config/vnc_openstack/vnc_openstack/tests/test_case.py index 29c8039f87b..2d2fb4e784e 100644 --- a/src/config/vnc_openstack/vnc_openstack/tests/test_case.py +++ b/src/config/vnc_openstack/vnc_openstack/tests/test_case.py @@ -46,10 +46,6 @@ def setup_flexmock(cls): setup_extra_flexmock([(keystone.Client, '__new__', get_keystone_client)]) # end setup_flexmock - def __init__(self, *args, **kwargs): - super(VncOpenstackTestCase, self).__init__(*args, **kwargs) - # end __init__ - @classmethod def setUpClass(cls, *args, **kwargs): setup_extra_flexmock([(stevedore.extension.ExtensionManager, '__new__', FakeExtensionManager)]) @@ -79,16 +75,16 @@ def setUpClass(cls, *args, **kwargs): class KeystoneSyncTestCase(VncOpenstackTestCase): @classmethod - def setUpClass(cls): + def setUpClass(cls, *args, **kwargs): cls.setup_flexmock() - super(KeystoneSyncTestCase, cls).setUpClass() + super(KeystoneSyncTestCase, cls).setUpClass(*args, **kwargs) # end setUpClass # end class KeystoneSyncTestCase class ResourceDriverTestCase(VncOpenstackTestCase): @classmethod - def setUpClass(cls): + def setUpClass(cls, *args, **kwargs): cls.setup_flexmock() - super(ResourceDriverTestCase, cls).setUpClass() - # end setUp + super(ResourceDriverTestCase, cls).setUpClass(*args, **kwargs) + # end setUpClass # end class ResourceDriverTestCase diff --git a/src/config/vnc_openstack/vnc_openstack/tests/test_naming.py b/src/config/vnc_openstack/vnc_openstack/tests/test_naming.py index fa4a20962c5..12df67f9f59 100644 --- a/src/config/vnc_openstack/vnc_openstack/tests/test_naming.py +++ b/src/config/vnc_openstack/vnc_openstack/tests/test_naming.py @@ -2,11 +2,15 @@ import json import uuid import logging +from gevent import monkey +monkey.patch_all() +from flexmock import flexmock from testtools.matchers import Equals, Contains, Not from testtools import content, content_type, ExpectedException from vnc_api.vnc_api import * +from pysandesh.connection_info import ConnectionState sys.path.append('../common/tests') from test_utils import * @@ -234,3 +238,60 @@ def test_dup_domain(self): openstack_driver._ks_domains_list = orig_ks_domains_list openstack_driver._ks_domain_get = orig_ks_domain_get # end class KeystoneSync + + +class KeystoneConnectionStatus(test_case.KeystoneSyncTestCase): + resync_interval = 0.5 + @classmethod + def setUpClass(cls): + super(KeystoneConnectionStatus, cls).setUpClass( + extra_config_knobs=[('DEFAULTS', 'keystone_resync_interval_secs', + cls.resync_interval)]) + # end setUpClass + + def test_connection_status_change(self): + # up->down->up transition check + openstack_driver = FakeExtensionManager.get_extension_objects( + 'vnc_cfg_api.resync')[0] + proj_id = str(uuid.uuid4()) + proj_name = self.id()+'verify-active' + test_case.get_keystone_client().tenants.add_tenant(proj_id, proj_name) + proj_obj = self._vnc_lib.project_read(id=proj_id) + conn_info = [ConnectionState._connection_map[x] + for x in ConnectionState._connection_map if x[1] == 'Keystone'][0] + self.assertThat(conn_info.status.lower(), Equals('up')) + + fake_list_invoked = [] + def fake_list(*args, **kwargs): + fake_list_invoked.append(True) + raise Exception("Fake Keystone Projects List exception") + + with test_common.flexmocks([ + (openstack_driver._ks.tenants, 'list', fake_list)]): + proj_id = str(uuid.uuid4()) + proj_name = self.id()+'verify-down' + test_case.get_keystone_client().tenants.add_tenant( + proj_id, proj_name) + openstack_driver._ks = None # force to re-connect on next poll + def verify_down(): + conn_info = [ConnectionState._connection_map[x] + for x in ConnectionState._connection_map + if x[1] == 'Keystone'][0] + self.assertThat(conn_info.status.lower(), Equals('down')) + + # verify up->down + gevent.sleep(self.resync_interval) + verify_down() + self.assertThat(len(fake_list_invoked), Equals(1)) + # should remain down + gevent.sleep(self.resync_interval) + verify_down() + self.assertThat(len(fake_list_invoked), Equals(2)) + + # sleep for a retry and verify down->up + gevent.sleep(self.resync_interval) + conn_info = [ConnectionState._connection_map[x] + for x in ConnectionState._connection_map if x[1] == 'Keystone'][0] + self.assertThat(conn_info.status.lower(), Equals('up')) + # end test_connection_status_change +# end class KeystoneConnectionStatus