From 61cafbf3e7f8377fb5917f684a4753b2e73992f2 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Fri, 22 Jan 2016 15:49:55 -0800 Subject: [PATCH] Alarmgen should ensure that we always able to catch the gevent kill exception in worker gevents. We go into a sleep after a kafka error. We should sleep in a section that can catch exceptions AddressMismatchCompute and VrouterInterface alarms should not be raised when the VrouterAgent struct is absent. AddressMismatchControl alarms should not be raised when the BgpRouterState struct is absent When we check the object-type in api-server (during ContrailConfig insertion), we must replace "-" by "_" Use "oper" instead of obj_dict to decide whether the object is being deleted. See: https://github.com/Juniper/contrail-controller/commit/bf6cba2bb5b061bfd56fa99c72b7b2323f08486f Change-Id: I2d3f8f66ac8ad8df13db816936e98ed7320c54a0 Closes-Bug: #1533158 Closes-Bug: #1536085 --- src/config/api-server/vnc_cfg_ifmap.py | 9 +- src/opserver/partition_handler.py | 6 +- .../plugins/alarm_address_mismatch/main.py | 90 +++++++++++++------ .../plugins/alarm_config_incorrect/main.py | 1 + .../plugins/alarm_vrouter_interface/main.py | 4 +- src/opserver/uveserver.py | 7 +- 6 files changed, 81 insertions(+), 36 deletions(-) diff --git a/src/config/api-server/vnc_cfg_ifmap.py b/src/config/api-server/vnc_cfg_ifmap.py index d1b785478d1..fb6e8fa6dca 100644 --- a/src/config/api-server/vnc_cfg_ifmap.py +++ b/src/config/api-server/vnc_cfg_ifmap.py @@ -1678,7 +1678,7 @@ def dbe_uve_trace(self, oper, typ, uuid, obj_dict): else: oo['name'] = self.uuid_to_fq_name(uuid) oo['value'] = obj_dict - oo['type'] = typ + oo['type'] = typ.replace('-', '_') req_id = get_trace_id() db_trace = DBRequestTrace(request_id=req_id) @@ -1708,10 +1708,11 @@ def dbe_uve_trace(self, oper, typ, uuid, obj_dict): else: return - if oo['value']: - cc = ContrailConfig(name=ukey, elements=emap) + if oper.upper() == 'DELETE': + cc = ContrailConfig(name=ukey, elements=emap, deleted=True) else: - cc = ContrailConfig(name=ukey, elements={}, deleted=True) + cc = ContrailConfig(name=ukey, elements=emap) + cfg_msg = ContrailConfigTrace(data=cc, table=utab, sandesh=self._sandesh) cfg_msg.send(sandesh=self._sandesh) diff --git a/src/opserver/partition_handler.py b/src/opserver/partition_handler.py index 5a937d9b1c2..64ca4194a16 100644 --- a/src/opserver/partition_handler.py +++ b/src/opserver/partition_handler.py @@ -557,8 +557,12 @@ def msg_handler(self, mlist): def _run(self): pcount = 0 + pause = False while True: try: + if pause: + gevent.sleep(2) + pause = False self._logger.error("New KafkaClient %s" % self._topic) self._kfk = KafkaClient(self._brokers , "kc-" + self._topic) try: @@ -618,7 +622,7 @@ def _run(self): self._logger.error("%s : traceback %s" % \ (messag, traceback.format_exc())) self.stop_partition() - gevent.sleep(2) + pause = True self._logger.error("Stopping %s pcount %d" % (self._topic, pcount)) partdb = self.stop_partition() diff --git a/src/opserver/plugins/alarm_address_mismatch/main.py b/src/opserver/plugins/alarm_address_mismatch/main.py index 32361f48fc7..58864ad3c73 100644 --- a/src/opserver/plugins/alarm_address_mismatch/main.py +++ b/src/opserver/plugins/alarm_address_mismatch/main.py @@ -11,36 +11,62 @@ def __init__(self): def __call__(self, uve_key, uve_data): or_list = [] - try: - uattr = uve_data["ContrailConfig"]["elements"] - if isinstance(uattr,list): - uattr = uattr[0][0] - lval = json.loads(uattr["virtual_router_ip_address"]) - except KeyError: - lval = None + and_list = [] + trigger = True - try: - rval1 = uve_data["VrouterAgent"]["self_ip_list"] - except KeyError: - rval1 = None + if trigger: + if "ContrailConfig" not in uve_data: + trigger = False + else: + and_list.append(AlarmElement(\ + rule=AlarmTemplate(oper="!=", + operand1=Operand1(keys=["ContrailConfig"]), + operand2=Operand2(json_value="null")), + json_operand1_value=json.dumps({}))) - try: - rval2 = uve_data["VrouterAgent"]["control_ip"] - except KeyError: - rval2 = None + try: + uattr = uve_data["ContrailConfig"]["elements"] + if isinstance(uattr,list): + uattr = uattr[0][0] + lval = json.loads(uattr["virtual_router_ip_address"]) + except KeyError: + lval = None - if not isinstance(rval1,list) or lval not in rval1: - and_list = [] - and_list.append(AlarmElement(\ - rule=AlarmTemplate(oper="not in", - operand1=Operand1(keys=\ - ["ContrailConfig","elements","virtual_router_ip_address"], - json=2), - operand2=Operand2(keys=["VrouterAgent","self_ip_list"])), - json_operand1_value=json.dumps(lval), - json_operand2_value=json.dumps(rval1))) + if trigger: + if "VrouterAgent" not in uve_data: + trigger = False + else: + and_list.append(AlarmElement(\ + rule=AlarmTemplate(oper="!=", + operand1=Operand1(keys=["VrouterAgent"]), + operand2=Operand2(json_value="null")), + json_operand1_value=json.dumps({}))) + + if trigger: + try: + rval1 = uve_data["VrouterAgent"]["self_ip_list"] + except KeyError: + rval1 = None + + if not isinstance(rval1,list) or lval not in rval1: + and_list.append(AlarmElement(\ + rule=AlarmTemplate(oper="not in", + operand1=Operand1(keys=\ + ["ContrailConfig","elements","virtual_router_ip_address"], + json=2), + operand2=Operand2(keys=["VrouterAgent","self_ip_list"])), + json_operand1_value=json.dumps(lval), + json_operand2_value=json.dumps(rval1))) + else: + trigger = False - if len(and_list) > 0 and lval != rval2: + if trigger: + try: + rval2 = uve_data["VrouterAgent"]["control_ip"] + except KeyError: + rval2 = None + + if lval != rval2: and_list.append(AlarmElement(\ rule=AlarmTemplate(oper="!=", operand1=Operand1(keys=\ @@ -50,9 +76,11 @@ def __call__(self, uve_key, uve_data): operand2=Operand2(keys=["VrouterAgent","control_ip"])), json_operand1_value=json.dumps(lval), json_operand2_value=json.dumps(rval2))) - or_list.append(AllOf(all_of=and_list)) + else: + trigger = False - if len(or_list): + if trigger: + or_list.append(AllOf(all_of=and_list)) return or_list else: return None @@ -66,6 +94,9 @@ def __init__(self): def __call__(self, uve_key, uve_data): + if "ContrailConfig" not in uve_data: + return None + try: uattr = uve_data["ContrailConfig"]["elements"] if isinstance(uattr,list): @@ -74,6 +105,9 @@ def __call__(self, uve_key, uve_data): except KeyError: lval = None + if "BgpRouterState" not in uve_data: + return None + try: rval = uve_data["BgpRouterState"]["bgp_router_ip_list"] except KeyError: diff --git a/src/opserver/plugins/alarm_config_incorrect/main.py b/src/opserver/plugins/alarm_config_incorrect/main.py index 2c4b617a6fb..8df38d49303 100644 --- a/src/opserver/plugins/alarm_config_incorrect/main.py +++ b/src/opserver/plugins/alarm_config_incorrect/main.py @@ -1,5 +1,6 @@ from opserver.plugins.alarm_base import * from opserver.sandesh.alarmgen_ctrl.sandesh_alarm_base.ttypes import * +import json class ConfIncorrect(AlarmBase): def __init__(self, sev = AlarmBase.SYS_ERR): diff --git a/src/opserver/plugins/alarm_vrouter_interface/main.py b/src/opserver/plugins/alarm_vrouter_interface/main.py index 672a6c6b0ef..ae5b25a5557 100644 --- a/src/opserver/plugins/alarm_vrouter_interface/main.py +++ b/src/opserver/plugins/alarm_vrouter_interface/main.py @@ -11,7 +11,7 @@ def __init__(self): def __call__(self, uve_key, uve_data): or_list = [] - if not "VrouterAgent" in uve_data: + if "VrouterAgent" in uve_data: and_list = [] and_list.append(AlarmElement(\ rule=AlarmTemplate(oper="!=", @@ -21,7 +21,7 @@ def __call__(self, uve_key, uve_data): ust = uve_data["VrouterAgent"] - if not "error_intf_list" in ust: + if "error_intf_list" in ust: and_list.append(AlarmElement(\ rule=AlarmTemplate(oper="!=", operand1=Operand1(keys=["VrouterAgent","error_intf_list"]), diff --git a/src/opserver/uveserver.py b/src/opserver/uveserver.py index c60fd1aa3b1..9898a664d17 100644 --- a/src/opserver/uveserver.py +++ b/src/opserver/uveserver.py @@ -46,10 +46,11 @@ def redis_instances(self): def update_redis_uve_list(self, redis_uve_list): newlist = set(redis_uve_list) - + chg = False # if some redis instances are gone, remove them from our map for test_elem in self._redis_uve_map.keys(): if test_elem not in newlist: + chg = True r_ip = test_elem[0] r_port = test_elem[1] del self._redis_uve_map[test_elem] @@ -59,11 +60,15 @@ def update_redis_uve_list(self, redis_uve_list): # new redis instances need to be inserted into the map for test_elem in newlist: if test_elem not in self._redis_uve_map: + chg = True r_ip = test_elem[0] r_port = test_elem[1] self._redis_uve_map[test_elem] = None ConnectionState.update(ConnectionType.REDIS_UVE,\ r_ip+":"+str(r_port), ConnectionStatus.INIT) + if chg: + self._logger.error("updated redis_uve_list %s" % str(self._redis_uve_map)) + # Exercise redis connections to update health if len(newlist): self.get_uve("ObjectCollectorInfo:__NONE__", False, None)