config-exception: Handle dbe_read raising NoIdError

dbe_read will raise this if object not available and caller
will determine if this can be gracefully handled.
Handled with 404 in GET/PUT/DEL for object in URL.
Handled with 500 if a parent/referred object is missing.

Database errors remapped to DatabaseUnavailable exception.

Partial-Bug: #1452835

Conflicts:
	src/config/api-server/tests/test_ip_alloc.py
	src/config/api-server/vnc_addr_mgmt.py
	src/config/api-server/vnc_cfg_api_server.py
	src/config/api-server/vnc_cfg_ifmap.py
	src/config/api-server/vnc_cfg_types.py
	src/config/common/tests/test_common.py

Change-Id: I991e311c20d7020bc84cd571f6484349cdabcfaa
(cherry picked from commit c874418)

src/config/api-server/tests/test_ip_alloc.py

@@ -658,7 +658,6 @@ def test_req_ip_allocation(self):
         self._vnc_lib.domain_delete(id=domain.uuid)
     #end
 
-
 #end class TestIpAlloc
 
 if __name__ == '__main__':

src/config/api-server/utils.py

@@ -52,7 +52,7 @@ def parse_args(args_str):
         'logging_conf': '',
         'logger_class': None,
         'multi_tenancy': True,
-        'disc_server_ip': None,
+        'disc_server_ip': '127.0.0.1',
         'disc_server_port': '5998',
         'zk_server_ip': '127.0.0.1:2181',
         'worker_id': '0',

src/config/api-server/vnc_addr_mgmt.py

@@ -8,6 +8,7 @@
 from pprint import pformat
 from cfgm_common import jsonutils as json
 import cfgm_common.exceptions
+import cfgm_common.utils
 try:
     #python2.7
     from collections import OrderedDict
@@ -364,6 +365,7 @@ def _get_subnet_dicts(self, vn_fq_name, vn_dict=None):
                                 obj_type='virtual-network',
                                 obj_ids={'uuid': vn_uuid},
                                 obj_fields=['network_ipam_refs'])
+
             if not ok:
                 raise VncError(result)
 
@@ -734,11 +736,18 @@ def net_check_subnet_delete(self, db_vn_dict, req_vn_dict):
                     'instance-ip', {'uuid': ref['uuid']})
             except cfgm_common.exceptions.NoIdError:
                 continue
-
             if not ok:
+                self.config_log(
+                    "Error in subnet delete instance-ip check: %s" %(result),
+                    level=SandeshLevel.SYS_ERR)
+                return False, result
+
+            inst_ip = result.get('instance_ip_address')
+            if not inst_ip:
+                self.config_log(
+                    "Error in subnet delete ip null: %s" %(ref['uuid']),
+                    level=SandeshLevel.SYS_ERR)
                 continue
-
-            inst_ip = result.get('instance_ip_address', None)
             if not all_matching_cidrs(inst_ip, requested_subnets):
                 return False,\
                     "Cannot Delete IP Block, IP(%s) is in use"\
@@ -751,9 +760,12 @@ def net_check_subnet_delete(self, db_vn_dict, req_vn_dict):
                     'floating-ip-pool', {'uuid': ref['uuid']})
             except cfgm_common.exceptions.NoIdError:
                 continue
-
             if not ok:
-                continue
+                self.config_log(
+                    "Error in subnet delete floating-ip-pool check: %s"
+                        %(result),
+                    level=SandeshLevel.SYS_ERR)
+                return False, result
 
             floating_ips = result.get('floating_ips', [])
             for floating_ip in floating_ips:
@@ -764,11 +776,20 @@ def net_check_subnet_delete(self, db_vn_dict, req_vn_dict):
                         'floating-ip', {'uuid': floating_ip['uuid']})
                 except cfgm_common.exceptions.NoIdError:
                     continue
-
-                if not ok:
+                if not read_ok:
+                    self.config_log(
+                        "Error in subnet delete floating-ip check: %s"
+                            %(read_result),
+                        level=SandeshLevel.SYS_ERR)
+                    return False, result
+
+                fip_addr = read_result.get('floating_ip_address')
+                if not fip_addr:
+                    self.config_log(
+                        "Error in subnet delete fip null: %s"
+                        %(floating_ip['uuid']),
+                        level=SandeshLevel.SYS_ERR)
                     continue
-
-                fip_addr = read_result.get('floating_ip_address', None)
                 if not all_matching_cidrs(fip_addr, requested_subnets):
                     return False,\
                         "Cannot Delete IP Block, Floating IP(%s) is in use"\
@@ -848,7 +869,11 @@ def ip_alloc_req(self, vn_fq_name, vn_dict=None, sub=None, asked_ip_addr=None,
 
     def ip_alloc_notify(self, ip_addr, vn_fq_name):
         vn_fq_name_str = ':'.join(vn_fq_name)
-        subnet_dicts = self._get_subnet_dicts(vn_fq_name)
+        try:
+            subnet_dicts = self._get_subnet_dicts(vn_fq_name)
+        except cfgm_common.exceptions.NoIdError:
+            return
+
         for subnet_name in subnet_dicts:
             # create subnet_obj internally if it was created by some other
             # api-server before
@@ -957,6 +982,41 @@ def ip_free_notify(self, ip_addr, vn_fq_name):
                 break
     # end ip_free_notify
 
+    # Given IP address count on given virtual network, subnet/List of subnet
+    def ip_count(self, obj_dict, subnet=None):
+        db_conn = self._get_db_conn()
+        addr_num = 0
+        if not subnet:
+            return addr_num
+
+        instip_refs = obj_dict.get('instance_ip_back_refs', None)
+        if instip_refs:
+            for ref in instip_refs:
+                uuid = ref['uuid']
+                try:
+                    (ok, result) = db_conn.dbe_read(
+                        'instance-ip', {'uuid': uuid})
+                    inst_ip = result.get('instance_ip_address')
+                    if not inst_ip:
+                        self.config_log(
+                            "Error in ip_count, ip null: %s" %(uuid),
+                            level=SandeshLevel.SYS_ERR)
+                        continue
+                    if IPAddress(inst_ip) in IPNetwork(subnet):
+                        addr_num += 1
+                except cfgm_common.exceptions.NoIdError:
+                    continue
+                except Exception as e:
+                    ok = False
+                    result = cfgm_common.utils.detailed_traceback()
+                if not ok:
+                    self.config_log(result, level=SandeshLevel.SYS_ERR)
+                    continue
+
+
+        return addr_num
+    # end ip_count
+
     def mac_alloc(self, obj_dict):
         uid = obj_dict['uuid']
         return '02:%s:%s:%s:%s:%s' % (uid[0:2], uid[2:4], uid[4:6],

src/config/api-server/vnc_cfg_api_server.py

@@ -597,17 +597,32 @@ def documentation_http_get(self, filename):
 
     def ref_update_http_post(self):
         self._post_common(bottle.request, None, None)
-        obj_type = bottle.request.json['type']
-        obj_uuid = bottle.request.json['uuid']
-        ref_type = bottle.request.json['ref-type'].replace('-', '_')
-        operation = bottle.request.json['operation']
+        # grab fields
+        obj_type = bottle.request.json.get('type')
+        obj_uuid = bottle.request.json.get('uuid')
+        ref_type = bottle.request.json.get('ref-type')
+        operation = bottle.request.json.get('operation')
         ref_uuid = bottle.request.json.get('ref-uuid')
         ref_fq_name = bottle.request.json.get('ref-fq-name')
         attr = bottle.request.json.get('attr')
 
+        # validate fields
+        if None in (obj_type, obj_uuid, ref_type, operation):
+            err_msg = 'Bad Request: type/uuid/ref-type/operation is null: '
+            err_msg += '%s, %s, %s, %s.' \
+                        %(obj_type, obj_uuid, ref_type, operation)
+            bottle.abort(400, err_msg)
+
+        if operation.upper() not in ['ADD', 'DELETE']:
+            err_msg = 'Bad Request: operation should be add or delete: %s' \
+                      %(operation)
+            bottle.abort(400, err_msg)
+
         if not ref_uuid and not ref_fq_name:
-            bottle.abort(404, 'Either ref-uuid or ref-fq-name must be specified')
+            err_msg = 'Bad Request: Either ref-uuid or ref-fq-name must be specified'
+            bottle.abort(400, err_msg)
 
+        ref_type = ref_type.replace('-', '_')
         if not ref_uuid:
             try:
                 ref_uuid = self._db_conn.fq_name_to_uuid(ref_type, ref_fq_name)
@@ -636,6 +651,19 @@ def ref_update_http_post(self):
                 fq_name = self._db_conn.uuid_to_fq_name(obj_uuid)
             except NoIdError:
                 bottle.abort(404, 'UUID ' + obj_uuid + ' not found')
+            try:
+                (read_ok, read_result) = self._db_conn.dbe_read(
+                                              obj_type, bottle.request.json)
+            except NoIdError:
+                bottle.abort(404, 'Object Not Found: ' + obj_uuid)
+            except Exception as e:
+                read_ok = False
+                read_result = cfgm_common.utils.detailed_traceback()
+            if not read_ok:
+                self.config_object_error(obj_uuid, None, obj_type, 'ref_update', read_result)
+                bottle.abort(500, read_result)
+
+            obj_dict = read_result
             if operation == 'ADD':
                 if ref_type+'_refs' not in obj_dict:
                     obj_dict[ref_type+'_refs'] = []
@@ -645,10 +673,6 @@ def ref_update_http_post(self):
                     if old_ref['to'] == ref_fq_name or old_ref['uuid'] == ref_uuid:
                         obj_dict[ref_type+'_refs'].remove(old_ref)
                         break
-            else:
-                msg = 'Unknown operation ' + operation
-                self.config_object_error(obj_uuid, None, obj_type, 'ref_update', msg)
-                bottle.abort(409, msg)
 
             (ok, put_result) = r_class.http_put(obj_uuid, fq_name, obj_dict, self._db_conn)
             if not ok:
@@ -1472,7 +1496,17 @@ def vn_subnet_ip_count_http_post(self, id):
 
         # expected format {"subnet" : ["2.1.1.0/24", "1.1.1.0/24"]
         req_dict = bottle.request.json
-        (_, obj_dict) = self._db_conn.dbe_read('virtual-network', {'uuid': id})
+        try:
+            (ok, result) = self._db_conn.dbe_read('virtual-network', {'uuid': id})
+        except NoIdError as e:
+            bottle.abort(404, str(e))
+        except Exception as e:
+            ok = False
+            result = cfgm_common.utils.detailed_traceback()
+        if not ok:
+            bottle.abort(500, result)
+
+        obj_dict = result
         subnet_list = req_dict[
             'subnet_list'] if 'subnet_list' in req_dict else []
         result = vnc_cfg_types.VirtualNetworkServer.subnet_ip_count(

src/config/api-server/vnc_cfg_ifmap.py

@@ -870,9 +870,11 @@ def dbe_update_publish(self, obj_type, obj_ids):
     # end dbe_update_publish
 
     def _dbe_update_notification(self, obj_info):
-        (ok, result) = self._db_client_mgr.dbe_read(obj_info['type'], obj_info)
-        if not ok:
-            raise Exception(result)
+        try:
+            (ok, result) = self._db_client_mgr.dbe_read(obj_info['type'], obj_info)
+        except NoIdError as e:
+            # No error, we will hear a delete shortly
+            return
 
         new_obj_dict = result
 
@@ -1114,9 +1116,17 @@ def _update_default_quota(self):
 
         proj_id = self.fq_name_to_uuid('project',
                                        ['default-domain', 'default-project'])
-        (ok, proj_dict) = self.dbe_read('project', {'uuid':proj_id})
+        try:
+            (ok, result) = self.dbe_read('project', {'uuid':proj_id})
+        except NoIdError as e:
+            ok = False
+            result = 'Project Not Found: %s' %(proj_id)
         if not ok:
+            self.config_log("Updating default quota failed: %s." %(result),
+                level=SandeshLevel.SYS_ERR)
             return
+
+        proj_dict = result
         quota = QuotaType()
 
         proj_dict['quota'] = default_quota
@@ -1414,6 +1424,12 @@ def dbe_read(self, obj_type, obj_ids, obj_fields=None):
                                                              [obj_ids['uuid']],
                                                              obj_fields)
         except NoIdError as e:
+            # if NoIdError is for obj itself (as opposed to say for parent
+            # or ref), let caller decide if this can be handled gracefully
+            # by re-raising
+            if e._unknown_id == obj_ids['uuid']:
+                raise
+
             return (False, str(e))
 
         return (ok, cassandra_result[0])
@@ -1428,7 +1444,7 @@ def dbe_count_children(self, obj_type, obj_id, child_type):
             return (False, str(e))
 
         return (ok, cassandra_result)
-    # end dbe_read
+    # end dbe_count_children
 
     def dbe_read_multi(self, obj_type, obj_ids_list, obj_fields=None):
         if not obj_ids_list: