diff --git a/ifmap_frontend.py b/ifmap_frontend.py index a392c0a..73f15bd 100644 --- a/ifmap_frontend.py +++ b/ifmap_frontend.py @@ -1435,7 +1435,7 @@ def _generate_client_impl(self, gen_fname, gen_type_pfx): list_args = list_args + ", parent_id = None, parent_fq_name = None" if ident.getReferences(): list_args = list_args + ", back_ref_id = None" - list_args = list_args + ", obj_uuids = None, fields = None, detail = False, count = False" + list_args = list_args + ", obj_uuids = None, fields = None, detail = False, count = False, filters = None" write(gen_file, " def %ss_list(%s):" \ %(method_name, list_args)) if parents: @@ -1454,7 +1454,7 @@ def _generate_client_impl(self, gen_fname, gen_type_pfx): resource_list_args += "parent_id = parent_id, parent_fq_name = parent_fq_name, " if ident.getReferences(): resource_list_args += "back_ref_id = back_ref_id, " - resource_list_args += "obj_uuids = obj_uuids, fields = fields, detail = detail, count = count" + resource_list_args += "obj_uuids=obj_uuids, fields=fields, detail=detail, count=count, filters=filters" write(gen_file, " return self.resource_list('%s', %s)" %(ident_name, resource_list_args)) write(gen_file, " #end %ss_list" %(method_name)) write(gen_file, "") @@ -2117,6 +2117,11 @@ def _generate_server_impl(self, gen_fname, gen_type_pfx): # GET on collection write(gen_file, " def %ss_http_get(self):" %(method_name)) + write(gen_file, " # gather list of uuids using 1. any specified anchors") + write(gen_file, " # 2. any specified filters") + write(gen_file, " # if not 'detail' return list with any specified 'fields'") + write(gen_file, " # if 'detail' return list with props+refs + any specified 'fields'") + write(gen_file, "") write(gen_file, " env = request.headers.environ") write(gen_file, " tenant_name = env.get(hdr_server_tenant(), 'default-project')") write(gen_file, " parent_uuids = None") @@ -2148,9 +2153,20 @@ def _generate_server_impl(self, gen_fname, gen_type_pfx): write(gen_file, " else:") write(gen_file, " count = False") write(gen_file, "") + write(gen_file, " filter_params = request.query.filters") + write(gen_file, " if filter_params:") + write(gen_file, " try:") + write(gen_file, " ff_key_vals = filter_params.split(',')") + write(gen_file, " ff_names = [ff.split('==')[0] for ff in ff_key_vals]") + write(gen_file, " ff_values = [ff.split('==')[1] for ff in ff_key_vals]") + write(gen_file, " filters = {'field_names': ff_names, 'field_values': ff_values}") + write(gen_file, " except Exception as e:") + write(gen_file, " abort(400, 'Invalid filter ' + filter_params)") + write(gen_file, " else:") + write(gen_file, " filters = None") write(gen_file, " db_conn = self._db_conn") write(gen_file, " (ok, result) = \\") - write(gen_file, " db_conn.dbe_list('%s', parent_uuids, back_ref_uuids, obj_uuids, count)" %(ident_name)) + write(gen_file, " db_conn.dbe_list('%s', parent_uuids, back_ref_uuids, obj_uuids, count, filters)" %(ident_name)) write(gen_file, " if not ok:") write(gen_file, " self.config_object_error(None, None, '%ss', 'http_get_collection', result)" % (method_name)) write(gen_file, " abort(404, result)") @@ -3045,6 +3061,42 @@ def _generate_cassandra_db_impl(self, gen_fname, gen_type_pfx): write(gen_file, " def _cassandra_%s_list(self, parent_uuids=None, back_ref_uuids=None," %(method_name)) write(gen_file, " obj_uuids=None, count=False, filters=None):") write(gen_file, " children_fq_names_uuids = []") + write(gen_file, " if filters:") + write(gen_file, " fnames = filters.get('field_names', [])") + write(gen_file, " fvalues = filters.get('field_values', [])") + write(gen_file, " filter_fields = [(fnames[i], fvalues[i]) for i in range(len(fnames))]") + write(gen_file, " else:") + write(gen_file, " filter_fields = []") + write(gen_file, "") + write(gen_file, " def filter_rows(coll_infos, filter_cols, filter_params):") + write(gen_file, " filt_infos = {}") + write(gen_file, " coll_rows = obj_uuid_cf.multiget(coll_infos.keys(),") + write(gen_file, " columns=filter_cols,") + write(gen_file, " column_count=10000000)") + write(gen_file, " for row in coll_rows:") + write(gen_file, " # give chance for zk heartbeat/ping") + write(gen_file, " gevent.sleep(0)") + write(gen_file, " full_match = True") + write(gen_file, " for fname, fval in filter_params:") + write(gen_file, " if coll_rows[row]['prop:%s' %(fname)] != fval:") + write(gen_file, " full_match = False") + write(gen_file, " break") + write(gen_file, " if full_match:") + write(gen_file, " filt_infos[row] = coll_infos[row]") + write(gen_file, " return filt_infos") + write(gen_file, " # end filter_rows") + write(gen_file, "") + write(gen_file, " def get_fq_name_uuid_list(obj_uuids):") + write(gen_file, " ret_list = []") + write(gen_file, " for obj_uuid in obj_uuids:") + write(gen_file, " try:") + write(gen_file, " obj_fq_name = self.uuid_to_fq_name(obj_uuid)") + write(gen_file, " ret_list.append((obj_fq_name, obj_uuid))") + write(gen_file, " except cfgm_common.exceptions.NoIdError:") + write(gen_file, " pass") + write(gen_file, " return ret_list") + write(gen_file, " # end get_fq_name_uuid_list") + write(gen_file, "") write(gen_file, " if parent_uuids:") write(gen_file, " # go from parent to child") write(gen_file, " obj_uuid_cf = self._obj_uuid_cf") @@ -3062,33 +3114,36 @@ def _generate_cassandra_db_impl(self, gen_fname, gen_type_pfx): write(gen_file, " else:") write(gen_file, " return (True, children_fq_names_uuids)") write(gen_file, "") - write(gen_file, " if count:") - write(gen_file, " child_count = sum([len(obj_rows[row]) for row in obj_rows])") - write(gen_file, " return (True, child_count)") - write(gen_file, "") - write(gen_file, " unsorted_fq_names_uuids = []") - write(gen_file, " for row in obj_rows:") - write(gen_file, " # give chance for zk heartbeat/ping") - write(gen_file, " gevent.sleep(0)") - write(gen_file, " cols = obj_rows[row]") - write(gen_file, " for col_name, col_val_ts in cols.items():") + write(gen_file, " def filter_rows_parent_anchor(sort=False):") + write(gen_file, " # flatten to [('children::', (,), *]") + write(gen_file, " all_cols = [cols for obj_key in obj_rows.keys() for cols in obj_rows[obj_key].items()]") + write(gen_file, " all_child_infos = {}") + write(gen_file, " for col_name, col_val_ts in all_cols:") + write(gen_file, " # give chance for zk heartbeat/ping") + write(gen_file, " gevent.sleep(0)") write(gen_file, " child_uuid = col_name.split(':')[2]") write(gen_file, " if obj_uuids and child_uuid not in obj_uuids:") write(gen_file, " continue") - write(gen_file, " try:") - write(gen_file, " child_fq_name = self.uuid_to_fq_name(child_uuid)") - write(gen_file, " except cfgm_common.exceptions.NoIdError:") - write(gen_file, " msg = 'uuid %s has no fq_name, present in parent!' %(child_uuid)") - write(gen_file, " self._logger(msg, level=SandeshLevel.SYS_ERR)") - write(gen_file, " continue") - write(gen_file, " tstamp = col_val_ts[1]") - write(gen_file, " unsorted_fq_names_uuids.append({'fq_name': child_fq_name, 'uuid': child_uuid,") - write(gen_file, " 'tstamp': tstamp})") + write(gen_file, " all_child_infos[child_uuid] = {'uuid': child_uuid, 'tstamp': col_val_ts[1]}") write(gen_file, "") - write(gen_file, " # sort children; TODO do this based on schema") - write(gen_file, " sorted_children = sorted(unsorted_fq_names_uuids, key = itemgetter('tstamp'))") - write(gen_file, " # re-write result's children without timestamp") - write(gen_file, " children_fq_names_uuids = [(child['fq_name'], child['uuid']) for child in sorted_children]") + write(gen_file, " filter_cols = ['prop:%s' %(fname) for fname, _ in filter_fields]") + write(gen_file, " if filter_cols:") + write(gen_file, " filt_child_infos = filter_rows(all_child_infos, filter_cols, filter_fields)") + write(gen_file, " else: # no filter specified") + write(gen_file, " filt_child_infos = all_child_infos") + write(gen_file, "") + write(gen_file, " if not sort:") + write(gen_file, " ret_child_infos = filt_child_infos.values()") + write(gen_file, " else:") + write(gen_file, " ret_child_infos = sorted(filt_child_infos.values(), key=itemgetter('tstamp'))") + write(gen_file, "") + write(gen_file, " return get_fq_name_uuid_list(r['uuid'] for r in ret_child_infos)") + write(gen_file, " # end filter_rows_parent_anchor") + write(gen_file, "") + write(gen_file, " if count:") + write(gen_file, " return (True, len(filter_rows_parent_anchor()))") + write(gen_file, "") + write(gen_file, " children_fq_names_uuids = filter_rows_parent_anchor(sort=True)") write(gen_file, "") write(gen_file, " if back_ref_uuids:") write(gen_file, " # go from anchor to backrefs") @@ -3107,31 +3162,56 @@ def _generate_cassandra_db_impl(self, gen_fname, gen_type_pfx): write(gen_file, " else:") write(gen_file, " return (True, children_fq_names_uuids)") write(gen_file, "") - write(gen_file, " if count:") - write(gen_file, " child_count = sum([len(obj_rows[row]) for row in obj_rows])") - write(gen_file, " return (True, child_count)") - write(gen_file, "") - write(gen_file, " for row in obj_rows:") - write(gen_file, " # give chance for zk heartbeat/ping") - write(gen_file, " gevent.sleep(0)") - write(gen_file, " cols = obj_rows[row]") - write(gen_file, " for col_name, col_val in cols.items():") + write(gen_file, " def filter_rows_backref_anchor():") + write(gen_file, " # flatten to [(':', (,), *]") + write(gen_file, " all_cols = [cols for obj_key in obj_rows.keys() for cols in obj_rows[obj_key].items()]") + write(gen_file, " all_backref_infos = {}") + write(gen_file, " for col_name, col_val_ts in all_cols:") + write(gen_file, " # give chance for zk heartbeat/ping") + write(gen_file, " gevent.sleep(0)") write(gen_file, " col_name_arr = col_name.split(':')") write(gen_file, " fq_name = col_name_arr[:-1]") write(gen_file, " obj_uuid = col_name_arr[-1]") write(gen_file, " if obj_uuids and obj_uuid not in obj_uuids:") write(gen_file, " continue") - write(gen_file, " children_fq_names_uuids.append((fq_name, obj_uuid))") + write(gen_file, " all_backref_infos[obj_uuid] = \\") + write(gen_file, " {'uuid': obj_uuid, 'fq_name': fq_name, 'tstamp': col_val_ts[1]}") + write(gen_file, "") + write(gen_file, " filter_cols = ['prop:%s' %(fname) for fname, _ in filter_fields]") + write(gen_file, " if filter_cols:") + write(gen_file, " filt_backref_infos = filter_rows(all_backref_infos, filter_cols, filter_fields)") + write(gen_file, " else: # no filter specified") + write(gen_file, " filt_backref_infos = all_backref_infos") + write(gen_file, "") + write(gen_file, " return [(br_info['fq_name'], br_info['uuid']) for br_info in filt_backref_infos.values()]") + write(gen_file, " # end filter_rows_backref_anchor") + write(gen_file, "") + write(gen_file, " if count:") + write(gen_file, " return (True, len(filter_rows_backref_anchor()))") + write(gen_file, "") + write(gen_file, " children_fq_names_uuids = filter_rows_backref_anchor()") write(gen_file, "") write(gen_file, " if not parent_uuids and not back_ref_uuids:") + write(gen_file, " obj_uuid_cf = self._obj_uuid_cf") write(gen_file, " if obj_uuids:") write(gen_file, " # exact objects specified") - write(gen_file, " for obj_uuid in obj_uuids:") - write(gen_file, " try:") - write(gen_file, " fq_name = self.uuid_to_fq_name(obj_uuid)") - write(gen_file, " except cfgm_common.exceptions.NoIdError:") - write(gen_file, " continue") - write(gen_file, " children_fq_names_uuids.append((fq_name, obj_uuid))") + write(gen_file, " def filter_rows_object_list():") + write(gen_file, " all_obj_infos = {}") + write(gen_file, " for obj_uuid in obj_uuids:") + write(gen_file, " all_obj_infos[obj_uuid] = None") + write(gen_file, "") + write(gen_file, " filter_cols = ['prop:%s' %(fname) for fname, _ in filter_fields]") + write(gen_file, " if filter_cols:") + write(gen_file, " filt_obj_infos = filter_rows(all_obj_infos, filter_cols, filter_fields)") + write(gen_file, " else: # no filters specified") + write(gen_file, " filt_obj_infos = all_obj_infos") + write(gen_file, "") + write(gen_file, " return get_fq_name_uuid_list(filt_obj_infos.keys())") + write(gen_file, " # end filter_rows_object_list") + write(gen_file, "") + write(gen_file, " if count:") + write(gen_file, " return (True, len(filter_rows_object_list()))") + write(gen_file, " children_fq_names_uuids = filter_rows_object_list()") write(gen_file, "") write(gen_file, " else: # grab all resources of this type") write(gen_file, " obj_fq_name_cf = self._obj_fq_name_cf") @@ -3143,16 +3223,28 @@ def _generate_cassandra_db_impl(self, gen_fname, gen_type_pfx): write(gen_file, " else:") write(gen_file, " return (True, children_fq_names_uuids)") write(gen_file, "") + write(gen_file, " def filter_rows_no_anchor():") + write(gen_file, " all_obj_infos = {}") + write(gen_file, " for col_name, col_val in cols.items():") + write(gen_file, " # give chance for zk heartbeat/ping") + write(gen_file, " gevent.sleep(0)") + write(gen_file, " col_name_arr = utils.decode_string(col_name).split(':')") + write(gen_file, " obj_uuid = col_name_arr[-1]") + write(gen_file, " all_obj_infos[obj_uuid] = (col_name_arr[:-1], obj_uuid)") + write(gen_file, "") + write(gen_file, " filter_cols = ['prop:%s' %(fname) for fname, _ in filter_fields]") + write(gen_file, " if filter_cols:") + write(gen_file, " filt_obj_infos = filter_rows(all_obj_infos, filter_cols, filter_fields)") + write(gen_file, " else: # no filters specified") + write(gen_file, " filt_obj_infos = all_obj_infos") + write(gen_file, "") + write(gen_file, " return filt_obj_infos.values()") + write(gen_file, " # end filter_rows_no_anchor") + write(gen_file, "") write(gen_file, " if count:") - write(gen_file, " return (True, len(cols.items()))") + write(gen_file, " return (True, len(filter_rows_no_anchor()))") write(gen_file, "") - write(gen_file, " for col_name, col_val in cols.items():") - write(gen_file, " # give chance for zk heartbeat/ping") - write(gen_file, " gevent.sleep(0)") - write(gen_file, " col_name_arr = utils.decode_string(col_name).split(':')") - write(gen_file, " fq_name = col_name_arr[:-1]") - write(gen_file, " obj_uuid = col_name_arr[-1]") - write(gen_file, " children_fq_names_uuids.append((fq_name, obj_uuid))") + write(gen_file, " children_fq_names_uuids = filter_rows_no_anchor()") write(gen_file, "") write(gen_file, " return (True, children_fq_names_uuids)") write(gen_file, " #end _cassandra_%s_list" %(method_name))