From fe6c25151e8ad84a6c87f050dc9624ae9f520466 Mon Sep 17 00:00:00 2001 From: Sundaresan Rajangam Date: Tue, 5 Apr 2016 23:53:43 -0700 Subject: [PATCH] Support for prefix match for all index fields in the system/object log queries This patch adds support for prefix match for key fields such as Source, ModuleId, Messagetype and Category in the WHERE clause for system and object log queries. contrail-logs to support prefix match queries for --source, --message-type and --category Change-Id: I7afabe51a19766802981455fc7ebe3bb2b156acc Partial-Bug: #1566449 --- src/opserver/log.py | 27 +++++-- src/opserver/test/test_analytics_sys.py | 1 + src/opserver/test/test_log.py | 83 ++++++++++---------- src/opserver/test/utils/analytics_fixture.py | 20 +++++ src/query_engine/where_query.cc | 36 +++++++-- 5 files changed, 111 insertions(+), 56 deletions(-) diff --git a/src/opserver/log.py b/src/opserver/log.py index 21cae39aaef..f768424bc35 100755 --- a/src/opserver/log.py +++ b/src/opserver/log.py @@ -214,9 +214,14 @@ def query(self): and_filter = [] or_filter = [] if self._args.source is not None: + if self._args.source.endswith('*'): + val = self._args.source[:-1] + oper = OpServerUtils.MatchOp.PREFIX + else: + val = self._args.source + oper = OpServerUtils.MatchOp.EQUAL source_match = OpServerUtils.Match(name=VizConstants.SOURCE, - value=self._args.source, - op=OpServerUtils.MatchOp.EQUAL) + value=val, op=oper) where_msg.append(source_match.__dict__) if self._args.module is not None: @@ -226,17 +231,27 @@ def query(self): where_msg.append(module_match.__dict__) if self._args.category is not None: + if self._args.category.endswith('*'): + val = self._args.category[:-1] + oper = OpServerUtils.MatchOp.PREFIX + else: + val = self._args.category + oper = OpServerUtils.MatchOp.EQUAL category_match = OpServerUtils.Match( name=VizConstants.CATEGORY, - value=self._args.category, - op=OpServerUtils.MatchOp.EQUAL) + value=val, op=oper) where_msg.append(category_match.__dict__) if self._args.message_type is not None: + if self._args.message_type.endswith('*'): + val = self._args.message_type[:-1] + oper = OpServerUtils.MatchOp.PREFIX + else: + val = self._args.message_type + oper = OpServerUtils.MatchOp.EQUAL message_type_match = OpServerUtils.Match( name=VizConstants.MESSAGE_TYPE, - value=self._args.message_type, - op=OpServerUtils.MatchOp.EQUAL) + value=val, op=oper) where_msg.append(message_type_match.__dict__) if self._args.level is not None: diff --git a/src/opserver/test/test_analytics_sys.py b/src/opserver/test/test_analytics_sys.py index 69646984269..f95b32234f8 100755 --- a/src/opserver/test/test_analytics_sys.py +++ b/src/opserver/test/test_analytics_sys.py @@ -141,6 +141,7 @@ def test_02_message_table_query(self): assert vizd_obj.verify_message_table_messagetype() assert vizd_obj.verify_message_table_where_or() assert vizd_obj.verify_message_table_where_and() + assert vizd_obj.verify_message_table_where_prefix() assert vizd_obj.verify_message_table_filter() assert vizd_obj.verify_message_table_filter2() assert vizd_obj.verify_message_table_sort() diff --git a/src/opserver/test/test_log.py b/src/opserver/test/test_log.py index a17f0e9683d..a45a1bbc76b 100755 --- a/src/opserver/test/test_log.py +++ b/src/opserver/test/test_log.py @@ -12,18 +12,23 @@ test_num = 0 query_dict = {} -result_1 = [ +query_result = { +1: [ {u'Category': None, u'NodeType': u'Config', u'Level': 2147483647, u'InstanceId': u'0', u'Messagetype': u'UveVirtualNetworkConfigTrace', u'Source': u'a6s45', u'SequenceNum': 6867683, u'MessageTS': 1442429588898861, u'Xmlmessage': u'default-domain:demo:svc-vn-leftsvc-vn-left0', u'Type': 6, u'ModuleId': u'contrail-schema'} - ] -result_2 = [ +], +2: [ {u'Category': None, u'NodeType': u'Analytics', u'Level': 2147483647, u'InstanceId': u'0', u'Messagetype': u'GeneratorDbStatsUve', u'Source': u'a6s45', u'SequenceNum': 56411, u'MessageTS': 1442429889555171, u'Xmlmessage': u'a6s10:Compute:contrail-vrouter-agent:0MessageTable00120MessageTableCategory00120MessageTableKeyword0000MessageTableMessageType00120MessageTableModuleId00120MessageTableSource00120MessageTableTimestamp00120ObjectTable00120ObjectValueTable00120StatsTableByDblTagV30010StatsTableByStrTagV3001020StatsTableByU64TagV300200000000ComputeCpuState:cpu_info0050FieldNames:fields00960VrouterStatsAgent:flow_rate0040', u'Type': 6, u'ModuleId': u'contrail-collector'} -] -result_3 = [ +], +3: [ {u'ObjectId': u'virtual_network:default-domain:admin:vn1-take2'} -] -result_4 = [ +], +4: [ {u'ObjectLog': u'virtual_networkdefault-domain:admin:vn1-take2http://127.0.0.1:9100/virtual-networksposta6s45:/usr/bin/contrail-api127.0.0.1:9100{'virtual-network': {'fq_name': ['default-domain', 'admin', 'vn1-take2'], 'uuid': None, 'network_policy_refs': [], 'router_external': False, 'parent_type': 'project', 'id_perms': {u'enable': True, u'uuid': None, u'creator': None, u'created': 0, u'user_visible': True, u'last_modified': 0, u'permissions': {u'owner': u'cloud-admin', u'owner_access': 7, u'other_access': 7, u'group': u'cloud-admin-group', u'group_access': 7}, u'description': None}, 'display_name': 'vn1-take2', 'is_shared': False}}default-domain', u'Messagetype': u'VncApiConfigLog', u'Source': u'a6s45', u'MessageTS': 1442434711187905, u'SystemLog': None, u'ModuleId': u'contrail-api'} +], +5: [ +{u'Category': None, u'NodeType': u'Config', u'Level': 2147483647, u'InstanceId': u'0', u'Messagetype': u'UveVirtualNetworkConfigTrace', u'Source': u'nodec39', u'SequenceNum': 6867683, u'MessageTS': 1442429588898861, u'Xmlmessage': u'default-domain:demo:svc-vn-leftsvc-vn-left0', u'Type': 6, u'ModuleId': u'contrail-schema'} ] +} class LogQuerierTest(unittest.TestCase): @@ -35,34 +40,19 @@ def custom_post_url_http(url, params): @staticmethod def custom_get_query_result(opserver_ip, opserver_port, qid): - if (test_num == 1): - return result_1 - elif (test_num == 2): - return result_2 - elif (test_num == 3): - return result_3 - elif (test_num == 4): - return result_4 - else: + try: + return query_result[test_num] + except KeyError: return [] def custom_display(self, result): - if (test_num == 1): - self.assertTrue(result == result_1) - return - elif (test_num == 2): - self.assertTrue(result == result_2) - return - elif (test_num == 3): - self.assertTrue(result == result_3) - return - elif (test_num == 4): - self.assertTrue(result == result_4) - return - else: + try: + self.assertTrue(result == query_result[test_num]) + except KeyError: self.assertTrue(False) def setUp(self): + self.maxDiff = None self._querier = LogQuerier() flexmock(OpServerUtils).should_receive('post_url_http').replace_with(lambda x, y: self.custom_post_url_http(x, y)) @@ -83,9 +73,7 @@ def test_1_no_arg(self): expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable"}' expected_result_dict = json.loads(expected_result_str) - for key in expected_result_dict: - self.assertTrue(key in query_dict) - self.assertTrue(expected_result_dict[key] == query_dict[key]) + self.assertEqual(expected_result_dict, query_dict) # a few args #@unittest.skip("skip test_2_message_query") @@ -99,11 +87,9 @@ def test_2_message_query(self): self._querier.run() sys.argv = argv - expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable"}' + expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable", "where": [[{"suffix": null, "value2": null, "name": "Source", "value": "a6s45", "op": 1}, {"suffix": null, "value2": null, "name": "ModuleId", "value": "contrail-collector", "op": 1}, {"suffix": null, "value2": null, "name": "Messagetype", "value": "GeneratorDbStatsUve", "op": 1}]], "filter": [[{"suffix": null, "value2": null, "name": "NodeType", "value": "Analytics", "op": 1}, {"suffix": null, "value2": null, "name": "InstanceId", "value": 0, "op": 1}]]}' expected_result_dict = json.loads(expected_result_str) - for key in expected_result_dict: - self.assertTrue(key in query_dict) - self.assertTrue(expected_result_dict[key] == query_dict[key]) + self.assertEqual(expected_result_dict, query_dict) # a object values query #@unittest.skip("skip test_3_object_value") @@ -119,9 +105,7 @@ def test_3_object_value(self): expected_result_str = '{"table": "ConfigObjectTable", "start_time": "now-10m", "end_time": "now", "select_fields": ["ObjectId"]}' expected_result_dict = json.loads(expected_result_str) - for key in expected_result_dict: - self.assertTrue(key in query_dict) - self.assertTrue(expected_result_dict[key] == query_dict[key]) + self.assertEqual(expected_result_dict, query_dict) # a object id query #@unittest.skip("skip test_4_object_id") @@ -137,9 +121,24 @@ def test_4_object_id(self): expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Messagetype", "ObjectLog", "SystemLog"], "table": "ConfigObjectTable", "where": [[{"suffix": null, "value2": null, "name": "ObjectId", "value": "virtual_network:default-domain:admin:vn1-take2", "op": 1}]]}' expected_result_dict = json.loads(expected_result_str) - for key in expected_result_dict: - self.assertTrue(key in query_dict) - self.assertTrue(expected_result_dict[key] == query_dict[key]) + self.assertEqual(expected_result_dict, query_dict) + + # prefix query + #@unittest.skip("skip test_5_prefix_query") + def test_5_prefix_query(self): + global test_num + global query_dict + test_num = 5 + + argv = sys.argv + sys.argv = "contrail-logs --source node* --message-type UveVirtualNetwork*".split() + self._querier.run() + sys.argv = argv + + expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable", "where": [[{"suffix": null, "value2": null, "name": "Source", "value": "node", "op": 7}, {"suffix": null, "value2": null, "name": "Messagetype", "value": "UveVirtualNetwork", "op": 7}]]}' + expected_result_dict = json.loads(expected_result_str) + self.assertEqual(expected_result_dict, query_dict) + # end test_5_prefix_query if __name__ == '__main__': unittest.main() diff --git a/src/opserver/test/utils/analytics_fixture.py b/src/opserver/test/utils/analytics_fixture.py index 801141f8182..8ee304648f0 100644 --- a/src/opserver/test/utils/analytics_fixture.py +++ b/src/opserver/test/utils/analytics_fixture.py @@ -971,6 +971,26 @@ def verify_message_table_where_and(self): else: return False + @retry(delay=1, tries=6) + def verify_message_table_where_prefix(self): + self.logger.info('verify_message_table_where_prefix') + vns = VerificationOpsSrv('127.0.0.1', self.opserver_port) + prefix_key_value_map = {'Source': socket.gethostname()[:-1], + 'ModuleId': 'contrail-', 'Messagetype': 'Collector', + 'Category': 'Discovery'} + for key, value in prefix_key_value_map.iteritems(): + self.logger.info('verify where_prefix: %s = %s*' % (key, value)) + res = vns.post_query('MessageTable', start_time='-10m', + end_time='now', select_fields=[key], + where_clause='%s = %s*' % (key, value)) + if not len(res): + return False + self.logger.info(str(res)) + for r in res: + assert(r[key].startswith(value)) + return True + # end verify_message_table_where_prefix + @retry(delay=1, tries=6) def verify_message_table_filter(self): self.logger.info("verify_message_table_where_filter") diff --git a/src/query_engine/where_query.cc b/src/query_engine/where_query.cc index 4f66dcf0ce1..c26184204b4 100644 --- a/src/query_engine/where_query.cc +++ b/src/query_engine/where_query.cc @@ -496,12 +496,17 @@ WhereQuery::WhereQuery(const std::string& where_json_string, int direction, db_query->t_only_col = true; // only EQUAL op supported currently - QE_INVALIDARG_ERROR(op == EQUAL); + QE_INVALIDARG_ERROR((op == EQUAL) || (op == PREFIX)); // string encoding #ifdef USE_CASSANDRA_CQL db_query->cr.start_.push_back(value); - db_query->cr.finish_.push_back(value); + if (op == PREFIX) { + value2 = value + "\x7f"; + db_query->cr.finish_.push_back(value2); + } else { + db_query->cr.finish_.push_back(value); + } #else db_query->row_key_suffix.push_back(value); #endif @@ -543,12 +548,17 @@ WhereQuery::WhereQuery(const std::string& where_json_string, int direction, db_query->t_only_col = true; // only EQUAL op supported currently - QE_INVALIDARG_ERROR(op == EQUAL); + QE_INVALIDARG_ERROR((op == EQUAL) || (op == PREFIX)); // string encoding #ifdef USE_CASSANDRA_CQL db_query->cr.start_.push_back(value); - db_query->cr.finish_.push_back(value); + if (op == PREFIX) { + value2 = value + "\x7f"; + db_query->cr.finish_.push_back(value2); + } else { + db_query->cr.finish_.push_back(value); + } #else db_query->row_key_suffix.push_back(value); #endif @@ -569,12 +579,17 @@ WhereQuery::WhereQuery(const std::string& where_json_string, int direction, db_query->t_only_col = true; // only EQUAL op supported currently - QE_INVALIDARG_ERROR(op == EQUAL); + QE_INVALIDARG_ERROR((op == EQUAL) || (op == PREFIX)); // string encoding #ifdef USE_CASSANDRA_CQL db_query->cr.start_.push_back(value); - db_query->cr.finish_.push_back(value); + if (op == PREFIX) { + value2 = value + "\x7f"; + db_query->cr.finish_.push_back(value2); + } else { + db_query->cr.finish_.push_back(value); + } #else db_query->row_key_suffix.push_back(value); #endif @@ -590,12 +605,17 @@ WhereQuery::WhereQuery(const std::string& where_json_string, int direction, db_query->t_only_col = true; // only EQUAL op supported currently - QE_INVALIDARG_ERROR(op == EQUAL); + QE_INVALIDARG_ERROR((op == EQUAL) || (op == PREFIX)); // string encoding #ifdef USE_CASSANDRA_CQL db_query->cr.start_.push_back(value); - db_query->cr.finish_.push_back(value); + if (op == PREFIX) { + value2 = value + "\x7f"; + db_query->cr.finish_.push_back(value2); + } else { + db_query->cr.finish_.push_back(value); + } #else db_query->row_key_suffix.push_back(value); #endif