Merge "Support for prefix match for all index fields in the system/ob…

…ject log queries"

src/opserver/log.py

@@ -214,9 +214,14 @@ def query(self):
         and_filter = []
         or_filter = []
         if self._args.source is not None:
+            if self._args.source.endswith('*'):
+                val = self._args.source[:-1]
+                oper = OpServerUtils.MatchOp.PREFIX
+            else:
+                val = self._args.source
+                oper = OpServerUtils.MatchOp.EQUAL
             source_match = OpServerUtils.Match(name=VizConstants.SOURCE,
-                                               value=self._args.source,
-                                               op=OpServerUtils.MatchOp.EQUAL)
+                                               value=val, op=oper)
             where_msg.append(source_match.__dict__)
 
         if self._args.module is not None:
@@ -226,17 +231,27 @@ def query(self):
             where_msg.append(module_match.__dict__)
 
         if self._args.category is not None:
+            if self._args.category.endswith('*'):
+                val = self._args.category[:-1]
+                oper = OpServerUtils.MatchOp.PREFIX
+            else:
+                val = self._args.category
+                oper = OpServerUtils.MatchOp.EQUAL
             category_match = OpServerUtils.Match(
                 name=VizConstants.CATEGORY,
-                value=self._args.category,
-                op=OpServerUtils.MatchOp.EQUAL)
+                value=val, op=oper)
             where_msg.append(category_match.__dict__)
 
         if self._args.message_type is not None:
+            if self._args.message_type.endswith('*'):
+                val = self._args.message_type[:-1]
+                oper = OpServerUtils.MatchOp.PREFIX
+            else:
+                val = self._args.message_type
+                oper = OpServerUtils.MatchOp.EQUAL
             message_type_match = OpServerUtils.Match(
                 name=VizConstants.MESSAGE_TYPE,
-                value=self._args.message_type,
-                op=OpServerUtils.MatchOp.EQUAL)
+                value=val, op=oper)
             where_msg.append(message_type_match.__dict__)
 
         if self._args.level is not None:

src/opserver/test/test_analytics_sys.py

@@ -141,6 +141,7 @@ def test_02_message_table_query(self):
         assert vizd_obj.verify_message_table_messagetype()
         assert vizd_obj.verify_message_table_where_or()
         assert vizd_obj.verify_message_table_where_and()
+        assert vizd_obj.verify_message_table_where_prefix()
         assert vizd_obj.verify_message_table_filter()
         assert vizd_obj.verify_message_table_filter2()
         assert vizd_obj.verify_message_table_sort()

src/opserver/test/test_log.py

@@ -12,18 +12,23 @@
 
 test_num = 0
 query_dict = {}
-result_1 = [
+query_result = {
+1: [
 {u'Category': None, u'NodeType': u'Config', u'Level': 2147483647, u'InstanceId': u'0', u'Messagetype': u'UveVirtualNetworkConfigTrace', u'Source': u'a6s45', u'SequenceNum': 6867683, u'MessageTS': 1442429588898861, u'Xmlmessage': u'<UveVirtualNetworkConfigTrace type="sandesh"><data type="struct" identifier="1"><UveVirtualNetworkConfig><name type="string" identifier="1" key="ObjectVNTable">default-domain:demo:svc-vn-left</name><connected_networks type="list" identifier="4" aggtype="union"><list type="string" size="0" /></connected_networks><partially_connected_networks type="list" identifier="5" aggtype="union"><list type="string" size="0" /></partially_connected_networks><routing_instance_list type="list" identifier="6" aggtype="union"><list type="string" size="1"><element>svc-vn-left</element></list></routing_instance_list><total_acl_rules type="i32" identifier="7">0</total_acl_rules></UveVirtualNetworkConfig></data></UveVirtualNetworkConfigTrace>', u'Type': 6, u'ModuleId': u'contrail-schema'}
-    ]
-result_2 = [
+],
+2: [
 {u'Category': None, u'NodeType': u'Analytics', u'Level': 2147483647, u'InstanceId': u'0', u'Messagetype': u'GeneratorDbStatsUve', u'Source': u'a6s45', u'SequenceNum': 56411, u'MessageTS': 1442429889555171, u'Xmlmessage': u'<GeneratorDbStatsUve type="sandesh"><data type="struct" identifier="1"><GeneratorDbStats><name type="string" identifier="1" key="ObjectGeneratorInfo">a6s10:Compute:contrail-vrouter-agent:0</name><table_info type="list" identifier="3" tags=".table_name"><list type="struct" size="12"><DbTableInfo><table_name type="string" identifier="1">MessageTable</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">MessageTableCategory</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">MessageTableKeyword</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">0</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">MessageTableMessageType</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">MessageTableModuleId</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">MessageTableSource</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">MessageTableTimestamp</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">ObjectTable</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">ObjectValueTable</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">12</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">StatsTableByDblTagV3</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">1</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">StatsTableByStrTagV3</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">102</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">StatsTableByU64TagV3</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">2</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo></list></table_info><errors type="list" identifier="4" tags=""><list type="struct" size="1"><DbErrors><write_tablespace_fails type="u64" identifier="1">0</write_tablespace_fails><read_tablespace_fails type="u64" identifier="2">0</read_tablespace_fails><write_table_fails type="u64" identifier="3">0</write_table_fails><read_table_fails type="u64" identifier="4">0</read_table_fails><write_column_fails type="u64" identifier="5">0</write_column_fails><write_batch_column_fails type="u64" identifier="6">0</write_batch_column_fails><read_column_fails type="u64" identifier="7">0</read_column_fails></DbErrors></list></errors><statistics_table_info type="list" identifier="5" tags=".table_name"><list type="struct" size="3"><DbTableInfo><table_name type="string" identifier="1">ComputeCpuState:cpu_info</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">5</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">FieldNames:fields</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">96</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo><DbTableInfo><table_name type="string" identifier="1">VrouterStatsAgent:flow_rate</table_name><reads type="u64" identifier="2">0</reads><read_fails type="u64" identifier="3">0</read_fails><writes type="u64" identifier="4">4</writes><write_fails type="u64" identifier="5">0</write_fails></DbTableInfo></list></statistics_table_info></GeneratorDbStats></data></GeneratorDbStatsUve>', u'Type': 6, u'ModuleId': u'contrail-collector'}
-]
-result_3 = [
+],
+3: [
 {u'ObjectId': u'virtual_network:default-domain:admin:vn1-take2'}
-]
-result_4 = [
+],
+4: [
 {u'ObjectLog': u'<VncApiConfigLog type="sandesh"><api_log type="struct" identifier="1"><VncApiCommon><object_type type="string" identifier="2" key="ConfigObjectTable">virtual_network</object_type><identifier_name type="string" identifier="3" key="ConfigObjectTable">default-domain:admin:vn1-take2</identifier_name><url type="string" identifier="4">http://127.0.0.1:9100/virtual-networks</url><operation type="string" identifier="5">post</operation><useragent type="string" identifier="6">a6s45:/usr/bin/contrail-api</useragent><remote_ip type="string" identifier="7">127.0.0.1:9100</remote_ip><body type="string" identifier="9">{&apos;virtual-network&apos;: {&apos;fq_name&apos;: [&apos;default-domain&apos;, &apos;admin&apos;, &apos;vn1-take2&apos;], &apos;uuid&apos;: None, &apos;network_policy_refs&apos;: [], &apos;router_external&apos;: False, &apos;parent_type&apos;: &apos;project&apos;, &apos;id_perms&apos;: {u&apos;enable&apos;: True, u&apos;uuid&apos;: None, u&apos;creator&apos;: None, u&apos;created&apos;: 0, u&apos;user_visible&apos;: True, u&apos;last_modified&apos;: 0, u&apos;permissions&apos;: {u&apos;owner&apos;: u&apos;cloud-admin&apos;, u&apos;owner_access&apos;: 7, u&apos;other_access&apos;: 7, u&apos;group&apos;: u&apos;cloud-admin-group&apos;, u&apos;group_access&apos;: 7}, u&apos;description&apos;: None}, &apos;display_name&apos;: &apos;vn1-take2&apos;, &apos;is_shared&apos;: False}}</body><domain type="string" identifier="10" key="ConfigObjectTableByUser">default-domain</domain></VncApiCommon></api_log></VncApiConfigLog>', u'Messagetype': u'VncApiConfigLog', u'Source': u'a6s45', u'MessageTS': 1442434711187905, u'SystemLog': None, u'ModuleId': u'contrail-api'}
+],
+5: [
+{u'Category': None, u'NodeType': u'Config', u'Level': 2147483647, u'InstanceId': u'0', u'Messagetype': u'UveVirtualNetworkConfigTrace', u'Source': u'nodec39', u'SequenceNum': 6867683, u'MessageTS': 1442429588898861, u'Xmlmessage': u'<UveVirtualNetworkConfigTrace type="sandesh"><data type="struct" identifier="1"><UveVirtualNetworkConfig><name type="string" identifier="1" key="ObjectVNTable">default-domain:demo:svc-vn-left</name><connected_networks type="list" identifier="4" aggtype="union"><list type="string" size="0" /></connected_networks><partially_connected_networks type="list" identifier="5" aggtype="union"><list type="string" size="0" /></partially_connected_networks><routing_instance_list type="list" identifier="6" aggtype="union"><list type="string" size="1"><element>svc-vn-left</element></list></routing_instance_list><total_acl_rules type="i32" identifier="7">0</total_acl_rules></UveVirtualNetworkConfig></data></UveVirtualNetworkConfigTrace>', u'Type': 6, u'ModuleId': u'contrail-schema'}
 ]
+}
 
 class LogQuerierTest(unittest.TestCase):
 
@@ -35,34 +40,19 @@ def custom_post_url_http(url, params):
 
     @staticmethod
     def custom_get_query_result(opserver_ip, opserver_port, qid):
-        if (test_num == 1):
-            return result_1
-        elif (test_num == 2):
-            return result_2
-        elif (test_num == 3):
-            return result_3
-        elif (test_num == 4):
-            return result_4
-        else:
+        try:
+            return query_result[test_num]
+        except KeyError:
             return []
 
     def custom_display(self, result):
-        if (test_num == 1):
-            self.assertTrue(result == result_1)
-            return
-        elif (test_num == 2):
-            self.assertTrue(result == result_2)
-            return
-        elif (test_num == 3):
-            self.assertTrue(result == result_3)
-            return
-        elif (test_num == 4):
-            self.assertTrue(result == result_4)
-            return
-        else:
+        try:
+            self.assertTrue(result == query_result[test_num])
+        except KeyError:
             self.assertTrue(False)
 
     def setUp(self):
+        self.maxDiff = None
         self._querier = LogQuerier()
 
         flexmock(OpServerUtils).should_receive('post_url_http').replace_with(lambda x, y: self.custom_post_url_http(x, y))
@@ -83,9 +73,7 @@ def test_1_no_arg(self):
 
         expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable"}'
         expected_result_dict = json.loads(expected_result_str)
-        for key in expected_result_dict:
-            self.assertTrue(key in query_dict)
-            self.assertTrue(expected_result_dict[key] == query_dict[key])
+        self.assertEqual(expected_result_dict, query_dict)
 
     # a few args
     #@unittest.skip("skip test_2_message_query")
@@ -99,11 +87,9 @@ def test_2_message_query(self):
         self._querier.run()
         sys.argv = argv
 
-        expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable"}'
+        expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable", "where": [[{"suffix": null, "value2": null, "name": "Source", "value": "a6s45", "op": 1}, {"suffix": null, "value2": null, "name": "ModuleId", "value": "contrail-collector", "op": 1}, {"suffix": null, "value2": null, "name": "Messagetype", "value": "GeneratorDbStatsUve", "op": 1}]], "filter": [[{"suffix": null, "value2": null, "name": "NodeType", "value": "Analytics", "op": 1}, {"suffix": null, "value2": null, "name": "InstanceId", "value": 0, "op": 1}]]}'
         expected_result_dict = json.loads(expected_result_str)
-        for key in expected_result_dict:
-            self.assertTrue(key in query_dict)
-            self.assertTrue(expected_result_dict[key] == query_dict[key])
+        self.assertEqual(expected_result_dict, query_dict)
 
     # a object values query
     #@unittest.skip("skip test_3_object_value")
@@ -119,9 +105,7 @@ def test_3_object_value(self):
 
         expected_result_str = '{"table": "ConfigObjectTable", "start_time": "now-10m", "end_time": "now", "select_fields": ["ObjectId"]}'
         expected_result_dict = json.loads(expected_result_str)
-        for key in expected_result_dict:
-            self.assertTrue(key in query_dict)
-            self.assertTrue(expected_result_dict[key] == query_dict[key])
+        self.assertEqual(expected_result_dict, query_dict)
 
     # a object id query
     #@unittest.skip("skip test_4_object_id")
@@ -137,9 +121,24 @@ def test_4_object_id(self):
 
         expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Messagetype", "ObjectLog", "SystemLog"], "table": "ConfigObjectTable", "where": [[{"suffix": null, "value2": null, "name": "ObjectId", "value": "virtual_network:default-domain:admin:vn1-take2", "op": 1}]]}'
         expected_result_dict = json.loads(expected_result_str)
-        for key in expected_result_dict:
-            self.assertTrue(key in query_dict)
-            self.assertTrue(expected_result_dict[key] == query_dict[key])
+        self.assertEqual(expected_result_dict, query_dict)
+
+    # prefix query
+    #@unittest.skip("skip test_5_prefix_query")
+    def test_5_prefix_query(self):
+        global test_num
+        global query_dict
+        test_num = 5
+
+        argv = sys.argv
+        sys.argv = "contrail-logs --source node* --message-type UveVirtualNetwork*".split()
+        self._querier.run()
+        sys.argv = argv
+
+        expected_result_str = '{"sort": 1, "start_time": "now-10m", "sort_fields": ["MessageTS"], "end_time": "now", "select_fields": ["MessageTS", "Source", "ModuleId", "Category", "Messagetype", "SequenceNum", "Xmlmessage", "Type", "Level", "NodeType", "InstanceId"], "table": "MessageTable", "where": [[{"suffix": null, "value2": null, "name": "Source", "value": "node", "op": 7}, {"suffix": null, "value2": null, "name": "Messagetype", "value": "UveVirtualNetwork", "op": 7}]]}'
+        expected_result_dict = json.loads(expected_result_str)
+        self.assertEqual(expected_result_dict, query_dict)
+    # end test_5_prefix_query
 
 if __name__ == '__main__':
     unittest.main()

src/opserver/test/utils/analytics_fixture.py

@@ -971,6 +971,26 @@ def verify_message_table_where_and(self):
             else:
                 return False
 
+    @retry(delay=1, tries=6)
+    def verify_message_table_where_prefix(self):
+        self.logger.info('verify_message_table_where_prefix')
+        vns = VerificationOpsSrv('127.0.0.1', self.opserver_port)
+        prefix_key_value_map = {'Source': socket.gethostname()[:-1],
+            'ModuleId': 'contrail-', 'Messagetype': 'Collector',
+            'Category': 'Discovery'}
+        for key, value in prefix_key_value_map.iteritems():
+            self.logger.info('verify where_prefix: %s = %s*' % (key, value))
+            res = vns.post_query('MessageTable', start_time='-10m',
+                    end_time='now', select_fields=[key],
+                    where_clause='%s = %s*' % (key, value))
+            if not len(res):
+                return False
+            self.logger.info(str(res))
+            for r in res:
+                assert(r[key].startswith(value))
+        return True
+    # end verify_message_table_where_prefix
+
     @retry(delay=1, tries=6)
     def verify_message_table_filter(self):
         self.logger.info("verify_message_table_where_filter")