Fix collector crash due to non ASCII character

Message keyword processing parser can crash if it receives
non ASCII character in certain scenarios. Sanitize the string
before passing it to the message keyword processing parser.
Note that the message write to cassandra will still fail
due to non UTF8 encoded string. Add test case to validate that
contrail-collector does not crash on receiving such a message.

Closes-Bug: #1645885

Conflicts:
	src/analytics/parser_util.cc

Change-Id: Ic3c9ee1e9e3c26fce21014c0f86a22061cca78b0

src/analytics/parser_util.cc

@@ -30,8 +30,8 @@ LineParser::GetAtrributes(const pugi::xml_node &node,
 {
      for (pugi::xml_attribute attr = node.first_attribute(); attr;
              attr = attr.next_attribute()) {
-         std::string s =  boost::algorithm::to_lower_copy(std::string(
-                     attr.value()));
+         std::string s =  MakeSane(boost::algorithm::to_lower_copy(std::string(
+                     attr.value())));
          if (!s.empty()) {
              LineParser::WordListType w = ParseDoc(s.begin(), s.end());
              words->insert(w.begin(), w.end());
@@ -49,8 +49,8 @@ LineParser::Travarse(const pugi::xml_node &node,
         if (check_attr)
             GetAtrributes(node, words);
     } else if (type == pugi::node_pcdata || type == pugi::node_cdata) {
-         std::string s =  boost::algorithm::to_lower_copy(std::string(
-                     node.value()));
+         std::string s =  MakeSane(boost::algorithm::to_lower_copy(std::string(
+                     node.value())));
          if (!s.empty()) {
              LineParser::WordListType w = ParseDoc(s.begin(), s.end());
              words->insert(w.begin(), w.end());
@@ -74,7 +74,7 @@ LineParser::ParseXML(const pugi::xml_node &node, bool check_attr)
 
 LineParser::WordListType
 LineParser::Parse(std::string s) {
-    std::string ls = boost::algorithm::to_lower_copy(s);
+    std::string ls = MakeSane(boost::algorithm::to_lower_copy(s));
     return ParseDoc(ls.begin(), ls.end());
 }
 

src/opserver/test/test_analytics_sys.py

@@ -466,9 +466,31 @@ def test_11_verify_syslog_table_query(self):
         line = 'football ' + chr(201) + chr(203) + chr(70) + ' and baseball'
         syslogger.critical(line)
         assert vizd_obj.verify_keyword_query(line, ['football', 'baseball'])
-
     # end test_11_verify_syslog_table_query
 
+    #@unittest.skip('verify message non ascii')
+    def test_12_verify_message_non_ascii(self):
+        '''
+        This test verifies message sent with non ascii character does not
+        crash vizd.
+        '''
+        logging.info('%%% test_12_verify_message_non_ascii %%%')
+        analytics = self.useFixture(
+            AnalyticsFixture(logging, builddir,
+                             self.__class__.redis_port,
+                             self.__class__.cassandra_port))
+        assert analytics.verify_on_setup()
+        collectors = [analytics.get_collector()]
+        generator_obj = self.useFixture(
+            GeneratorFixture('contrail-vrouter-agent-12', collectors,
+                             logging, None, node_type='Compute'))
+        assert generator_obj.verify_on_setup()
+        generator_obj.send_vrouterinfo(socket.gethostname(),
+            b_info = True, deleted = False, non_ascii=True)
+        # Verify vizd is still running
+        assert analytics.verify_collector_gen(analytics.collectors[0])
+    # end test_12_verify_message_non_ascii
+
     @staticmethod
     def get_free_port():
         cs = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

src/opserver/test/utils/generator_fixture.py

@@ -400,16 +400,20 @@ def send_vn_config_uve(self, name, conn_nw=None, partial_conn_nw=None,
         vn_uve.send(sandesh=self._sandesh_instance)
     # end send_vn_config_uve
 
-    def send_vrouterinfo(self, name, b_info = False, deleted = False):
+    def send_vrouterinfo(self, name, b_info = False, deleted = False,
+                         non_ascii = False):
         vinfo = None
 
         if deleted:
             vinfo = VrouterAgent(name=name,
                                  deleted = True)
         else:
             if b_info:
+                build_info="testinfo"
+                if non_ascii:
+                    build_info += ' ' + chr(201) + chr(203) + chr(213) + ' ' + build_info
                 vinfo = VrouterAgent(name=name,
-                                     build_info="testinfo",
+                                     build_info=build_info,
                                      state="OK")
             else:
                 vinfo = VrouterAgent(name=name, state="OK")