Merge "Add support for names with non-ascii (utf-8) characters." into…

… R1.10

src/config/api-server/vnc_cfg_api_server.py

@@ -15,6 +15,8 @@
 gevent.pywsgi.MAX_REQUEST_LINE = 65535
 
 import sys
+reload(sys)
+sys.setdefaultencoding('UTF8')
 import re
 import logging
 import signal

src/config/api-server/vnc_cfg_ifmap.py

@@ -27,7 +27,7 @@
 from netaddr import IPNetwork
 from bitarray import bitarray
 
-from cfgm_common import ignore_exceptions
+from cfgm_common import ignore_exceptions, utils
 from cfgm_common.ifmap.client import client, namespaces
 from cfgm_common.ifmap.request import NewSessionRequest, RenewSessionRequest,\
     EndSessionRequest, PublishRequest, SearchRequest, SubscribeRequest,\
@@ -412,22 +412,22 @@ def _publish_update(self, self_imid, update):
         def _build_request_id_self(imid, metalist):
             request = ''
             for m in metalist:
-                request += str(PublishUpdateOperation(
-                        id1=str(Identity(name=self_imid, type="other",
+                request += unicode(PublishUpdateOperation(
+                        id1=unicode(Identity(name=self_imid, type="other",
                                          other_type="extended")),
-                        metadata=str(m),
+                        metadata=unicode(m),
                         lifetime='forever'))
             return request
 
         def _build_request_id_pair(id1, id2, metalist):
             request = ''
             for m in metalist:
-                request += str(PublishUpdateOperation(
-                    id1=str(Identity(name=id1, type="other",
+                request += unicode(PublishUpdateOperation(
+                    id1=unicode(Identity(name=id1, type="other",
                                      other_type="extended")),
-                    id2=str(Identity(name=id2, type="other",
+                    id2=unicode(Identity(name=id2, type="other",
                                      other_type="extended")),
-                    metadata=str(m),
+                    metadata=unicode(m),
                     lifetime='forever'))
             return request
 
@@ -726,12 +726,16 @@ def _cassandra_ensure_keyspace(self, server_list,
                 if comparator_type:
                     sys_mgr.create_column_family(
                         keyspace_name, cf_name,
-                        comparator_type=comparator_type)
+                        comparator_type=comparator_type,
+                        default_validation_class='UTF8Type')
                 else:
-                    sys_mgr.create_column_family(keyspace_name, cf_name)
+                    sys_mgr.create_column_family(keyspace_name, cf_name,
+                                                 default_validation_class='UTF8Type')
             except pycassa.cassandra.ttypes.InvalidRequestException as e:
                 # TODO verify only EEXISTS
                 print "Warning! " + str(e)
+                sys_mgr.alter_column_family(keyspace_name, cf_name,
+                                            default_validation_class='UTF8Type')
     # end _cassandra_ensure_keyspace
 
     def _create_prop(self, bch, obj_uuid, prop_name, prop_val):
@@ -958,8 +962,8 @@ def uuid_to_obj_type(self, id):
     def fq_name_to_uuid(self, obj_type, fq_name):
         method_name = obj_type.replace('-', '_')
         fq_name_str = ':'.join(fq_name)
-        col_start = '%s:' % (fq_name_str)
-        col_fin = '%s;' % (fq_name_str)
+        col_start = '%s:' % (utils.encode_string(fq_name_str))
+        col_fin = '%s;' % (utils.encode_string(fq_name_str))
         try:
             col_info_iter = self._obj_fq_name_cf.xget(
                 method_name, column_start=col_start, column_finish=col_fin)

src/config/common/SConscript

@@ -22,6 +22,7 @@ local_sources = [
                   '__init__.py',
                   'zkclient.py',
                   'exceptions.py',
+                  'utils.py',
                   'imid.py',
                   'svc_info.py',
                   'vnc_cpu_info.py',

src/config/common/tests/test_utils.py

@@ -417,6 +417,7 @@ def call(method, body):
         if method == 'publish':
             pub_env = cls._PUBLISH_ENVELOPE % {
                 'body': body._PublishRequest__operations}
+            pub_env = pub_env.encode('utf-8')
             env_root = etree.fromstring(pub_env)
             poll_result = etree.Element('pollResult')
             for pub_root in env_root[0]:

src/config/common/utils.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2015 Juniper Networks
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# @author: Numan Siddique, eNovance.
+
+
+import urllib
+
+def encode_string(enc_str, encoding='utf-8'):
+    """Encode the string using urllib.quote_plus
+
+    Eg. @input:
+            enc_str = 'neté
+            type - 'unicode' or 'str'
+        @retval
+            enc_str = 'net%C3%A9%C3%B9'
+            type - str
+    """
+    try:
+        enc_str.encode()
+    except (UnicodeDecodeError, UnicodeEncodeError):
+        if type(enc_str) is unicode:
+            enc_str = enc_str.encode(encoding)
+        enc_str = urllib.quote_plus(enc_str)
+    except Exception:
+        pass
+    return enc_str
+
+
+def decode_string(dec_str, encoding='utf-8'):
+    """Decode the string previously encoded using urllib.quote_plus.
+
+    Eg. If dec_str = 'net%C3%A9%C3%B9'
+           type - 'unicode' or 'str'
+        @retval
+            ret_dec_str = 'neté
+            type - unicode
+    """
+    ret_dec_str = dec_str
+    try:
+        if type(ret_dec_str) is unicode:
+            ret_dec_str = str(ret_dec_str)
+        ret_dec_str = urllib.unquote_plus(ret_dec_str)
+        return ret_dec_str.decode(encoding)
+    except Exception:
+        return dec_str
+

src/config/schema-transformer/to_bgp.py

@@ -14,6 +14,8 @@
 from gevent import monkey
 monkey.patch_all()
 import sys
+reload(sys)
+sys.setdefaultencoding('UTF8')
 import requests
 import ConfigParser
 import cgitb