Merge "Migrate LBaaS Custom Attributes Code to svc_monitor"

src/config/svc-monitor/SConscript

@@ -41,6 +41,9 @@ sources = [
     'svc_monitor/services/loadbalancer/drivers/ha_proxy/__init__.py',
     'svc_monitor/services/loadbalancer/drivers/ha_proxy/driver.py',
     'svc_monitor/services/loadbalancer/drivers/ha_proxy/haproxy_config.py',
+    'svc_monitor/services/loadbalancer/drivers/ha_proxy/custom_attributes/custom_attributes.yml',
+    'svc_monitor/services/loadbalancer/drivers/ha_proxy/custom_attributes/__init__.py',
+    'svc_monitor/services/loadbalancer/drivers/ha_proxy/custom_attributes/haproxy_validator.py',
     'svc_monitor/snat_agent.py',
     'svc_monitor/tests/__init__.py',
     'svc_monitor/tests/test_common_utils.py',
@@ -58,6 +61,7 @@ sources = [
     'svc_monitor/tests/fake_lb_driver.py',
     'svc_monitor/tests/scheduler/__init__.py',
     'svc_monitor/tests/scheduler/test_vrouter_schedulers.py',
+    'svc_monitor/tests/test_lbaas_custom_attributes.py',
 ]
 
 sources += env.SandeshGenPy('svc_mon_introspect.sandesh',

src/config/svc-monitor/requirements.txt

@@ -6,3 +6,4 @@ psutil>=0.6.0
 bottle
 kombu
 kazoo
+pyyaml

src/config/svc-monitor/svc_monitor/config_db.py

@@ -126,6 +126,7 @@ def __init__(self, uuid, obj_dict=None):
         self.loadbalancer_listener = None
         self.loadbalancer_id = None
         self.last_sent = None
+        self.custom_attributes = []
         self.update(obj_dict)
     # end __init__
 
@@ -136,6 +137,9 @@ def update(self, obj=None):
         self.fq_name = obj['fq_name']
         self.params = obj.get('loadbalancer_pool_properties', None)
         self.provider = obj.get('loadbalancer_pool_provider', None)
+        kvpairs = obj.get('loadbalancer_pool_custom_attributes', None)
+        if kvpairs:
+            self.custom_attributes = kvpairs.get('key_value_pair', [])
         self.members = set([lm['uuid']
                             for lm in obj.get('loadbalancer_members', [])])
         self.id_perms = obj.get('id_perms', None)

src/config/svc-monitor/svc_monitor/services/loadbalancer/drivers/ha_proxy/custom_attributes/custom_attributes.yml

@@ -0,0 +1,60 @@
+# HAProxy custom attributes
+global:
+    max_conn:
+      type: int
+      limits: [1, 65535]
+      cmd: maxconn %d
+    max_conn_rate:
+      type: int
+      limits: [1, 65535]
+      cmd: maxconnrate %d
+    max_sess_rate:
+      type: int
+      limits: [1, 65535]
+      cmd: maxsessrate %d
+    max_ssl_conn:
+      type: int
+      limits: [1, 65535]
+      cmd: maxsslconn %d
+    max_ssl_rate:
+      type: int
+      limits: [1, 65535]
+      cmd: maxsslrate %d
+    ssl_ciphers:
+      type: str
+      limits: [1, 100]
+      cmd: ssl-default-bind-ciphers %s
+    tune_http_max_header:
+      type: int
+      limits: [1, 128]
+      cmd: tune.http.maxhdr %d
+    tune_ssl_max_record:
+      type: int
+      limits: [1, 16384]
+      cmd: tune.ssl.maxrecord %d
+default:
+    server_timeout:
+      type: int
+      limits: [1, 5000000]
+      cmd: timeout server %d
+    client_timeout:
+      type: int
+      limits: [1, 5000000]
+      cmd: timeout client %d
+    connect_timeout:
+      type: int
+      limits: [1, 5000000]
+      cmd: timeout connect %d
+frontend:
+    http_server_close:
+      type: bool
+      limits: [True, False]
+      cmd: "%soption http-server-close"
+    rate_limit_sessions:
+      type: int
+      limits: [1, 65535]
+      cmd: rate-limit sessions %d
+    tls_container:
+      type: CustomAttrTlsContainer
+      limits: ""
+      cmd: ""

src/config/svc-monitor/svc_monitor/services/loadbalancer/drivers/ha_proxy/custom_attributes/haproxy_validator.py

@@ -0,0 +1,74 @@
+import logging
+import inspect
+import os
+
+class CustomAttr(object):
+    """This type handles non-flat data-types like
+       int, str, bool.
+    """
+    def __init__(self, key, value):
+        self._value = value
+        self._key =  key
+
+    def validate(self):
+        pass
+
+    def post_validation(self):
+        pass
+
+class CustomAttrTlsContainer(CustomAttr):
+    def __init__(self, key, value):
+        super(CustomAttrTlsContainer, self).__init__(key, value)
+
+    def validate(self):
+        return True
+
+    def post_validation(self):
+        return self._value
+
+def validate_custom_attributes(custom_attributes_dict, section,
+                               custom_attributes):
+    section_dict = {}
+    if custom_attributes and section in custom_attributes_dict:
+        for key, value in custom_attributes.iteritems():
+            if key in custom_attributes_dict[section]:
+                #Sanitize the value
+                try:
+                    type_attr = custom_attributes_dict[section][key]['type']
+                    limits = custom_attributes_dict[section][key]['limits']
+                    if type_attr == 'int':
+                        value = int(value)
+                        if value in range(limits[0], limits[1]):
+                            section_dict.update({key:value})
+                        else:
+                            logging.info("Skipping key: %s, value: %s due to" \
+                               "validation failure" % (key, value))
+                    elif type_attr == 'str':
+                        if len(value) in range(limits[0], limits[1]):
+                            section_dict.update({key:value})
+                        else:
+                            logging.info("Skipping key: %s, value: %s due to" \
+                               "validation failure" % (key, value))
+                    elif type_attr == 'bool':
+                        if value in limits:
+                            if value == 'True':
+                                value = ''
+                            elif value == 'False':
+                                value = 'no '
+                            section_dict.update({key:value})
+                        else:
+                            logging.info("Skipping key: %s, value: %s due to" \
+                               "validation failure" % (key, value))
+                    elif inspect.isclass(eval(type_attr)):
+                        new_custom_attr = eval(type_attr)(key, value)
+                        if new_custom_attr.validate():
+                            value = new_custom_attr.post_validation()
+                            section_dict.update({key:value})
+                        else:
+                            logging.info("Skipping key: %s, value: %s due to" \
+                               "validation failure" % (key, value))
+                except Exception as e:
+                    logging.error(str(e))
+                    continue
+
+    return section_dict

src/config/svc-monitor/svc_monitor/services/loadbalancer/drivers/ha_proxy/haproxy_config.py

@@ -1,4 +1,15 @@
 from svc_monitor.config_db import *
+from os.path import dirname, exists, join
+import logging
+import yaml
+
+try:
+    from custom_attributes.haproxy_validator \
+        import validate_custom_attributes as get_valid_attrs
+except ImportError:
+    custom_attr_dict = {}
+    def get_valid_attrs(custom_attr_dict, section, custom_attrs):
+        return {}
 
 PROTO_HTTP = 'HTTP'
 PROTO_HTTPS = 'HTTPS'
@@ -29,25 +40,54 @@
 def get_config_v2(lb):
     sock_path = '/var/lib/contrail/loadbalancer/haproxy/'
     sock_path += lb.uuid + '/haproxy.sock'
-    conf = set_globals(sock_path) + '\n\n'
-    conf += set_defaults() + '\n\n'
-    conf += set_v2_frontend_backend(lb)
+    custom_attr_dict = {}
+    custom_attrs = {}
+    conf = set_globals(sock_path, custom_attr_dict, custom_attrs) + '\n\n'
+    conf += set_defaults(custom_attr_dict, custom_attrs) + '\n\n'
+    conf += set_v2_frontend_backend(lb, custom_attr_dict, custom_attrs)
     return conf
 
 def get_config_v1(pool):
     sock_path = '/var/lib/contrail/loadbalancer/haproxy/'
     sock_path += pool.uuid + '/haproxy.sock'
-    conf = set_globals(sock_path) + '\n\n'
-    conf += set_defaults() + '\n\n'
-    conf += set_v1_frontend_backend(pool)
+    custom_attr_dict = get_custom_attributes_dict()
+    custom_attrs = get_custom_attributes(pool)
+    conf = set_globals(sock_path, custom_attr_dict, custom_attrs) + '\n\n'
+    conf += set_defaults(custom_attr_dict, custom_attrs) + '\n\n'
+    conf += set_v1_frontend_backend(pool, custom_attr_dict, custom_attrs)
     return conf
 
-def set_globals(sock_path):
-    maxconn = 65000
-    ssl_ciphers = \
-        'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:' \
-        'ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:' \
-        'RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS'
+
+def get_custom_attributes_dict():
+    custom_attr_dict = {}
+    script_dir = dirname(__file__)
+    rel_path = "custom_attributes/custom_attributes.yml"
+    abs_file_path = join(script_dir, rel_path)
+    if exists(abs_file_path):
+        with open(abs_file_path, 'r') as f:
+            custom_attr_dict = yaml.safe_load(f)
+    return custom_attr_dict
+
+def get_custom_attributes(pool):
+    custom_attrs = {}
+    for kvp in pool.custom_attributes or []:
+        custom_attrs[kvp['key']] = kvp['value']
+    return custom_attrs
+
+def set_globals(sock_path, custom_attr_dict, custom_attrs):
+    global_custom_attrs = get_valid_attrs(custom_attr_dict, 'global', custom_attrs)
+    if 'max_conn' in global_custom_attrs:
+        maxconn = global_custom_attrs.pop('max_conn', None)
+    else:
+        maxconn = 65000
+
+    if 'ssl_ciphers' in global_custom_attrs:
+        ssl_ciphers = global_custom_attrs.pop('ssl_ciphers', None)
+    else:
+        ssl_ciphers = \
+            'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:' \
+            'ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:' \
+            'RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS'
 
     conf = [
         'global',
@@ -62,13 +102,31 @@ def set_globals(sock_path):
         'maxconn %d' % maxconn
     ]
     conf.append('stats socket %s mode 0666 level user' % sock_path)
+
+    # Adding custom_attributes config
+    for key, value in global_custom_attrs.iteritems():
+        cmd = custom_attr_dict['global'][key]['cmd']
+        conf.append(cmd % value)
+
     res = "\n\t".join(conf)
     return res
 
-def set_defaults():
-    client_timeout = 300000
-    server_timeout = 300000
-    connect_timeout = 5000
+def set_defaults(custom_attr_dict, custom_attrs):
+    default_custom_attrs = get_valid_attrs(custom_attr_dict, 'default', custom_attrs)
+    if 'client_timeout' in default_custom_attrs:
+        client_timeout = default_custom_attrs.pop('client_timeout', None)
+    else:
+        client_timeout = 300000
+
+    if 'server_timeout' in default_custom_attrs:
+        server_timeout = default_custom_attrs.pop('server_timeout', None)
+    else:
+        server_timeout = 300000
+
+    if 'connect_timeout' in default_custom_attrs:
+        connect_timeout = default_custom_attrs.pop('connect_timeout', None)
+    else:
+        connect_timeout = 5000
 
     conf = [
         'defaults',
@@ -80,10 +138,15 @@ def set_defaults():
         'timeout server %d' % server_timeout,
     ]
 
+    # Adding custom_attributes config
+    for key, value in default_custom_attrs.iteritems():
+        cmd = custom_attr_dict['default'][key]['cmd']
+        conf.append(cmd % value)
+
     res = "\n\t".join(conf)
     return res
 
-def set_v1_frontend_backend(pool):
+def set_v1_frontend_backend(pool, custom_attr_dict, custom_attrs):
     conf = []
     vip = VirtualIpSM.get(pool.virtual_ip)
     if not vip and not vip.params['admin_state']:
@@ -104,15 +167,20 @@ def set_v1_frontend_backend(pool):
             vip.params['protocol'] == PROTO_HTTPS:
         lconf.append('option forwardfor')
 
+    frontend_custom_attrs = get_valid_attrs(custom_attr_dict, 'frontend', custom_attrs)
     if pool and pool.params['admin_state']:
         lconf.append('default_backend %s' % pool.uuid)
+        # Adding custom_attributes config
+        for key, value in frontend_custom_attrs.iteritems():
+            cmd = custom_attr_dict['frontend'][key]['cmd']
+            lconf.append(cmd % value)
         res = "\n\t".join(lconf) + '\n\n'
-        res += set_backend(pool)
+        res += set_backend(pool, custom_attr_dict, custom_attrs)
         conf.append(res)
 
     return "\n".join(conf)
 
-def set_v2_frontend_backend(lb):
+def set_v2_frontend_backend(lb, custom_attr_dict, custom_attrs):
     conf = []
     for ll_id in lb.loadbalancer_listeners:
         ll = LoadbalancerListenerSM.get(ll_id)
@@ -150,12 +218,13 @@ def set_v2_frontend_backend(lb):
         if pool and pool.params['admin_state']:
             lconf.append('default_backend %s' % pool.uuid)
             res = "\n\t".join(lconf) + '\n\n'
-            res += set_backend(pool)
+            res += set_backend(pool, custom_attr_dict, custom_attrs)
             conf.append(res)
 
     return "\n".join(conf)
 
-def set_backend(pool):
+def set_backend(pool, custom_attr_dict, custom_attrs):
+    backend_custom_attrs = get_valid_attrs(custom_attr_dict, 'backend', custom_attrs)
     conf = [
         'backend %s' % pool.uuid,
         'mode %s' % PROTO_MAP[pool.params['protocol']],
@@ -184,6 +253,11 @@ def set_backend(pool):
                   member.params['weight'])) + server_suffix
         conf.append(server)
 
+    # Adding custom_attributes config
+    for key, value in backend_custom_attrs.iteritems():
+        cmd = custom_attr_dict['backend'][key]['cmd']
+        conf.append(cmd % value)
+
     return "\n\t".join(conf) + '\n'
 
 def set_health_monitor(hm):
@@ -197,7 +271,7 @@ def set_health_monitor(hm):
     ]
 
     if hm.params['monitor_type'] in (HEALTH_MONITOR_HTTP, HEALTH_MONITOR_HTTPS):
-        conf.append('option httpchk %s %s' % 
+        conf.append('option httpchk %s %s' %
             (hm.params['http_method'], hm.params['url_path']))
         conf.append(
             'http-check expect rstatus %s' %