From 245f6d85c7d28aedb5bb28f4b09b8c2a766b5c56 Mon Sep 17 00:00:00 2001 From: Rudra Rugge Date: Thu, 14 May 2015 13:41:45 -0700 Subject: [PATCH] LBAAS haproxy process manager Manage haproxy daemon for lbaas. Two options avaialable: - Manage through supervisor. This will run on non-daemon mode as the process cannot be managed by supervisord if it runs in background. Process monitoring provided by supervisor. - Start/stop the daemon as we do today. Need additional changes to ensure monitoring/restarting of the process. Additional commit needed to enable this code from vrouter_netns. Change-Id: I05c13d7c96c86bee2fcddc73342ba28c6010c8e6 Partial-Bug: #1452928 Enable haproxy config translation Enable haproxy config translation from json format Also enable haproxy daemon handling by supervisord Change-Id: If3489ea66430ec0ac50bb6198093a0689fa16219 Closes-Bug: #1452928 Conflicts: src/nodemgr/haproxy_stats.py Generate mac from instance ip for service VMs Generate the same mac-address for all interfaces sharing the same IP. In addition a change to daemonize the haproxy process instead of managing through supervisor. Change-Id: I2394f29c4a11bffeee4b0184ce6cd6867b01e0e9 Closes-Bug: #1461882 Haproxy config generation fixes for HTTPS protocol Change-Id: I140361ad4785be2a87d23a04181e73ca999e8e2b Closes-bug: #1466318 Fix for poodle vulnerability; ChangeId: I9432d035eb59b1ff53cb5d33350cd5f8063e077c; Closes-Bug: #1475392 Change-Id: I390a77261bc0d3257108c06951c79f1d2c3dadaa Fix for FREAK SSL vulnerability This fix pushes selected set of secure ciphers into haproxy config file Change-Id: Idfc11ce0411024e7154d3b2c46a095fb4f80337d Closes-Bug: #1477400 HAProxy Performance Tuning HAProxy's default config is non-performant. This fix updates following config in HAProxy: 1) Increase TCP client/server timeouts. 2) Increase ulimit globally per HAProxy process. 3) Increase maxconn globally per HAProxy process. Change-Id: I28be29d5ab3dcb2a35fcbe9168300edf18b2c23c Closes-Bug: #1477781 Allow custom configs with LBaaS This fix takes care of haproxy parsing and validation changes on vrouter agent. Removing extra white spaces Closes-Bug: #1475393 Change-Id: I822e27792f78168a178d555db5703fa1e73d0cc9 --- src/nodemgr/haproxy_stats.py | 2 +- .../opencontrail-vrouter-netns/SConscript | 2 + .../haproxy_config.py | 89 +++++++++++-- .../haproxy_process.py | 117 ++++++++++++++++++ .../haproxy_validator.py | 105 ++++++++++++++++ .../vrouter_netns.py | 48 ++----- 6 files changed, 308 insertions(+), 55 deletions(-) create mode 100644 src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_process.py create mode 100644 src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_validator.py diff --git a/src/nodemgr/haproxy_stats.py b/src/nodemgr/haproxy_stats.py index 745f791f3c7..0a1982fc96e 100644 --- a/src/nodemgr/haproxy_stats.py +++ b/src/nodemgr/haproxy_stats.py @@ -33,7 +33,7 @@ def __init__(self): pass def get_stats(self, pool_id): - sock_path = os.path.join(LB_BASE_DIR, pool_id, 'etc/haproxy/haproxy.cfg.sock') + sock_path = os.path.join(LB_BASE_DIR, pool_id, 'haproxy.sock') if not os.path.exists(sock_path): sys.stderr.write('\nStats socket not found for pool ' + pool_id) return {} diff --git a/src/vnsw/opencontrail-vrouter-netns/SConscript b/src/vnsw/opencontrail-vrouter-netns/SConscript index 615f1378f56..97349ab1e52 100644 --- a/src/vnsw/opencontrail-vrouter-netns/SConscript +++ b/src/vnsw/opencontrail-vrouter-netns/SConscript @@ -17,6 +17,8 @@ sources = [ 'opencontrail_vrouter_netns/__init__.py', 'opencontrail_vrouter_netns/vrouter_netns.py', 'opencontrail_vrouter_netns/haproxy_config.py', + 'opencontrail_vrouter_netns/haproxy_process.py', + 'opencontrail_vrouter_netns/haproxy_validator.py', 'opencontrail_vrouter_netns/vrouter_docker.py', 'opencontrail_vrouter_netns/daemon_start.py', 'opencontrail_vrouter_netns/daemon_stop.py', diff --git a/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_config.py b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_config.py index 73f5468037c..f29c8e46a49 100644 --- a/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_config.py +++ b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_config.py @@ -1,3 +1,16 @@ +import json +import os + +def validate_custom_attributes(config, section): + return {} + +try: + from haproxy_validator import validate_custom_attributes as validator + from haproxy_validator import custom_attributes_dict +except ImportError: + validator = validate_custom_attributes + custom_attributes_dict = {} + PROTO_TCP = 'TCP' PROTO_HTTP = 'HTTP' PROTO_HTTPS = 'HTTPS' @@ -5,7 +18,7 @@ PROTO_MAP = { PROTO_TCP: 'tcp', PROTO_HTTP: 'http', - PROTO_HTTPS: 'tcp' + PROTO_HTTPS: 'http' } LB_METHOD_MAP = { @@ -25,47 +38,82 @@ HTTPS_PORT = 443 -def build_config(config, conf_dir): +def build_config(conf_file): + with open(conf_file) as data_file: + config = json.load(data_file) + conf_dir = os.path.dirname(conf_file) + conf = [] - sock_path = conf_dir + 'sock' + sock_path = conf_dir + '/haproxy.sock' conf = _set_global_config(config, sock_path) + '\n\n' conf += _set_defaults(config) + '\n\n' conf += _set_frontend(config) + '\n\n' conf += _set_backend(config) + '\n' - print conf - filename = conf_dir + 'conf' + filename = conf_dir + '/haproxy.conf' conf_file = open(filename, 'w') conf_file.write(conf) + return filename def _set_global_config(config, sock_path): + global_custom_attributes = validator(config, 'global') + maxconn = global_custom_attributes.pop('maxconn', None) \ + if 'maxconn' in global_custom_attributes else 65000 + ssl_ciphers = global_custom_attributes.pop('ssl_ciphers', None) \ + if 'ssl_ciphers' in global_custom_attributes else \ + 'ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:' \ + 'ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:' \ + 'RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS' + conf = [ 'global', 'daemon', 'user nobody', - 'group nogroup' + 'group nogroup', 'log /dev/log local0', - 'log /dev/log local1 notice' + 'log /dev/log local1 notice', + 'tune.ssl.default-dh-param 2048', + 'ssl-default-bind-ciphers %s' % ssl_ciphers, + 'ulimit-n 200000', + 'maxconn %d' % maxconn ] conf.append('stats socket %s mode 0666 level user' % sock_path) + for key, value in global_custom_attributes.iteritems(): + cmd = custom_attributes_dict['global'][key]['cmd'] + conf.append(cmd % value) + return ("\n\t".join(conf)) def _set_defaults(config): + default_custom_attributes = validator(config, 'default') + client_timeout = default_custom_attributes.pop('client_timeout', None) \ + if 'client_timeout' in default_custom_attributes else 300000 + server_timeout = default_custom_attributes.pop('server_timeout', None) \ + if 'server_timeout' in default_custom_attributes else 300000 + connect_timeout = default_custom_attributes.pop('connect_timeout', None) \ + if 'connect_timeout' in default_custom_attributes else 5000 + conf = [ 'defaults', 'log global', 'retries 3', 'option redispatch', - 'timeout connect 5000', - 'timeout client 50000', - 'timeout server 50000', + 'timeout connect %d' % connect_timeout, + 'timeout client %d' % client_timeout, + 'timeout server %d' % server_timeout, ] + + for key, value in default_custom_attributes.iteritems(): + cmd = custom_attributes_dict['default'][key]['cmd'] + conf.append(cmd % value) + return ("\n\t".join(conf)) def _set_frontend(config): port = config['vip']['port'] + vip_custom_attributes = validator(config, 'vip') ssl = '' - if port == HTTPS_PORT: - ssl = 'ssl crt %s' % ssl_cert_path + if config['vip']['protocol'] == PROTO_HTTPS: + ssl = 'ssl crt %s no-sslv3' % config['ssl-crt'] conf = [ 'frontend %s' % config['vip']['id'], 'option tcplog', @@ -75,11 +123,18 @@ def _set_frontend(config): ] if config['vip']['connection-limit'] >= 0: conf.append('maxconn %s' % config['vip']['connection-limit']) - if config['vip']['protocol'] == PROTO_HTTP: + if config['vip']['protocol'] == PROTO_HTTP or \ + config['vip']['protocol'] == PROTO_HTTPS: conf.append('option forwardfor') + + for key, value in vip_custom_attributes.iteritems(): + cmd = custom_attributes_dict['vip'][key]['cmd'] + conf.append(cmd % value) + return ("\n\t".join(conf)) def _set_backend(config): + pool_custom_attributes = validator(config, 'pool') conf = [ 'backend %s' % config['pool']['id'], 'mode %s' % PROTO_MAP[config['pool']['protocol']], @@ -88,6 +143,10 @@ def _set_backend(config): if config['pool']['protocol'] == PROTO_HTTP: conf.append('option forwardfor') + for key, value in pool_custom_attributes.iteritems(): + cmd = custom_attributes_dict['pool'][key]['cmd'] + conf.append(cmd % value) + server_suffix, monitor_conf = _set_health_monitor(config) conf.extend(monitor_conf) session_conf = _set_session_persistence(config) @@ -102,6 +161,10 @@ def _set_backend(config): server += ' cookie %d' % config['members'].index(member) conf.append(server) + for key, value in pool_custom_attributes.iteritems(): + cmd = custom_attributes_dict['pool'][key]['cmd'] + conf.append(cmd % value) + return ("\n\t".join(conf)) def _set_health_monitor(config): diff --git a/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_process.py b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_process.py new file mode 100644 index 00000000000..aa9629d9c4f --- /dev/null +++ b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_process.py @@ -0,0 +1,117 @@ +import itertools +import os +import shlex +import subprocess +import haproxy_config + +SUPERVISOR_BASE_DIR = '/etc/contrail/supervisord_vrouter_files/lbaas-haproxy-' + +def stop_haproxy(conf_file, daemon_mode=False): + pool_id = os.path.split(os.path.dirname(conf_file))[1] + try: + if daemon_mode: + _stop_haproxy_daemon(pool_id) + else: + _stop_supervisor_haproxy(pool_id) + except Exception as e: + pass + +def start_update_haproxy(conf_file, netns, daemon_mode=False): + pool_id = os.path.split(os.path.dirname(conf_file))[1] + haproxy_cfg_file = haproxy_config.build_config(conf_file) + try: + if daemon_mode: + _start_haproxy_daemon(pool_id, netns, haproxy_cfg_file) + else: + _start_supervisor_haproxy(pool_id, netns, haproxy_cfg_file) + except Exception as e: + pass + +def _get_lbaas_pid(pool_id): + cmd_list = shlex.split('ps aux') + p1 = subprocess.Popen(cmd_list, stdout=subprocess.PIPE) + cmd_list = shlex.split('grep haproxy') + p2 = subprocess.Popen(cmd_list, stdin=p1.stdout, stdout=subprocess.PIPE) + cmd_list = shlex.split('grep ' + pool_id) + p = subprocess.Popen(cmd_list, stdin=p2.stdout, stdout=subprocess.PIPE) + out, err = p.communicate() + try: + pid = out.split()[1] + except Exception: + pid = None + return pid + +def _stop_haproxy_daemon(pool_id): + last_pid = _get_lbaas_pid(pool_id) + if last_pid: + cmd_list = shlex.split('kill -9 ' + last_pid) + subprocess.Popen(cmd_list) + +def _start_haproxy_daemon(pool_id, netns, conf_file): + last_pid = _get_lbaas_pid(pool_id) + if last_pid: + sf_opt = '-sf ' + last_pid + else: + sf_opt = '' + conf_dir = os.path.dirname(conf_file) + pid_file = conf_dir + '/haproxy.pid' + + cmd = 'ip netns exec %s haproxy -f %s -p %s %s' % \ + (netns, conf_file, pid_file, sf_opt) + cmd_list = shlex.split(cmd) + subprocess.Popen(cmd_list) + +def _stop_supervisor_haproxy(pool_id): + pool_suffix = _get_pool_suffix(pool_id) + file_name = SUPERVISOR_BASE_DIR + pool_suffix + '.ini' + cmd = "rm " + file_name + cmd_list = shlex.split(cmd) + subprocess.Popen(cmd_list) + _update_supervisor() + +def _start_supervisor_haproxy(pool_id, netns, conf_file): + data = [] + data.extend(_set_config(pool_id, netns, conf_file)) + pool_suffix = _get_pool_suffix(pool_id) + with open(SUPERVISOR_BASE_DIR + pool_suffix + '.ini', "w") as f: + f.write('\n'.join(data) + '\n') + _update_supervisor() + +def _get_pool_suffix(pool_id): + return pool_id.split('-')[0] + +def _update_supervisor(): + cmd = "supervisorctl -s unix:///tmp/supervisord_vrouter.sock update" + cmd_list = shlex.split(cmd) + subprocess.Popen(cmd_list) + +def _set_config(pool_id, netns, conf_file): + pool_suffix = _get_pool_suffix(pool_id) + program_name = 'lbaas-haproxy-%s' % pool_suffix + cmd = "supervisorctl -s unix:///tmp/supervisord_vrouter.sock pid " + cmd += program_name + cmd_list = shlex.split(cmd) + p = subprocess.Popen(cmd_list, stdout=subprocess.PIPE) + last_pid, err = p.communicate() + try: + int(last_pid) + sf_opt = '-sf ' + last_pid + except ValueError: + sf_opt = '' + + opts = [ + '[program:%s]' % program_name, + 'command=ip netns exec %s haproxy -f %s -db %s' % \ + (netns, conf_file, sf_opt), + 'priority=420', + 'autostart=true', + 'killasgroup=true', + 'stdout_capture_maxbytes=1MB', + 'redirect_stderr=true', + 'stdout_logfile=/var/log/contrail/lbaas-haproxy-stdout.log', + 'stderr_logfile=/dev/null', + 'startsecs=5', + 'exitcodes=0' + ] + + return itertools.chain(o for o in opts) diff --git a/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_validator.py b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_validator.py new file mode 100644 index 00000000000..8b9d957dc8e --- /dev/null +++ b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/haproxy_validator.py @@ -0,0 +1,105 @@ +custom_attributes_dict = { + 'global': { + 'max_conn': { + 'type': int, + 'limits': [1, 65535], + 'cmd': 'maxconn %d' + }, + 'max_conn_rate': { + 'type': int, + 'limits': [1, 65535], + 'cmd': 'maxconnrate %d' + }, + 'max_sess_rate': { + 'type': int, + 'limits': [1, 65535], + 'cmd': 'maxsessrate %d' + }, + 'max_ssl_conn': { + 'type': int, + 'limits': [1, 65535], + 'cmd': 'maxsslconn %d' + }, + 'max_ssl_rate': { + 'type': int, + 'limits': [1, 65535], + 'cmd': 'maxsslrate %d' + }, + 'ssl_ciphers': { + 'type': str, + 'limits': [1, 100], + 'cmd': 'ssl-default-bind-ciphers %s' + }, + 'tune_http_max_header': { + 'type': int, + 'limits': [1, 128], + 'cmd': 'tune.http.maxhdr %d' + }, + 'tune_ssl_max_record': { + 'type': int, + 'limits': [1, 16384], + 'cmd': 'tune.ssl.maxrecord %d' + } + }, + 'default': { + 'server_timeout': { + 'type': int, + 'limits': [1, 5000000], + 'cmd': 'timeout server %d' + }, + 'client_timeout': { + 'type': int, + 'limits': [1, 5000000], + 'cmd': 'timeout client %d' + }, + 'connect_timeout': { + 'type': int, + 'limits': [1, 5000000], + 'cmd': 'timeout connect %d' + } + }, + 'vip': { + 'http_server_close': { + 'type': bool, + 'limits': ['True', 'False'], + 'cmd': '%soption http-server-close' + }, + 'rate_limit_sessions': { + 'type': int, + 'limits': [1, 65535], + 'cmd': 'rate-limit sessions %d' + } + }, + 'pool': {}, +} + +def validate_custom_attributes(config, section): + section_dict = {} + if 'custom-attributes' in config and section in custom_attributes_dict: + custom_attributes = config['custom-attributes'] + for key, value in custom_attributes.iteritems(): + if key in custom_attributes_dict[section]: + #Sanitize the value + try: + type_attr = custom_attributes_dict[section][key]['type'] + limits = custom_attributes_dict[section][key]['limits'] + if type_attr == int: + value = type_attr(value) + if value in range(limits[0], limits[1]): + section_dict.update({key:value}) + elif type_attr == str: + if len(value) in range(limits[0], limits[1]): + section_dict.update({key:value}) + elif type_attr == bool: + if value in limits: + if value == 'True': + value = '' + elif value == 'False': + value = 'no ' + section_dict.update({key:value}) + except Exception as e: + print "Skipping key: %s, value: %s due to validation failure" \ + % (key, value) + continue + + return section_dict diff --git a/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/vrouter_netns.py b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/vrouter_netns.py index 8429f30a31f..13a088d7b5e 100644 --- a/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/vrouter_netns.py +++ b/src/vnsw/opencontrail-vrouter-netns/opencontrail_vrouter_netns/vrouter_netns.py @@ -34,6 +34,7 @@ import json from linux import ip_lib +import haproxy_process def validate_uuid(val): @@ -128,40 +129,13 @@ def set_snat(self): self.SNAT_RT_TABLES_ID, 'via', self.gw_ip, 'dev', str(self.nic_left['name'])]) - def _get_lbaas_pid(self): - cmd = """ps aux | grep \'%(process)s -f %(file)s\' | grep -v grep - """ % {'process':self.LBAAS_PROCESS, 'file':self.cfg_file} - try: - if "check_output" not in dir(subprocess): - s = _check_output(cmd) - else: - s = subprocess.check_output(cmd, shell=True) - - except subprocess.CalledProcessError: - return None - words = s.split() - pid = int(words[1]) - return pid - def set_lbaas(self): if not self.ip_ns.netns.exists(self.namespace): raise ValueError('Need to create the network namespace before set ' 'up the lbaas') - pid_file = self.cfg_file + ".pid" - pid = self._get_lbaas_pid() - if (self.update is False): - if pid is not None: - self.release_lbaas() - - self.ip_ns.netns.execute([self.LBAAS_PROCESS, '-f', self.cfg_file, '-D', - '-p', pid_file]) - self.ip_ns.netns.execute(['route', 'add', 'default', 'gw', self.gw_ip]) - else: - if pid is not None: - self.ip_ns.netns.execute([self.LBAAS_PROCESS, '-f', self.cfg_file, '-D', '-p', pid_file, '-sf', pid]) - else: - self.ip_ns.netns.execute([self.LBAAS_PROCESS, '-f', self.cfg_file, '-D', - '-p', pid_file]) + + haproxy_process.start_update_haproxy(self.cfg_file, self.namespace, True) + try: self.ip_ns.netns.execute(['route', 'add', 'default', 'gw', self.gw_ip]) except RuntimeError: @@ -171,17 +145,9 @@ def release_lbaas(self): if not self.ip_ns.netns.exists(self.namespace): raise ValueError('Need to create the network namespace before ' 'relasing lbaas') - pid = self._get_lbaas_pid() - if pid is not None: - cmd = """kill -9 %(pid)s""" % {'pid':pid} - try: - if "check_output" not in dir(subprocess): - s = _check_output(cmd) - else: - s = subprocess.check_output(cmd, shell=True) - print ("Haproxy process with pid %d config file %s killed" %(pid, self.cfg_file), file=sys.stderr) - except subprocess.CalledProcessError: - print ("SIGKILL Error for pid %d %s" %(pid, self.cfg_file), file=sys.stderr) + + haproxy_process.stop_haproxy(self.cfg_file, True) + try: self.ip_ns.netns.execute(['route', 'del', 'default']) except RuntimeError: