diff --git a/contrail_provisioning/collector/setup.py b/contrail_provisioning/collector/setup.py index bc0019d5..ea48d24b 100755 --- a/contrail_provisioning/collector/setup.py +++ b/contrail_provisioning/collector/setup.py @@ -272,6 +272,9 @@ def load_redis_upstart_file(self): local("sudo update-rc.d redis-server disable") + def restart_collector(self): + local("sudo service supervisor-analytics restart") + def run_services(self): #disable redis from init.d since upstart has been added if self.pdist == 'Ubuntu': @@ -287,5 +290,10 @@ def main(args_str = None): collector = CollectorSetup(args_str) collector.setup() +def fix_collector_config(args_str = None): + collector = CollectorSetup(args_str) + collector.fixup_config_files() + collector.restart_collector() + if __name__ == "__main__": main() diff --git a/contrail_provisioning/config/common.py b/contrail_provisioning/config/common.py index a6156578..00ff8469 100755 --- a/contrail_provisioning/config/common.py +++ b/contrail_provisioning/config/common.py @@ -330,6 +330,9 @@ def fixup_cassandra_config(self): template_vals, self._temp_dir_name + '/contrail-config-database.conf') local("sudo mv %s/contrail-config-database.conf /etc/contrail/contrail-database.conf" %(self._temp_dir_name)) + def restart_config(self): + local('sudo service supervisor-config restart') + def run_services(self): if self._args.internal_vip: # Assumption cfgm and openstack in same node. diff --git a/contrail_provisioning/config/setup.py b/contrail_provisioning/config/setup.py index 7e18eb21..5a2dcb28 100755 --- a/contrail_provisioning/config/setup.py +++ b/contrail_provisioning/config/setup.py @@ -122,5 +122,14 @@ def main(args_str = None): config = ConfigBaseSetup(config_args) config.setup() +def fix_cfgm_config_files(args_str = None): + config_args = ConfigSetup(args_str)._args + if config_args.orchestrator == 'openstack': + config = ConfigOpenstackSetup(config_args) + else: + config = ConfigBaseSetup(config_args) + config.fixup_config_files() + config.restart_config() + if __name__ == "__main__": main() diff --git a/contrail_provisioning/database/setup.py b/contrail_provisioning/database/setup.py index 0adac160..f4e921e5 100755 --- a/contrail_provisioning/database/setup.py +++ b/contrail_provisioning/database/setup.py @@ -36,7 +36,7 @@ def __init__(self, args_str = None): self.database_listen_ip = self._args.self_ip self.database_seed_list = self._args.seed_list self.database_dir = self._args.dir - + def parse_args(self, args_str): ''' Eg. setup-vnc-database @@ -205,14 +205,7 @@ def fixup_config_files(self): if self.pdist == 'Ubuntu': local('echo ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" >> /etc/zookeeper/conf/environment') - zk_index = 1 - for zk_ip in self._args.zookeeper_ip_list: - local('sudo echo "server.%d=%s:2888:3888" >> /etc/zookeeper/conf/zoo.cfg' %(zk_index, zk_ip)) - zk_index = zk_index + 1 - - #put cluster-unique zookeeper's instance id in myid - local('sudo echo "%s" > /var/lib/zookeeper/myid' %(self._args.database_index)) - + self.fix_zookeeper_servers_config() self.fixup_kafka_server_properties(listen_ip) def fixup_kafka_server_properties(self, listen_ip): @@ -221,7 +214,7 @@ def fixup_kafka_server_properties(self, listen_ip): cnd = os.path.exists(KAFKA_SERVER_PROPERTIES) if not cnd: raise RuntimeError('%s does not appear to be a kafka config directory' % KAFKA_SERVER_PROPERTIES) - if self._args.kafka_broker_id is not None: + if self._args.kafka_broker_id is not None: self.replace_in_file(KAFKA_SERVER_PROPERTIES, 'broker.id=', 'broker.id='+self._args.kafka_broker_id) #Handling for Kafka-0.8.3 @@ -232,7 +225,7 @@ def fixup_kafka_server_properties(self, listen_ip): #Add all the zoo keeper server address to the server.properties file zk_list = [server + ":2181" for server in self._args.zookeeper_ip_list] zk_list_str = ','.join(map(str, zk_list)) - self.replace_in_file(KAFKA_SERVER_PROPERTIES, 'zookeeper.connect=', 'zookeeper.connect='+zk_list_str) + self.replace_in_file(KAFKA_SERVER_PROPERTIES, 'zookeeper.connect=*', 'zookeeper.connect='+zk_list_str) self.replace_in_file(KAFKA_SERVER_PROPERTIES, '#advertised.host.name=',\ 'advertised.host.name='+listen_ip) #Set replication factor to 2 if more than one kafka broker is available @@ -297,9 +290,34 @@ def file_pattern_check(self, file_name, regexp): return True return False + def fix_zookeeper_servers_config(self): + zk_index = 1 + # Instead of inserting/deleting config, remove all the zoo keeper servers + # and re-generate. + local("sudo sed -i '/server.[1-9]*=/d' /etc/zookeeper/conf/zoo.cfg") + + for zk_ip in self._args.zookeeper_ip_list: + local('sudo echo "server.%d=%s:2888:3888" >> /etc/zookeeper/conf/zoo.cfg' %(zk_index, zk_ip)) + zk_index = zk_index + 1 + + #put cluster-unique zookeeper's instance id in myid + local('sudo echo "%s" > /var/lib/zookeeper/myid' %(self._args.database_index)) + + def restart_zookeeper(self): + local('sudo service zookeeper restart') + def main(args_str = None): database = DatabaseSetup(args_str) database.setup() +def update_zookeeper_servers(args_str = None): + database = DatabaseSetup(args_str) + database.fix_zookeeper_servers_config() + database.fixup_kafka_server_properties() + +def restart_zookeeper_server(args_str = None): + database = DatabaseSetup(args_str) + database.restart_zookeeper() + if __name__ == "__main__": main() diff --git a/contrail_provisioning/openstack/ha/galera_setup.py b/contrail_provisioning/openstack/ha/galera_setup.py index 08e186c0..84260761 100755 --- a/contrail_provisioning/openstack/ha/galera_setup.py +++ b/contrail_provisioning/openstack/ha/galera_setup.py @@ -35,6 +35,18 @@ def __init__(self, args_str = None): self._args = None if not args_str: args_str = ' '.join(sys.argv[1:]) + + if self.pdist in ['Ubuntu']: + local("ln -sf /bin/true /sbin/chkconfig") + self.mysql_svc = 'mysql' + self.mysql_conf = '/etc/mysql/my.cnf' + self.wsrep_conf = '/etc/mysql/conf.d/wsrep.cnf' + elif self.pdist in ['centos', 'redhat']: + self.mysql_svc = 'mysqld' + self.mysql_conf = '/etc/my.cnf' + self.wsrep_conf = self.mysql_conf + self.mysql_token_file = '/etc/contrail/mysql.token' + self.parse_args(args_str) self.mysql_redo_log_sz = '5242880' @@ -52,20 +64,13 @@ def parse_args(self, args_str): parser.add_argument("--openstack0_user", help = "Sudo user of this openstack node") parser.add_argument("--openstack0_passwd", help = "Sudo user password of this openstack node") parser.add_argument("--galera_ip_list", help = "List of IP Addresses of galera servers", nargs='+', type=str) - parser.add_argument("--internal_vip", help = "Internal Virtual IP Address of HA Openstack nodes") + parser.add_argument("--internal_vip", help = "Internal Virtual IP Address of HA Openstack nodes"), parser.add_argument("--external_vip", help = "External Virtual IP Address of HA Openstack nodes"), + parser.add_argument("--node_to_add", help = "IP address of the new node to add into the Galera cluster", type=str), parser.add_argument("--zoo_ip_list", help = "List of IP Addresses of zookeeper servers", nargs='+', type=str) self._args = parser.parse_args(self.remaining_argv) - def fixup_config_files(self): - with settings(warn_only=True): - local("service contrail-hamon stop") - local("service cmon stop") - local("service mysql stop") - local("rm -rf /var/lib/mysql/grastate.dat") - local("rm -rf /var/lib/mysql/galera.cache") - self.cleanup_redo_log() - + def fix_galera_config(self, bootstrap=True): # fix galera_param template_vals = {'__mysql_host__' : self._args.self_ip, '__mysql_wsrep_nodes__' : @@ -75,44 +80,30 @@ def fixup_config_files(self): self._temp_dir_name + '/galera_param') local("sudo mv %s/galera_param /etc/contrail/ha/" % (self._temp_dir_name)) - zk_servers_ports = ','.join(['%s:2181' %(s) for s in self._args.zoo_ip_list]) - - # fix cmon_param - template_vals = {'__internal_vip__' : self._args.internal_vip, - '__haproxy_dips__' : - '"' + '" "'.join(self._args.galera_ip_list) + '"', - '__external_vip__' : self._args.external_vip, - '__zooipports__' : zk_servers_ports - } - self._template_substitute_write(cmon_param_template.template, - template_vals, - self._temp_dir_name + '/cmon_param') - local("sudo mv %s/cmon_param /etc/contrail/ha/" % (self._temp_dir_name)) - - local("echo %s > /etc/contrail/galeraid" % self._args.openstack_index) if self.pdist in ['Ubuntu']: local("ln -sf /bin/true /sbin/chkconfig") - self.mysql_svc = 'mysql' - self.mysql_conf = '/etc/mysql/my.cnf' - wsrep_conf = '/etc/mysql/conf.d/wsrep.cnf' wsrep_conf_file = 'wsrep.cnf' wsrep_template = wsrep_conf_template.template elif self.pdist in ['centos', 'redhat']: - self.mysql_svc = 'mysqld' - self.mysql_conf = '/etc/my.cnf' - wsrep_conf = self.mysql_conf wsrep_conf_file = 'my.cnf' wsrep_template = wsrep_conf_centos_template.template - self.mysql_token_file = '/etc/contrail/mysql.token' - self.install_mysql_db() - if self._args.openstack_index == 1: - self.create_mysql_token_file() + if self._args.openstack_index == 1 and bootstrap == True: + wsrep_cluster_address= '' else: - self.get_mysql_token_file() - self.set_mysql_root_password() - self.setup_grants() - self.setup_cron() + wsrep_cluster_address = (':4567,'.join(self._args.galera_ip_list) + ':4567') + + template_vals = {'__wsrep_nodes__' : wsrep_cluster_address, + '__wsrep_node_address__' : self._args.self_ip, + '__mysql_token__' : self.mysql_token, + '__wsrep_cluster_size__': len(self._args.galera_ip_list), + '__wsrep_inc_offset__': self._args.openstack_index*100, + } + self._template_substitute_write(wsrep_template, template_vals, + self._temp_dir_name + '/%s' % wsrep_conf_file) + local("sudo mv %s/%s %s" % (self._temp_dir_name, wsrep_conf_file, + self.wsrep_conf)) + # fixup mysql/wsrep config local('sed -i -e "s/bind-address/#bind-address/" %s' % self.mysql_conf) local('sed -ibak "s/max_connections.*/max_connections=10000/" %s' % self.mysql_conf) @@ -124,6 +115,7 @@ def fixup_config_files(self): local('sed -i "/\[mysqld\]/a\lock_wait_timeout=600" %s' % self.mysql_conf) local('sed -i "/\[mysqld\]/a\interactive_timeout = 60" %s' % self.mysql_conf) local('sed -i "/\[mysqld\]/a\wait_timeout = 60" %s' % self.mysql_conf) + # FIX for UTF8 if self.pdist in ['Ubuntu']: sku = local("dpkg -p contrail-install-packages | grep Version: | cut -d'~' -f2", capture=True) @@ -132,23 +124,14 @@ def fixup_config_files(self): local('sed -i "/\[mysqld\]/a\init-connect=\'SET NAMES utf8\'" %s' % self.mysql_conf) local('sed -i "/\[mysqld\]/a\collation-server = utf8_general_ci" %s' % self.mysql_conf) - if self._args.openstack_index == 1: - wsrep_cluster_address= '' - else: - wsrep_cluster_address = (':4567,'.join(self._args.galera_ip_list) + ':4567') - - template_vals = {'__wsrep_nodes__' : wsrep_cluster_address, - '__wsrep_node_address__' : self._args.self_ip, - '__mysql_token__' : self.mysql_token, - '__wsrep_cluster_size__': len(self._args.galera_ip_list), - '__wsrep_inc_offset__': self._args.openstack_index*100, - } - self._template_substitute_write(wsrep_template, template_vals, - self._temp_dir_name + '/%s' % wsrep_conf_file) - local("sudo mv %s/%s %s" % (self._temp_dir_name, wsrep_conf_file, - wsrep_conf)) - if self._args.openstack_index == 1: - local('sed -ibak "s#wsrep_cluster_address=.*#wsrep_cluster_address=gcomm://#g" %s' % (wsrep_conf)) + def fix_cmon_config(self): + zk_servers_ports = ','.join(['%s:2181' %(s) for s in self._args.zoo_ip_list]) + template_vals = {'__internal_vip__' : self._args.internal_vip, + '__haproxy_dips__' : '"' + '" "'.join(self._args.galera_ip_list) + '"'} + self._template_substitute_write(cmon_param_template.template, + template_vals, + self._temp_dir_name + '/cmon_param') + local("sudo mv %s/cmon_param /etc/contrail/ha/" % (self._temp_dir_name)) # fixup cmon config template_vals = {'__mysql_nodes__' : ','.join(self._args.galera_ip_list), @@ -159,6 +142,32 @@ def fixup_config_files(self): local("sudo mv %s/cmon.cnf /etc/cmon.cnf" % (self._temp_dir_name)) local("sudo chmod 444 /etc/init.d/cmon") + def fixup_config_files(self): + with settings(warn_only=True): + local("service contrail-hamon stop") + local("service cmon stop") + local("service mysql stop") + local("rm -rf /var/lib/mysql/grastate.dat") + local("rm -rf /var/lib/mysql/galera.cache") + self.cleanup_redo_log() + + if self._args.openstack_index == 1: + self.create_mysql_token_file() + else: + self.get_mysql_token_file() + self.fix_galera_config() + + if self._args.openstack_index == 1: + local('sed -ibak "s#wsrep_cluster_address=.*#wsrep_cluster_address=gcomm://#g" %s' % (self.wsrep_conf)) + self.fix_cmon_config() + + local("echo %s > /etc/contrail/galeraid" % self._args.openstack_index) + self.install_mysql_db() + self.set_mysql_root_password() + ip_list = self._args.galera_ip_list + ['localhost', '127.0.0.1'] + self.setup_grants(ip_list) + self.setup_cron() + def install_mysql_db(self): local('chkconfig %s on' % self.mysql_svc) local('chown -R mysql:mysql /var/lib/mysql/') @@ -208,10 +217,9 @@ def set_mysql_root_password(self): else: raise RuntimeError("MySQL root password unknown, reset and retry") - def setup_grants(self): + def setup_grants(self, ip_list): mysql_cmd = "mysql --defaults-file=%s -uroot -p%s -e" % (self.mysql_conf, self.mysql_token) - host_list = self._args.galera_ip_list + ['localhost', '127.0.0.1'] - for host in host_list: + for host in ip_list: with settings(hide('everything'),warn_only=True): local('mysql -u root -p%s -e "CREATE USER \'root\'@\'%s\' IDENTIFIED BY %s"' % (self.mysql_token, host, self.mysql_token)) local('%s "SET WSREP_ON=0;SET SQL_LOG_BIN=0; GRANT ALL ON *.* TO root@%s IDENTIFIED BY \'%s\'"' % @@ -226,23 +234,61 @@ def setup_cron(self): local('crontab %s/galera_cron' % self._temp_dir_name) local('rm %s/galera_cron' % self._temp_dir_name) + def add_new_mysql_perm(self): + # Before adding the new node into the cluster + # we should add the new node's user as an + # authorized user in the other servers. + self.get_mysql_token_file() + self.setup_grants(self._args.node_to_add.split()) + + def verify_mysql_server_status(self, ip, mysql_token): + cmd = "mysql -h%s -uroot -p%s " % (ip, mysql_token) + cmd += "-e \"show global status where variable_name='wsrep_local_state'\" | awk '{print $2}' | sed '1d'" + for i in range(10): + wsrep_local_state = local(cmd, capture=True).strip() + if wsrep_local_state == '4': + return True + else: + time.sleep(12) + print "Waiting for first galera node to create new cluster." + if wsrep_local_state != '4': + return False + return False + + def restart_mysql_server(self): + # Restart the local MySQL server and wait + # for 10 seconds for it to restart. + local("service %s restart" % self.mysql_svc) + time.sleep(10) + wsrep_state_result = self.verify_mysql_server_status(self._args.self_ip, self.mysql_token) + if wsrep_state_result == False: + raise RuntimeError("Unable able to bring up galera in %s" % self._args.self_ip) + + def add_new_cluster_config(self): + with settings(warn_only=True): + local("service contrail-hamon stop") + local("service cmon stop") + self.get_mysql_token_file() + self.fix_galera_config(bootstrap=False) + self.fix_cmon_config() + self.restart_mysql_server() + wsrep_state_result = self.verify_mysql_server_status(self._args.self_ip, self.mysql_token) + if wsrep_state_result == False: + raise RuntimeError("Unable able to bring up galera in %s" % self._args.self_ip) + local("sudo update-rc.d -f mysql remove") + local("sudo update-rc.d mysql defaults") + local("service contrail-hamon start") + def run_services(self): self.cleanup_redo_log() if self._args.openstack_index == 1: local("service %s restart" % self.mysql_svc) else: - cmd = "mysql -h%s -uroot -p%s " % (self._args.galera_ip_list[0], self.mysql_token) - cmd += "-e \"show global status where variable_name='wsrep_local_state'\" | awk '{print $2}' | sed '1d'" - for i in range(10): - wsrep_local_state = local(cmd, capture=True).strip() - if wsrep_local_state == '4': - break - else: - time.sleep(2) - print "Waiting for first galera node to create new cluster." - if wsrep_local_state != '4': + wsrep_state_result = self.verify_mysql_server_status(self._args.galera_ip_list[0], self.mysql_token) + if wsrep_state_result == False: raise RuntimeError("Unable able to bring up galera in first node, please verify and continue.") local("service %s restart" % self.mysql_svc) + local("sudo update-rc.d -f mysql remove") local("sudo update-rc.d mysql defaults") @@ -259,5 +305,13 @@ def main(args_str = None): galera = GaleraSetup(args_str) galera.setup() +def add_mysql_perm(args_str=None): + galera = GaleraSetup(args_str) + galera.add_new_mysql_perm() + +def add_galera_cluster_config(args_str=None): + galera = GaleraSetup(args_str) + galera.add_new_cluster_config() + if __name__ == "__main__": main() diff --git a/contrail_provisioning/webui/setup.py b/contrail_provisioning/webui/setup.py index 6dfa8367..03dfbb23 100755 --- a/contrail_provisioning/webui/setup.py +++ b/contrail_provisioning/webui/setup.py @@ -160,6 +160,9 @@ def fixup_config_global_js(self): local("sudo sed \"/config.orchestration.Manager/ a \\\n// multi_tenancy\\nconfig.multi_tenancy = {};\\nconfig.multi_tenancy.enabled = false;\" /etc/contrail/config.global.js > config.global.js.new") local("sudo mv config.global.js.new /etc/contrail/config.global.js") + def restart_webui(self): + local("sudo service supervisor-webui restart") + def run_services(self): local("sudo webui-server-setup.sh") @@ -167,5 +170,11 @@ def main(args_str = None): webui = WebuiSetup(args_str) webui.setup() +def fix_webui_config(args_str = None): + webui = WebuiSetup(args_str) + webui.fixup_config_files() + webui.restart_webui() + + if __name__ == "__main__": main() diff --git a/setup.py b/setup.py index d183d9ae..208c9f1c 100755 --- a/setup.py +++ b/setup.py @@ -34,6 +34,13 @@ def requirements(filename): 'setup-vnc-storage-webui = contrail_provisioning.storage.webui_setup:main', 'setup-vcenter-plugin = contrail_provisioning.vcenter_plugin.setup:main', 'setup-vnc-tor-agent = contrail_provisioning.compute.toragent.setup:main', + 'add-mysql-perm = contrail_provisioning.openstack.ha.galera_setup:add_mysql_perm', + 'add-galera-config = contrail_provisioning.openstack.ha.galera_setup:add_galera_cluster_config', + 'update-zoo-servers = contrail_provisioning.database.setup:update_zookeeper_servers', + 'restart-zoo-server = contrail_provisioning.database.setup:restart_zookeeper_server', + 'update-cfgm-config = contrail_provisioning.config.setup:fix_cfgm_config_files', + 'update-collector-config = contrail_provisioning.collector.setup:fix_collector_config', + 'update-webui-config = contrail_provisioning.webui.setup:fix_webui_config', # Reset scripts 'reset-vnc-database = contrail_provisioning.database.reset:main', # Upgrade scripts