Fixes: Add new controller to an existing cluster

Adding a new controller to an existing cluster. Changes in
provisioning code so that SM can also call these entry points.

* Change GaleraSetup to add mysql permissions to a new node
* Independent commands to restart different services like
config, openstack, zookeeper services
* Change GaleraSetup to add new node to an existing cluster
and change the relevant config in the Galera nodes
* Change zookeeper config to add a new node.

Change-Id: I776fbddc41a89e32850a575d5ef9108f06b33246
ClosesBug: 1449232

contrail_provisioning/collector/setup.py

@@ -216,6 +216,9 @@ def fixup_contrail_analytics_api(self):
                                         template_vals, self._temp_dir_name + '/contrail-analytics-api.conf')
         local("sudo mv %s/contrail-analytics-api.conf /etc/contrail/contrail-analytics-api.conf" %(self._temp_dir_name))
 
+    def restart_collector(self):
+        local("sudo service supervisor-analytics restart")
+
     def run_services(self):
         if self._args.num_nodes:
             local("sudo collector-server-setup.sh multinode")
@@ -227,5 +230,10 @@ def main(args_str = None):
     collector = CollectorSetup(args_str)
     collector.setup()
 
+def fix_collector_config(args_str = None):
+    collector = CollectorSetup(args_str)
+    collector.fixup_config_files()
+    collector.restart_collector()
+
 if __name__ == "__main__":
     main()

contrail_provisioning/config/common.py

@@ -293,6 +293,9 @@ def fixup_contrail_config_nodemgr(self):
                                         template_vals, self._temp_dir_name + '/contrail-config-nodemgr.conf')
         local("sudo mv %s/contrail-config-nodemgr.conf /etc/contrail/contrail-config-nodemgr.conf" %(self._temp_dir_name))
 
+    def restart_config(self):
+        local('sudo service supervisor-config restart')
+
     def run_services(self):
         if self._args.internal_vip:
             # Assumption cfgm and openstack in same node.

contrail_provisioning/config/setup.py

@@ -121,5 +121,14 @@ def main(args_str = None):
         config = ConfigBaseSetup(config_args)
     config.setup()
 
+def fix_cfgm_config_files(args_str = None):
+    config_args = ConfigSetup(args_str)._args
+    if config_args.orchestrator == 'openstack':
+        config = ConfigOpenstackSetup(config_args)
+    else:
+        config = ConfigBaseSetup(config_args)
+    config.fixup_config_files()
+    config.restart_config()
+
 if __name__ == "__main__":
     main() 

contrail_provisioning/database/setup.py

@@ -34,7 +34,7 @@ def __init__(self, args_str = None):
         self.database_listen_ip = self._args.self_ip
         self.database_seed_list = self._args.seed_list
         self.database_dir = self._args.dir
-        
+
     def parse_args(self, args_str):
         '''
         Eg. setup-vnc-database
@@ -176,14 +176,7 @@ def fixup_config_files(self):
         if self.pdist == 'Ubuntu':
             local('echo ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" >> /etc/zookeeper/conf/environment')
 
-        zk_index = 1
-        for zk_ip in self._args.zookeeper_ip_list:
-            local('sudo echo "server.%d=%s:2888:3888" >> /etc/zookeeper/conf/zoo.cfg' %(zk_index, zk_ip))
-            zk_index = zk_index + 1
-
-        #put cluster-unique zookeeper's instance id in myid
-        local('sudo echo "%s" > /var/lib/zookeeper/myid' %(self._args.database_index))
-
+        self.fix_zookeeper_servers_config()
         self.fixup_kafka_server_properties()
 
     def fixup_kafka_server_properties(self):
@@ -192,7 +185,7 @@ def fixup_kafka_server_properties(self):
         cnd = os.path.exists(KAFKA_SERVER_PROPERTIES)
         if not cnd:
             raise RuntimeError('%s does not appear to be a kafka config directory' % KAFKA_SERVER_PROPERTIES)
-	if self._args.kafka_broker_id is not None:
+        if self._args.kafka_broker_id is not None:
             self.replace_in_file(KAFKA_SERVER_PROPERTIES, 'broker.id=', 'broker.id='+self._args.kafka_broker_id)
         else:
             #kafka not enabled
@@ -201,7 +194,7 @@ def fixup_kafka_server_properties(self):
         #Add all the zoo keeper server address to the server.properties file
         zk_list = [server + ":2181" for server in self._args.zookeeper_ip_list]
         zk_list_str = ','.join(map(str, zk_list))
-        self.replace_in_file(KAFKA_SERVER_PROPERTIES, 'zookeeper.connect=', 'zookeeper.connect='+zk_list_str)
+        self.replace_in_file(KAFKA_SERVER_PROPERTIES, 'zookeeper.connect=*', 'zookeeper.connect='+zk_list_str)
         #Set replication factor to 2 if more than one kafka broker is available
         if (len(zk_list)>1):
             if not self.file_pattern_check(KAFKA_SERVER_PROPERTIES, 'default.replication.factor'):
@@ -231,9 +224,34 @@ def file_pattern_check(self, file_name, regexp):
                     return True
         return False
 
+    def fix_zookeeper_servers_config(self):
+        zk_index = 1
+        # Instead of inserting/deleting config, remove all the zoo keeper servers
+        # and re-generate.
+        local("sudo sed -i '/server.[1-9]*=/d' /etc/zookeeper/conf/zoo.cfg")
+
+        for zk_ip in self._args.zookeeper_ip_list:
+            local('sudo echo "server.%d=%s:2888:3888" >> /etc/zookeeper/conf/zoo.cfg' %(zk_index, zk_ip))
+            zk_index = zk_index + 1
+
+        #put cluster-unique zookeeper's instance id in myid
+        local('sudo echo "%s" > /var/lib/zookeeper/myid' %(self._args.database_index))
+
+    def restart_zookeeper(self):
+        local('sudo service zookeeper restart')
+
 def main(args_str = None):
     database = DatabaseSetup(args_str)
     database.setup()
 
+def update_zookeeper_servers(args_str = None):
+    database = DatabaseSetup(args_str)
+    database.fix_zookeeper_servers_config()
+    database.fixup_kafka_server_properties()
+
+def restart_zookeeper_server(args_str = None):
+    database = DatabaseSetup(args_str)
+    database.restart_zookeeper()
+
 if __name__ == "__main__":
     main()

contrail_provisioning/openstack/ha/galera_setup.py

@@ -35,6 +35,18 @@ def __init__(self, args_str = None):
         self._args = None
         if not args_str:
             args_str = ' '.join(sys.argv[1:])
+
+        if self.pdist in ['Ubuntu']:
+            local("ln -sf /bin/true /sbin/chkconfig")
+            self.mysql_svc = 'mysql'
+            self.mysql_conf = '/etc/mysql/my.cnf'
+            self.wsrep_conf = '/etc/mysql/conf.d/wsrep.cnf'
+        elif self.pdist in ['centos', 'redhat']:
+            self.mysql_svc = 'mysqld'
+            self.mysql_conf = '/etc/my.cnf'
+            self.wsrep_conf = self.mysql_conf
+        self.mysql_token_file = '/etc/contrail/mysql.token'
+
         self.parse_args(args_str)
         self.mysql_redo_log_sz = '5242880'
 
@@ -53,18 +65,11 @@ def parse_args(self, args_str):
         parser.add_argument("--openstack0_passwd", help = "Sudo user password  of this openstack node")
         parser.add_argument("--galera_ip_list", help = "List of IP Addresses of galera servers", nargs='+', type=str)
         parser.add_argument("--internal_vip", help = "Internal Virtual IP Address of HA Openstack nodes")
-        parser.add_argument("--external_vip", help = "External Virtual IP Address of HA Openstack nodes"),
+        parser.add_argument("--external_vip", help = "External Virtual IP Address of HA Openstack nodes")
+        parser.add_argument("--node_to_add", help = "IP address of the new node to add into the Galera cluster", type=str),
         self._args = parser.parse_args(self.remaining_argv)
 
-    def fixup_config_files(self):
-        with settings(warn_only=True):
-            local("service contrail-hamon stop")
-            local("service cmon stop")
-            local("service mysql stop")
-            local("rm -rf /var/lib/mysql/grastate.dat")
-            local("rm -rf /var/lib/mysql/galera.cache")
-            self.cleanup_redo_log()
-
+    def fix_galera_config(self, bootstrap=True):
         # fix galera_param
         template_vals = {'__mysql_host__' : self._args.self_ip,
                          '__mysql_wsrep_nodes__' :
@@ -74,40 +79,30 @@ def fixup_config_files(self):
                                         self._temp_dir_name + '/galera_param')
         local("sudo mv %s/galera_param /etc/contrail/ha/" % (self._temp_dir_name))
 
-        # fix cmon_param
-        template_vals = {'__internal_vip__' : self._args.internal_vip,
-                         '__haproxy_dips__' :
-                         '"' + '" "'.join(self._args.galera_ip_list) + '"',
-                         '__external_vip__' : self._args.external_vip}
-        self._template_substitute_write(cmon_param_template.template,
-                                        template_vals,
-                                        self._temp_dir_name + '/cmon_param')
-        local("sudo mv %s/cmon_param /etc/contrail/ha/" % (self._temp_dir_name))
-
-        local("echo %s > /etc/contrail/galeraid" % self._args.openstack_index)
         if self.pdist in ['Ubuntu']:
             local("ln -sf /bin/true /sbin/chkconfig")
-            self.mysql_svc = 'mysql'
-            self.mysql_conf = '/etc/mysql/my.cnf'
-            wsrep_conf = '/etc/mysql/conf.d/wsrep.cnf'
             wsrep_conf_file = 'wsrep.cnf'
             wsrep_template = wsrep_conf_template.template
         elif self.pdist in ['centos', 'redhat']:
-            self.mysql_svc = 'mysqld'
-            self.mysql_conf = '/etc/my.cnf'
-            wsrep_conf = self.mysql_conf
             wsrep_conf_file = 'my.cnf'
             wsrep_template = wsrep_conf_centos_template.template
-        self.mysql_token_file = '/etc/contrail/mysql.token'
 
-        self.install_mysql_db()
-        if self._args.openstack_index == 1:
-            self.create_mysql_token_file()
+        if self._args.openstack_index == 1 and bootstrap == True:
+            wsrep_cluster_address= ''
         else:
-            self.get_mysql_token_file()
-        self.set_mysql_root_password()
-        self.setup_grants()
-        self.setup_cron()
+            wsrep_cluster_address =  (':4567,'.join(self._args.galera_ip_list) + ':4567')
+
+        template_vals = {'__wsrep_nodes__' : wsrep_cluster_address,
+                         '__wsrep_node_address__' : self._args.self_ip,
+                         '__mysql_token__' : self.mysql_token,
+                         '__wsrep_cluster_size__': len(self._args.galera_ip_list),
+                         '__wsrep_inc_offset__': self._args.openstack_index*100,
+                        }
+        self._template_substitute_write(wsrep_template, template_vals,
+                                  self._temp_dir_name + '/%s' % wsrep_conf_file)
+        local("sudo mv %s/%s %s" % (self._temp_dir_name, wsrep_conf_file,
+                                    self.wsrep_conf))
+
         # fixup mysql/wsrep config
         local('sed -i -e "s/bind-address/#bind-address/" %s' % self.mysql_conf)
         local('sed -ibak "s/max_connections.*/max_connections=10000/" %s' % self.mysql_conf)
@@ -119,6 +114,7 @@ def fixup_config_files(self):
         local('sed -i "/\[mysqld\]/a\lock_wait_timeout=600" %s' % self.mysql_conf)
         local('sed -i "/\[mysqld\]/a\interactive_timeout = 60" %s' % self.mysql_conf)
         local('sed -i "/\[mysqld\]/a\wait_timeout = 60" %s' % self.mysql_conf)
+
         # FIX for UTF8
         if self.pdist in ['Ubuntu']:
             sku = local("dpkg -p contrail-install-packages | grep Version: | cut -d'~' -f2", capture=True)
@@ -127,23 +123,13 @@ def fixup_config_files(self):
                 local('sed -i "/\[mysqld\]/a\init-connect=\'SET NAMES utf8\'" %s' % self.mysql_conf)
                 local('sed -i "/\[mysqld\]/a\collation-server = utf8_general_ci" %s' % self.mysql_conf)
 
-        if self._args.openstack_index == 1:
-            wsrep_cluster_address= ''
-        else:
-            wsrep_cluster_address =  (':4567,'.join(self._args.galera_ip_list) + ':4567')
-
-        template_vals = {'__wsrep_nodes__' : wsrep_cluster_address,
-                         '__wsrep_node_address__' : self._args.self_ip,
-                         '__mysql_token__' : self.mysql_token,
-                         '__wsrep_cluster_size__': len(self._args.galera_ip_list),
-                         '__wsrep_inc_offset__': self._args.openstack_index*100,
-                        }
-        self._template_substitute_write(wsrep_template, template_vals,
-                                self._temp_dir_name + '/%s' % wsrep_conf_file)
-        local("sudo mv %s/%s %s" % (self._temp_dir_name, wsrep_conf_file,
-                                    wsrep_conf))
-        if self._args.openstack_index == 1:
-            local('sed -ibak "s#wsrep_cluster_address=.*#wsrep_cluster_address=gcomm://#g" %s' % (wsrep_conf))
+    def fix_cmon_config(self):
+        template_vals = {'__internal_vip__' : self._args.internal_vip,
+                         '__haproxy_dips__' : '"' + '" "'.join(self._args.galera_ip_list) + '"'}
+        self._template_substitute_write(cmon_param_template.template,
+                                        template_vals,
+                                        self._temp_dir_name + '/cmon_param')
+        local("sudo mv %s/cmon_param /etc/contrail/ha/" % (self._temp_dir_name))
 
         # fixup cmon config
         template_vals = {'__mysql_nodes__' : ','.join(self._args.galera_ip_list),
@@ -153,6 +139,32 @@ def fixup_config_files(self):
                                         self._temp_dir_name + '/cmon.cnf')
         local("sudo mv %s/cmon.cnf /etc/cmon.cnf" % (self._temp_dir_name))
 
+    def fixup_config_files(self):
+        with settings(warn_only=True):
+            local("service contrail-hamon stop")
+            local("service cmon stop")
+            local("service mysql stop")
+            local("rm -rf /var/lib/mysql/grastate.dat")
+            local("rm -rf /var/lib/mysql/galera.cache")
+            self.cleanup_redo_log()
+
+        if self._args.openstack_index == 1:
+            self.create_mysql_token_file()
+        else:
+            self.get_mysql_token_file()
+
+        self.fix_galera_config()
+        if self._args.openstack_index == 1:
+            local('sed -ibak "s#wsrep_cluster_address=.*#wsrep_cluster_address=gcomm://#g" %s' % (self.wsrep_conf))
+
+        self.fix_cmon_config()
+        local("echo %s > /etc/contrail/galeraid" % self._args.openstack_index)
+        self.install_mysql_db()
+        self.set_mysql_root_password()
+        ip_list = self._args.galera_ip_list + ['localhost', '127.0.0.1']
+        self.setup_grants(ip_list)
+        self.setup_cron()
+
     def install_mysql_db(self):
         local('chkconfig %s on' % self.mysql_svc)
         local('chown -R mysql:mysql /var/lib/mysql/')
@@ -202,10 +214,9 @@ def set_mysql_root_password(self):
             else:
                 raise RuntimeError("MySQL root password unknown, reset and retry")
 
-    def setup_grants(self):
+    def setup_grants(self, ip_list):
         mysql_cmd =  "mysql --defaults-file=%s -uroot -p%s -e" % (self.mysql_conf, self.mysql_token)
-        host_list = self._args.galera_ip_list + ['localhost', '127.0.0.1']
-        for host in host_list:
+        for host in ip_list:
             with settings(hide('everything'),warn_only=True):
                  local('mysql -u root -p%s -e "CREATE USER \'root\'@\'%s\' IDENTIFIED BY %s"' % (self.mysql_token, host, self.mysql_token))
             local('%s "SET WSREP_ON=0;SET SQL_LOG_BIN=0; GRANT ALL ON *.* TO root@%s IDENTIFIED BY \'%s\'"' %
@@ -220,23 +231,61 @@ def setup_cron(self):
         local('crontab %s/galera_cron' % self._temp_dir_name)
         local('rm %s/galera_cron' % self._temp_dir_name)
 
+    def add_new_mysql_perm(self):
+        # Before adding the new node into the cluster
+        # we should add the new node's user as an
+        # authorized user in the other servers.
+        self.get_mysql_token_file()
+        self.setup_grants(self._args.node_to_add.split())
+
+    def verify_mysql_server_status(self, ip, mysql_token):
+        cmd = "mysql -h%s -uroot -p%s " % (ip, mysql_token)
+        cmd += "-e \"show global status where variable_name='wsrep_local_state'\" | awk '{print $2}' | sed '1d'"
+        for i in range(10):
+            wsrep_local_state = local(cmd, capture=True).strip()
+            if wsrep_local_state == '4':
+                return True
+            else:
+                time.sleep(12)
+                print "Waiting for first galera node to create new cluster."
+            if wsrep_local_state != '4':
+                return False
+        return False
+
+    def restart_mysql_server(self):
+        # Restart the local MySQL server and wait
+        # for 10 seconds for it to restart.
+        local("service %s restart" % self.mysql_svc)
+        time.sleep(10)
+        wsrep_state_result = self.verify_mysql_server_status(self._args.self_ip, self.mysql_token)
+        if wsrep_state_result == False:
+            raise RuntimeError("Unable able to bring up galera in %s" % self._args.self_ip)
+
+    def add_new_cluster_config(self):
+        with settings(warn_only=True):
+            local("service contrail-hamon stop")
+            local("service cmon stop")
+        self.get_mysql_token_file()
+        self.fix_galera_config(bootstrap=False)
+        self.fix_cmon_config()
+        self.restart_mysql_server()
+        wsrep_state_result = self.verify_mysql_server_status(self._args.self_ip, self.mysql_token)
+        if wsrep_state_result == False:
+            raise RuntimeError("Unable able to bring up galera in %s" % self._args.self_ip)
+        local("sudo update-rc.d -f mysql remove")
+        local("sudo update-rc.d mysql defaults")
+        local("service contrail-hamon start")
+
     def run_services(self):
         self.cleanup_redo_log()
         if self._args.openstack_index == 1:
             local("service %s restart" % self.mysql_svc)
         else:
-            cmd = "mysql -h%s -uroot -p%s " % (self._args.galera_ip_list[0], self.mysql_token)
-            cmd += "-e \"show global status where variable_name='wsrep_local_state'\" | awk '{print $2}' | sed '1d'"
-            for i in range(10):
-                wsrep_local_state = local(cmd, capture=True).strip()
-                if wsrep_local_state == '4':
-                    break
-                else:
-                    time.sleep(2)
-                    print "Waiting for first galera node to create new cluster."
-            if wsrep_local_state != '4':
+            wsrep_state_result = self.verify_mysql_server_status(self._args.galera_ip_list[0], self.mysql_token)
+            if wsrep_state_result == False:
                 raise RuntimeError("Unable able to bring up galera in first node, please verify and continue.")
             local("service %s restart" % self.mysql_svc)
+
         local("sudo update-rc.d -f mysql remove")
         local("sudo update-rc.d mysql defaults")
 
@@ -253,5 +302,13 @@ def main(args_str = None):
     galera = GaleraSetup(args_str)
     galera.setup()
 
+def add_mysql_perm(args_str=None):
+    galera = GaleraSetup(args_str)
+    galera.add_new_mysql_perm()
+
+def add_galera_cluster_config(args_str=None):
+    galera = GaleraSetup(args_str)
+    galera.add_new_cluster_config()
+
 if __name__ == "__main__":
     main()