Partial-Bug: #1413364

1)Generate config for mysql and nova similar to fab. 2)Run DB sync only on one node. 3)Handle db sync for cinder. 4)Make check-wsrep-status to re-cluster when galera enters a bad state. 5)Mysql was connecting to 3306 instead of 33306. Change-Id: I1231ef121b846fb7b35202ab548bdc37b861a1cd
Juniper · Feb 10, 2015 · 89ee946 · 89ee946
1 parent 36132f4
commit 89ee946
Show file tree

Hide file tree

Showing 29 changed files with 329 additions and 95 deletions.
diff --git a/contrail/environment/modules/cinder/manifests/api.pp b/contrail/environment/modules/cinder/manifests/api.pp
@@ -124,15 +124,29 @@
   if $enabled {
 
     Cinder_config<||> ~> Exec['cinder-manage db_sync']
+    $sync_db = $::contrail::params::sync_db
 
-    exec { 'cinder-manage db_sync':
-      command     => $::cinder::params::db_sync_command,
-      path        => '/usr/bin',
-      user        => 'cinder',
-      refreshonly => true,
-      logoutput   => 'on_failure',
-      require     => Package['cinder'],
+    if ($sync_db)
+    {
+      exec { 'cinder-manage db_sync':
+        command     => $::cinder::params::db_sync_command,
+        path        => '/usr/bin',
+        user        => 'cinder',
+        refreshonly => true,
+        logoutput   => 'on_failure',
+        require     => Package['cinder'],
+      }
+    } else {
+      exec { 'cinder-manage db_sync':
+        command     => "touch /tmp/cinder_db_sync.1",
+        path        => '/usr/bin',
+        user        => 'cinder',
+        refreshonly => true,
+        logoutput   => 'on_failure',
+        require     => Package['cinder'],
+      }
     }
+
     if $manage_service {
       $ensure = 'running'
     }

diff --git a/contrail/environment/modules/cinder/manifests/db/sync.pp b/contrail/environment/modules/cinder/manifests/db/sync.pp
@@ -2,13 +2,26 @@
 class cinder::db::sync {
 
   include cinder::params
+  $sync_db = $::contrail::params::sync_db
 
-  exec { 'cinder-manage db_sync':
-    command     => $::cinder::params::db_sync_command,
-    path        => '/usr/bin',
-    user        => 'cinder',
-    refreshonly => true,
-    require     => [File[$::cinder::params::cinder_conf], Class['cinder']],
-    logoutput   => 'on_failure',
+
+  if( $sync_db) {
+    exec { 'cinder-manage db_sync':
+      command     => $::cinder::params::db_sync_command,
+      path        => '/usr/bin',
+      user        => 'cinder',
+      refreshonly => true,
+      require     => [File[$::cinder::params::cinder_conf], Class['cinder']],
+      logoutput   => 'on_failure',
+    }
+  } else {
+    exec { 'cinder-manage db_sync':
+      command     => "touch /tmp/cinder_db_sync",
+      path        => '/usr/bin',
+      user        => 'cinder',
+      refreshonly => true,
+      require     => [File[$::cinder::params::cinder_conf], Class['cinder']],
+      logoutput   => 'on_failure',
+    }
   }
 }
diff --git a/contrail/environment/modules/contrail/files/check-mysql-status.py b/contrail/environment/modules/contrail/files/check-mysql-status.py
@@ -0,0 +1,23 @@
+import commands
+import sys
+import os.path
+
+def main(args_str=None):
+
+    status,output = commands.getstatusoutput("cat /etc/contrail/mysql.token")
+    mysql_token = output
+    status,output = commands.getstatusoutput('service mysql status')
+
+#    if status != 0:
+#        sys.exit(0)
+
+#   If we are not able to connect to mysql,its probably stucik , kill it!
+    status,output = commands.getstatusoutput('mysql -uroot -p%s -e "show status like \'wsrep_cluster_size\'"' %  mysql_token )
+    print "wsrep_cluster_size: %s" % output
+    #if output.find("4") == -1:
+    if status != 0:
+        status,output = commands.getstatusoutput('pkill -9 mysql')
+
+if __name__ == "__main__":
+         main(sys.argv[1:])
+
diff --git a/contrail/environment/modules/contrail/files/check-wsrep-status.py b/contrail/environment/modules/contrail/files/check-wsrep-status.py
@@ -22,11 +22,15 @@ def main(args_str=None):
     print "wsrep_cluster_size: %s" % output
     #if output.find("4") == -1:
     if output.find(str(number_openstack_nodes)) == -1:
+        uuid, output = commands.getstatusoutput("cat /var/lib/mysql/grastate.dat | grep uuid | awk '{print $2;}'")
+        commands.getstatusoutput("service mysql restart")
+
         for os_ip in os_ip_list:
-          if not os.path.exists("/etc/clear_mysql"):
-              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "pkill -9 mysql"' % (os_ip))
+#          if not os.path.exists("/etc/clear_mysql"):
+              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "rm -rf /var/lib/mysql/grastate.dat"' % (os_ip))
+              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "service mysql restart"' % (os_ip))
               print "cleaning mysql in %s" % (output)
-              opens("/etc/clear_mysql", 'a').close() 
+              open("/etc/clear_mysql", 'a').close() 
 
         sys.exit(1)
 if __name__ == "__main__":

diff --git a/contrail/environment/modules/contrail/files/fix-cmon-params-and-add-ssh-keys.py b/contrail/environment/modules/contrail/files/fix-cmon-params-and-add-ssh-keys.py
@@ -7,15 +7,6 @@
 import paramiko
 import os.path
 
-def create_ssh_keys():
-    if not os.path.isfile('/root/.ssh/id_rsa') and not os.path.isfile('/root/.ssh/id_rsa.pub'):
-        commands.getstatusoutput('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
-    elif not os.path.isfile('/root/.ssh/id_rsa') or not os.path.isfile('/root/.ssh/id_rsa.pub'):
-        commands.getstatusoutput('rm -rf /root/.ssh/id_rsa*')
-        commands.getstatusoutput('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
-
-
-
 def main(args_str=None):
     compute_host_list_str = sys.argv[1]
     config_host_list_str = sys.argv[2]
@@ -31,31 +22,14 @@ def main(args_str=None):
 
     computes = 'COMPUTES=("' + '" "'.join(compute_host_list) + '")'
     commands.getstatusoutput("echo '%s' >> %s" % (computes, cmon_param))
-    commands.getstatusoutput("echo 'COMPUTES_SIZE=%s' >> %s" % (compute_sz, cmon_param))
+    commands.getstatusoutput("echo 'COMPUTES_SIZE=%s' >> %s" % ("${#COMPUTES[@]}", cmon_param))
     commands.getstatusoutput("echo 'COMPUTES_USER=root' >> %s" % cmon_param)
     commands.getstatusoutput("echo 'PERIODIC_RMQ_CHK_INTER=60' >> %s" % cmon_param)
+    commands.getstatusoutput("echo 'RABBITMQ_RESET=True' >> %s" % cmon_param)
     amqps = 'DIPHOSTS=("' + '" "'.join(amqp_host_list) + '")'
     commands.getstatusoutput("echo '%s' >> %s" % (amqps, cmon_param))
-    commands.getstatusoutput("echo 'DIPS_HOST_SIZE=%s' >> %s" % (amqp_sz, cmon_param))
-
+    commands.getstatusoutput("echo 'DIPS_HOST_SIZE=%s' >> %s" % ("${#DIPHOSTS[@]}", cmon_param))
 
-    #Copy the ssh keys of openstack to every compute
-    create_ssh_keys()
-    status,output = commands.getstatusoutput("cat /root/.ssh/id_rsa.pub")
-    publick_key = output
-    port = 22
-    username = "root"
-    password = "c0ntrail123"
-    for compute_host in compute_host_list:
-        s = paramiko.SSHClient()
-        s.load_system_host_keys()
-        s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        s.connect(compute_host, port, username, password)
-        command = "mkdir -p /root/ssh/"
-        s.exec_command(command)
-        command = "echo %s > /root/.ssh/authorized_keys" % output
-        s.exec_command(command)
-        s.close()
 
 if __name__ == "__main__":
     main(sys.argv[1:])       
diff --git a/contrail/environment/modules/contrail/files/fix-mem-cache.py b/contrail/environment/modules/contrail/files/fix-mem-cache.py
@@ -43,6 +43,10 @@ def main(args_str=None):
         commands.getstatusoutput('echo "net.ipv4.tcp_tw_reuse = 1" >> /etc/sysctl.conf')
     if commands.getstatusoutput("grep '^net.ipv4.tcp_fin_timeout' /etc/sysctl.conf")[0] != 0:
         commands.getstatusoutput('echo "net.ipv4.tcp_fin_timeout = 30" >> /etc/sysctl.conf')
+    if commands.getstatusoutput("grep '^net.unix.max_dgram_qlen' /etc/sysctl.conf")[0] != 0:
+        commands.getstatusoutput('echo "net.unix.max_dgram_qlen  = 1000" >> /etc/sysctl.conf')
+
+    commands.getstatusoutput('sysctl -p')
 
 if __name__ == "__main__":
      main(sys.argv[1:])
diff --git a/contrail/environment/modules/contrail/files/set_rabbit_tcp_params.py b/contrail/environment/modules/contrail/files/set_rabbit_tcp_params.py
@@ -20,7 +20,10 @@ def main(args_str=None):
         commands.getstatusoutput("sed -i 's/net.ipv4.tcp_keepalive_intvl\s\s*/net.ipv4.tcp_keepalive_intvl = 1/' /etc/sysctl.conf")
 
 
+    status, output = commands.getstatusoutput("sysctl -p")
 
+    if status != 0:
+        sys.exit(-1)
 
 if __name__ == "__main__":
      main(sys.argv[1:])
diff --git a/contrail/environment/modules/contrail/manifests/common.pp b/contrail/environment/modules/contrail/manifests/common.pp
@@ -135,6 +135,25 @@
 	}
     }
 
+##HACk for mysql on openstack-ha
+  if ($::contrail::params::internal_vip != "") {
+    file { "/opt/check-mysql-status.py" :
+	ensure  => present,
+	mode => 0755,
+	group => root,
+	source => "puppet:///modules/$module_name/check-mysql-status.py"
+    }
+    ->
+    exec { "exec_check_mysql" :
+	command => "python /opt/check-mysql-status.py",  
+	cwd => "/opt/",
+#	unless  => "grep -qx exec_check_mysql /etc/contrail/contrail_common_exec.out",
+	provider => shell,
+	require => [ File["/opt/check-mysql-status.py"] ],
+	logoutput => 'true',
+    }
+}
+
     # Core pattern
     exec { 'core_pattern_1':
 	command   => 'echo \'kernel.core_pattern = /var/crashes/core.%e.%p.%h.%t\' >> /etc/sysctl.conf',

diff --git a/contrail/environment/modules/contrail/manifests/ha_config.pp b/contrail/environment/modules/contrail/manifests/ha_config.pp
@@ -50,10 +50,12 @@
     $config_passwd_list = $::contrail::params::config_passwd_list,
     $compute_passwd_list = $::contrail::params::compute_passwd_list,
     $openstack_user_list = $::contrail::params::openstack_user_list,
-    $keystone_ip = $::contrail::params::keystone_ip
+    $keystone_ip = $::contrail::params::keystone_ip,
+    $nfs_server = $::contrail::params::nfs_server,
+    $nfs_glance_path = $::contrail::params::nfs_glance_path,
 ) inherits ::contrail::params {
     # Main code for class
-    if($internal_vip != '') {
+    if($internal_vip != '' and $host_control_ip in $openstack_ip_list) {
         if ($operatingsystem == "Ubuntu") {
             $wsrep_conf='/etc/mysql/conf.d/wsrep.cnf'
         } else {
@@ -75,9 +77,23 @@
         $os_username = $openstack_user_list[0]
 	$os_passwd = $openstack_passwd_list[0]
 
-        $glance_path ="/var/lib/glance/images"
+        if ($nfs_server != "" and nfs_server != undef) {
 
-	$nfs_server = $openstack_ip_list[0]
+	   $contrail_nfs_server = $nfs_server
+
+        } else {
+	   $contrail_nfs_server = $openstack_ip_list[0]
+        }
+
+        if ($nfs_glance_path != "" and nfs_glance_path != undef) {
+
+	   $contrail_nfs_glance_path = $nfs_glance_path
+
+        } else {
+	   $contrail_nfs_glance_path = "/var/lib/glance/images"
+        }
+
+
 
         $openstack_mgmt_ip_list_shell = inline_template('<%= @openstack_mgmt_ip_list.map{ |ip| "#{ip}" }.join(",") %>')
         $openstack_ip_list_shell = inline_template('<%= @openstack_ip_list.map{ |name2| "#{name2}" }.join(" ") %>')
@@ -190,6 +206,19 @@
                 logoutput => 'true',
             }
             ->
+            exec { "ha-mon-restart":
+                command => "service contrail-hamon restart && echo contrail-ha-mon >> /etc/contrail/contrail_openstack_exec.out",
+                provider => shell,
+                logoutput => "true",
+                unless  => "grep -qx contrail-ha-mon  /etc/contrail/contrail_openstack_exec.out",
+        #                require => File["/opt/contrail/bin/transfer_keys.py"]
+            }
+
+
+
+        }
+        #This will be skipped if there is an external nfs server
+        if ($contrail_nfs_server == $host_control_ip) {
 	    package { 'nfs-kernel-server':
 		ensure  => present,
 	    }
@@ -203,6 +232,7 @@
             }
 
         }
+
         # setup_cmon
         file { "/opt/contrail/bin/setup-cmon-schema.py" :
             ensure  => present,
@@ -219,6 +249,7 @@
             require => [ File["/opt/contrail/bin/setup-cmon-schema.py"] ],
             logoutput => 'true',
         }
+/*
         ->
         exec { "setup-cluster-monitor" :
             command => "service contrail-hamon restart && chkconfig contrail-hamon on  && echo setup-cluster-monitor >> /etc/contrail/contrail_openstack_exec.out ",
@@ -228,6 +259,7 @@
             tries => 3,
             try_sleep => 15,
         }
+*/
         ->
         exec { "fix_xinetd_conf" :
             command => "sed -i -e 's#only_from = 0.0.0.0/0#only_from = $host_control_ip 127.0.0.1#' /etc/xinetd.d/contrail-mysqlprobe && service xinetd restart && chkconfig xinetd on && echo fix_xinetd_conf >> /etc/contrail/contrail_openstack_exec.out",
@@ -284,30 +316,22 @@
                 unless  => "grep -qx exec-transfer-keys  /etc/contrail/contrail_openstack_exec.out",
                 require => File["/opt/contrail/bin/transfer_keys.py"]
         }
-        ->
-        exec { "ha-mon-restart":
-                command => "service contrail-hamon restart && echo contrail-ha-mon >> /etc/contrail/contrail_openstack_exec.out",
-                provider => shell,
-                logoutput => "true",
-                unless  => "grep -qx contrail-ha-mon  /etc/contrail/contrail_openstack_exec.out",
-#                require => File["/opt/contrail/bin/transfer_keys.py"]
-        }
-
-        if ($openstack_index != "1" ) {
+        #This wil be executed for all openstacks ,if there is an external nfs server
+        if ($contrail_nfs_server != $host_control_ip ) {
 	    package { 'nfs-common':
 		ensure  => present,
 	    }
             ->
 	    exec { "mount-nfs" :
-		command => "sudo mount $nfs_server:$glance_path /var/lib/glance/images && echo mount-nfs >> /etc/contrail/contrail_openstack_exec.out",
+		command => "sudo mount $contrail_nfs_server:$contrail_nfs_glance_path /var/lib/glance/images && echo mount-nfs >> /etc/contrail/contrail_openstack_exec.out",
 		require => [  ],
 		unless  => "grep -qx mount-nfs  /etc/contrail/contrail_openstack_exec.out",
 		provider => shell,
 		logoutput => "true"
 	    }
 	    exec { "add-fstab" :
-		command => "echo \"$nfs_server:$glance_path /var/lib/glance/images nfs nfsvers=3,hard,intr,auto 0 0\" >> /etc/fstab && echo add-fstab >> /etc/contrail/contrail_openstack_exec.out ",
-		unless  => "grep -qx add-fstab  /etc/contrail/contrail_oprenstack_exec.out",
+		command => "echo \"$contrail_nfs_server:$contrail_nfs_glance_path /var/lib/glance/images nfs nfsvers=3,hard,intr,auto 0 0\" >> /etc/fstab && echo add-fstab >> /etc/contrail/contrail_openstack_exec.out ",
+		unless  => "grep -qx add-fstab  /etc/contrail/contrail_openstack_exec.out",
 		provider => shell,
 		logoutput => "true"
 	    }

diff --git a/contrail/environment/modules/contrail/manifests/keepalived.pp b/contrail/environment/modules/contrail/manifests/keepalived.pp
@@ -56,6 +56,7 @@
     }
 
     if ($vip != "") {
+        $num_nodes = inline_template('<%= @ip_list.length %>')
         $tmp_index = inline_template('<%= @ip_list.index(@host_control_ip) %>')
         if ($tmp_index == nil) {
             fail("Host $host_control_ip not found in servers of config roles")
@@ -64,11 +65,20 @@
 
         $config_index = $tmp_index + 1
         $keepalived_priority = $keepalived_vrid - $config_index
-        if ($config_index == 1) {
+        if ($config_index == 1 ) {
             $keepalived_state = "MASTER"
+            $contrail_garp_master_delay = 5
+            $contrail_preempt_delay = 7
+        } 
+        elsif ($config_index ==2 and $num_nodes > 2 ) {
+            $keepalived_state = "MASTER"
+            $contrail_garp_master_delay = 1
+            $contrail_preempt_delay = 1
         }
         else {
             $keepalived_state = "BACKUP"
+            $contrail_garp_master_delay = 1
+            $contrail_preempt_delay = 1
         }
 
 	include ::keepalived
@@ -100,6 +110,8 @@
 	  virtual_ipaddress => $vip,
           garp_master_refresh => 1,
           garp_master_repeat => 3,
+          garp_master_delay => $contrail_garp_master_delay,
+          preempt_delay => $contrail_preempt_delay,
           vmac_xmit_base => true,
           track_interface => $control_data_intf,
 	  track_script      => ['check_haproxy','check_peers'],

diff --git a/contrail/environment/modules/contrail/manifests/params.pp b/contrail/environment/modules/contrail/manifests/params.pp
@@ -510,6 +510,7 @@
     $compute_passwd_list,
     $host_roles = "",
     $external_bgp = "",
+    $sync_db = "",
     $contrail_plugin_location  = "NEUTRON_PLUGIN_CONFIG=\'/etc/neutron/plugins/opencontrail/ContrailPlugin.ini\'"
 ) {
     # Manifests use keystone_admin_token to refer to keystone_service_token too. Hence set

diff --git a/contrail/environment/modules/contrail/manifests/profile/openstack/glance/api.pp b/contrail/environment/modules/contrail/manifests/profile/openstack/glance/api.pp
@@ -4,7 +4,8 @@
 class contrail::profile::openstack::glance::api {
   $api_network = $::openstack::config::network_api
   $api_address = ip_for_network($api_network)
-  $sync_db = true
+ # $sync_db = true
+  $sync_db = $::contrail::params::sync_db
   $management_network = $::openstack::config::network_management
   $management_address = ip_for_network($management_network)