Closes-Bug: #1413364

Server Manager HA, configuration for mysql , nova , haproxy were different from fab provisioning

Mysql gets into error state due to locks.

Deviced a recover logic for that.

Change-Id: Ibb7463356ed0683ef05c2dc57e51c9a8fe0c82d9

contrail/environment/modules/contrail/files/check-wsrep-status.py

@@ -1,6 +1,6 @@
 import commands
 import sys
-
+import os.path
 
 def main(args_str=None):
     openstack_ip_list_str = sys.argv[1]
@@ -22,6 +22,12 @@ def main(args_str=None):
     print "wsrep_cluster_size: %s" % output
     #if output.find("4") == -1:
     if output.find(str(number_openstack_nodes)) == -1:
+        for os_ip in os_ip_list:
+          if not os.path.exists("/etc/clear_mysql"):
+              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "pkill -9 mysql"' % (os_ip))
+              print "cleaning mysql in %s" % (output)
+              opens("/etc/clear_mysql", 'a').close() 
+
         sys.exit(1)
 if __name__ == "__main__":
          main(sys.argv[1:])

contrail/environment/modules/contrail/manifests/ha_config.pp

@@ -162,6 +162,23 @@
 		require => [ File["/opt/contrail/bin/check_galera.py"] ],
 		logoutput => 'true',
 	    }
+            ->
+	    file { "/opt/contrail/bin/check-wsrep-status.py" :
+		ensure  => present,
+		mode => 0755,
+		group => root,
+		source => "puppet:///modules/$module_name/check-wsrep-status.py"
+	    }
+	    ->
+	    exec { "exec_check_wsrep" :
+		command => $contrail_exec_check_wsrep,  
+		cwd => "/opt/contrail/bin/",
+		unless  => "grep -qx exec_check_wsrep /etc/contrail/contrail_openstack_exec.out",
+		provider => shell,
+		require => [ File["/opt/contrail/bin/check-wsrep-status.py"] ],
+		logoutput => 'true',
+	    }
+
 	    ->
             exec { "fix_wsrep_cluster_address" :
                 command => "sudo sed -ibak 's#wsrep_cluster_address=.*#wsrep_cluster_address=gcomm://$openstack_ip_list_wsrep#g' $wsrep_conf && service mysql restart && echo exec_fix_wsrep_cluster_address >> /etc/contrail/contrail_openstack_exec.out",
@@ -267,6 +284,15 @@
                 unless  => "grep -qx exec-transfer-keys  /etc/contrail/contrail_openstack_exec.out",
                 require => File["/opt/contrail/bin/transfer_keys.py"]
         }
+        ->
+        exec { "ha-mon-restart":
+                command => "service contrail-hamon restart && echo contrail-ha-mon >> /etc/contrail/contrail_openstack_exec.out",
+                provider => shell,
+                logoutput => "true",
+                unless  => "grep -qx contrail-ha-mon  /etc/contrail/contrail_openstack_exec.out",
+#                require => File["/opt/contrail/bin/transfer_keys.py"]
+        }
+
         if ($openstack_index != "1" ) {
 	    package { 'nfs-common':
 		ensure  => present,