From 89ee946c0cb057761e4a164d600eb20b34ff6840 Mon Sep 17 00:00:00 2001
From: tsurendra <tsurendra@juniper.net>
Date: Thu, 5 Feb 2015 14:17:48 -0800
Subject: [PATCH] Partial-Bug: #1413364

1)Generate config for mysql and nova similar to fab.
2)Run DB sync only on one node.
3)Handle db sync for cinder.
4)Make check-wsrep-status to re-cluster when galera enters a bad state.
5)Mysql was connecting to 3306 instead of 33306.

Change-Id: I1231ef121b846fb7b35202ab548bdc37b861a1cd
---
 .../modules/cinder/manifests/api.pp           | 28 ++++++---
 .../modules/cinder/manifests/db/sync.pp       | 27 ++++++---
 .../contrail/files/check-mysql-status.py      | 23 ++++++++
 .../contrail/files/check-wsrep-status.py      | 10 +++-
 .../files/fix-cmon-params-and-add-ssh-keys.py | 32 +---------
 .../modules/contrail/files/fix-mem-cache.py   |  4 ++
 .../contrail/files/set_rabbit_tcp_params.py   |  3 +
 .../modules/contrail/manifests/common.pp      | 19 ++++++
 .../modules/contrail/manifests/ha_config.pp   | 58 +++++++++++++------
 .../modules/contrail/manifests/keepalived.pp  | 14 ++++-
 .../modules/contrail/manifests/params.pp      |  1 +
 .../manifests/profile/openstack/glance/api.pp |  3 +-
 .../manifests/profile/openstack_controller.pp |  5 ++
 .../contrail/manifests/provision_complete.pp  | 21 ++++++-
 .../horizon/templates/local_settings.py.erb   | 21 +++++++
 .../keepalived/templates/vrrp_instance.erb    |  3 -
 .../keepalived/templates/vrrp_script.erb      |  2 +-
 .../modules/mysql/manifests/params.pp         | 12 ++--
 .../modules/mysql/templates/my.cnf.erb        |  5 ++
 .../openstack/manifests/common/cinder.pp      |  1 +
 .../manifests/common/contrail/nova.pp         | 17 +++++-
 .../openstack/manifests/common/glance.pp      | 20 +++++++
 .../openstack/manifests/common/keystone.pp    | 26 ++++++++-
 .../openstack/manifests/common/neutron.pp     |  2 +-
 .../openstack/manifests/common/nova.pp        | 41 +++++++++----
 .../manifests/profile/contrail/glance/api.pp  |  2 +-
 .../openstack/manifests/profile/glance/api.pp |  2 +
 .../openstack/manifests/profile/horizon.pp    | 12 ++++
 .../manifests/resources/connectors.pp         | 10 +++-
 29 files changed, 329 insertions(+), 95 deletions(-)
 create mode 100644 contrail/environment/modules/contrail/files/check-mysql-status.py

diff --git a/contrail/environment/modules/cinder/manifests/api.pp b/contrail/environment/modules/cinder/manifests/api.pp
index 479a4dcc..2e20661b 100644
--- a/contrail/environment/modules/cinder/manifests/api.pp
+++ b/contrail/environment/modules/cinder/manifests/api.pp
@@ -124,15 +124,29 @@
   if $enabled {
 
     Cinder_config<||> ~> Exec['cinder-manage db_sync']
+    $sync_db = $::contrail::params::sync_db
 
-    exec { 'cinder-manage db_sync':
-      command     => $::cinder::params::db_sync_command,
-      path        => '/usr/bin',
-      user        => 'cinder',
-      refreshonly => true,
-      logoutput   => 'on_failure',
-      require     => Package['cinder'],
+    if ($sync_db)
+    {
+      exec { 'cinder-manage db_sync':
+        command     => $::cinder::params::db_sync_command,
+        path        => '/usr/bin',
+        user        => 'cinder',
+        refreshonly => true,
+        logoutput   => 'on_failure',
+        require     => Package['cinder'],
+      }
+    } else {
+      exec { 'cinder-manage db_sync':
+        command     => "touch /tmp/cinder_db_sync.1",
+        path        => '/usr/bin',
+        user        => 'cinder',
+        refreshonly => true,
+        logoutput   => 'on_failure',
+        require     => Package['cinder'],
+      }
     }
+
     if $manage_service {
       $ensure = 'running'
     }
diff --git a/contrail/environment/modules/cinder/manifests/db/sync.pp b/contrail/environment/modules/cinder/manifests/db/sync.pp
index 942f2521..740df2cd 100644
--- a/contrail/environment/modules/cinder/manifests/db/sync.pp
+++ b/contrail/environment/modules/cinder/manifests/db/sync.pp
@@ -2,13 +2,26 @@
 class cinder::db::sync {
 
   include cinder::params
+  $sync_db = $::contrail::params::sync_db
 
-  exec { 'cinder-manage db_sync':
-    command     => $::cinder::params::db_sync_command,
-    path        => '/usr/bin',
-    user        => 'cinder',
-    refreshonly => true,
-    require     => [File[$::cinder::params::cinder_conf], Class['cinder']],
-    logoutput   => 'on_failure',
+
+  if( $sync_db) {
+    exec { 'cinder-manage db_sync':
+      command     => $::cinder::params::db_sync_command,
+      path        => '/usr/bin',
+      user        => 'cinder',
+      refreshonly => true,
+      require     => [File[$::cinder::params::cinder_conf], Class['cinder']],
+      logoutput   => 'on_failure',
+    }
+  } else {
+    exec { 'cinder-manage db_sync':
+      command     => "touch /tmp/cinder_db_sync",
+      path        => '/usr/bin',
+      user        => 'cinder',
+      refreshonly => true,
+      require     => [File[$::cinder::params::cinder_conf], Class['cinder']],
+      logoutput   => 'on_failure',
+    }
   }
 }
diff --git a/contrail/environment/modules/contrail/files/check-mysql-status.py b/contrail/environment/modules/contrail/files/check-mysql-status.py
new file mode 100644
index 00000000..b9943ded
--- /dev/null
+++ b/contrail/environment/modules/contrail/files/check-mysql-status.py
@@ -0,0 +1,23 @@
+import commands
+import sys
+import os.path
+
+def main(args_str=None):
+
+    status,output = commands.getstatusoutput("cat /etc/contrail/mysql.token")
+    mysql_token = output
+    status,output = commands.getstatusoutput('service mysql status')
+    
+#    if status != 0:
+#        sys.exit(0)
+
+#   If we are not able to connect to mysql,its probably stucik , kill it!
+    status,output = commands.getstatusoutput('mysql -uroot -p%s -e "show status like \'wsrep_cluster_size\'"' %  mysql_token )
+    print "wsrep_cluster_size: %s" % output
+    #if output.find("4") == -1:
+    if status != 0:
+        status,output = commands.getstatusoutput('pkill -9 mysql')
+
+if __name__ == "__main__":
+         main(sys.argv[1:])
+
diff --git a/contrail/environment/modules/contrail/files/check-wsrep-status.py b/contrail/environment/modules/contrail/files/check-wsrep-status.py
index 46648a5c..ebd6388e 100644
--- a/contrail/environment/modules/contrail/files/check-wsrep-status.py
+++ b/contrail/environment/modules/contrail/files/check-wsrep-status.py
@@ -22,11 +22,15 @@ def main(args_str=None):
     print "wsrep_cluster_size: %s" % output
     #if output.find("4") == -1:
     if output.find(str(number_openstack_nodes)) == -1:
+        uuid, output = commands.getstatusoutput("cat /var/lib/mysql/grastate.dat | grep uuid | awk '{print $2;}'")
+        commands.getstatusoutput("service mysql restart")
+      
         for os_ip in os_ip_list:
-          if not os.path.exists("/etc/clear_mysql"):
-              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "pkill -9 mysql"' % (os_ip))
+#          if not os.path.exists("/etc/clear_mysql"):
+              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "rm -rf /var/lib/mysql/grastate.dat"' % (os_ip))
+              status,output = commands.getstatusoutput('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s "service mysql restart"' % (os_ip))
               print "cleaning mysql in %s" % (output)
-              opens("/etc/clear_mysql", 'a').close() 
+              open("/etc/clear_mysql", 'a').close() 
  
         sys.exit(1)
 if __name__ == "__main__":
diff --git a/contrail/environment/modules/contrail/files/fix-cmon-params-and-add-ssh-keys.py b/contrail/environment/modules/contrail/files/fix-cmon-params-and-add-ssh-keys.py
index d33c82da..2d489b03 100644
--- a/contrail/environment/modules/contrail/files/fix-cmon-params-and-add-ssh-keys.py
+++ b/contrail/environment/modules/contrail/files/fix-cmon-params-and-add-ssh-keys.py
@@ -7,15 +7,6 @@
 import paramiko
 import os.path
 
-def create_ssh_keys():
-    if not os.path.isfile('/root/.ssh/id_rsa') and not os.path.isfile('/root/.ssh/id_rsa.pub'):
-        commands.getstatusoutput('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
-    elif not os.path.isfile('/root/.ssh/id_rsa') or not os.path.isfile('/root/.ssh/id_rsa.pub'):
-        commands.getstatusoutput('rm -rf /root/.ssh/id_rsa*')
-        commands.getstatusoutput('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
-
-
-
 def main(args_str=None):
     compute_host_list_str = sys.argv[1]
     config_host_list_str = sys.argv[2]
@@ -31,31 +22,14 @@ def main(args_str=None):
 
     computes = 'COMPUTES=("' + '" "'.join(compute_host_list) + '")'
     commands.getstatusoutput("echo '%s' >> %s" % (computes, cmon_param))
-    commands.getstatusoutput("echo 'COMPUTES_SIZE=%s' >> %s" % (compute_sz, cmon_param))
+    commands.getstatusoutput("echo 'COMPUTES_SIZE=%s' >> %s" % ("${#COMPUTES[@]}", cmon_param))
     commands.getstatusoutput("echo 'COMPUTES_USER=root' >> %s" % cmon_param)
     commands.getstatusoutput("echo 'PERIODIC_RMQ_CHK_INTER=60' >> %s" % cmon_param)
+    commands.getstatusoutput("echo 'RABBITMQ_RESET=True' >> %s" % cmon_param)
     amqps = 'DIPHOSTS=("' + '" "'.join(amqp_host_list) + '")'
     commands.getstatusoutput("echo '%s' >> %s" % (amqps, cmon_param))
-    commands.getstatusoutput("echo 'DIPS_HOST_SIZE=%s' >> %s" % (amqp_sz, cmon_param))
-
+    commands.getstatusoutput("echo 'DIPS_HOST_SIZE=%s' >> %s" % ("${#DIPHOSTS[@]}", cmon_param))
 
-    #Copy the ssh keys of openstack to every compute
-    create_ssh_keys()
-    status,output = commands.getstatusoutput("cat /root/.ssh/id_rsa.pub")
-    publick_key = output
-    port = 22
-    username = "root"
-    password = "c0ntrail123"
-    for compute_host in compute_host_list:
-        s = paramiko.SSHClient()
-        s.load_system_host_keys()
-        s.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        s.connect(compute_host, port, username, password)
-        command = "mkdir -p /root/ssh/"
-        s.exec_command(command)
-        command = "echo %s > /root/.ssh/authorized_keys" % output
-        s.exec_command(command)
-        s.close()
 
 if __name__ == "__main__":
     main(sys.argv[1:])       
diff --git a/contrail/environment/modules/contrail/files/fix-mem-cache.py b/contrail/environment/modules/contrail/files/fix-mem-cache.py
index a2d1c5e7..c0a3c0b5 100755
--- a/contrail/environment/modules/contrail/files/fix-mem-cache.py
+++ b/contrail/environment/modules/contrail/files/fix-mem-cache.py
@@ -43,6 +43,10 @@ def main(args_str=None):
         commands.getstatusoutput('echo "net.ipv4.tcp_tw_reuse = 1" >> /etc/sysctl.conf')
     if commands.getstatusoutput("grep '^net.ipv4.tcp_fin_timeout' /etc/sysctl.conf")[0] != 0:
         commands.getstatusoutput('echo "net.ipv4.tcp_fin_timeout = 30" >> /etc/sysctl.conf')
+    if commands.getstatusoutput("grep '^net.unix.max_dgram_qlen' /etc/sysctl.conf")[0] != 0:
+        commands.getstatusoutput('echo "net.unix.max_dgram_qlen  = 1000" >> /etc/sysctl.conf')
+
+    commands.getstatusoutput('sysctl -p')
 
 if __name__ == "__main__":
      main(sys.argv[1:])
diff --git a/contrail/environment/modules/contrail/files/set_rabbit_tcp_params.py b/contrail/environment/modules/contrail/files/set_rabbit_tcp_params.py
index cc919139..e9e813a5 100644
--- a/contrail/environment/modules/contrail/files/set_rabbit_tcp_params.py
+++ b/contrail/environment/modules/contrail/files/set_rabbit_tcp_params.py
@@ -20,7 +20,10 @@ def main(args_str=None):
         commands.getstatusoutput("sed -i 's/net.ipv4.tcp_keepalive_intvl\s\s*/net.ipv4.tcp_keepalive_intvl = 1/' /etc/sysctl.conf")
 
 
+    status, output = commands.getstatusoutput("sysctl -p")
 
+    if status != 0:
+        sys.exit(-1)
 
 if __name__ == "__main__":
      main(sys.argv[1:])
diff --git a/contrail/environment/modules/contrail/manifests/common.pp b/contrail/environment/modules/contrail/manifests/common.pp
index ad0d4ced..6ef0ec95 100644
--- a/contrail/environment/modules/contrail/manifests/common.pp
+++ b/contrail/environment/modules/contrail/manifests/common.pp
@@ -135,6 +135,25 @@
 	}
     }
 
+##HACk for mysql on openstack-ha
+  if ($::contrail::params::internal_vip != "") {
+    file { "/opt/check-mysql-status.py" :
+	ensure  => present,
+	mode => 0755,
+	group => root,
+	source => "puppet:///modules/$module_name/check-mysql-status.py"
+    }
+    ->
+    exec { "exec_check_mysql" :
+	command => "python /opt/check-mysql-status.py",  
+	cwd => "/opt/",
+#	unless  => "grep -qx exec_check_mysql /etc/contrail/contrail_common_exec.out",
+	provider => shell,
+	require => [ File["/opt/check-mysql-status.py"] ],
+	logoutput => 'true',
+    }
+}
+
     # Core pattern
     exec { 'core_pattern_1':
 	command   => 'echo \'kernel.core_pattern = /var/crashes/core.%e.%p.%h.%t\' >> /etc/sysctl.conf',
diff --git a/contrail/environment/modules/contrail/manifests/ha_config.pp b/contrail/environment/modules/contrail/manifests/ha_config.pp
index b9772950..a9402d55 100644
--- a/contrail/environment/modules/contrail/manifests/ha_config.pp
+++ b/contrail/environment/modules/contrail/manifests/ha_config.pp
@@ -50,10 +50,12 @@
     $config_passwd_list = $::contrail::params::config_passwd_list,
     $compute_passwd_list = $::contrail::params::compute_passwd_list,
     $openstack_user_list = $::contrail::params::openstack_user_list,
-    $keystone_ip = $::contrail::params::keystone_ip
+    $keystone_ip = $::contrail::params::keystone_ip,
+    $nfs_server = $::contrail::params::nfs_server,
+    $nfs_glance_path = $::contrail::params::nfs_glance_path,
 ) inherits ::contrail::params {
     # Main code for class
-    if($internal_vip != '') {
+    if($internal_vip != '' and $host_control_ip in $openstack_ip_list) {
         if ($operatingsystem == "Ubuntu") {
             $wsrep_conf='/etc/mysql/conf.d/wsrep.cnf'
         } else {
@@ -75,9 +77,23 @@
         $os_username = $openstack_user_list[0]
 	$os_passwd = $openstack_passwd_list[0]
 
-        $glance_path ="/var/lib/glance/images"
+        if ($nfs_server != "" and nfs_server != undef) {
 
-	$nfs_server = $openstack_ip_list[0]
+	   $contrail_nfs_server = $nfs_server
+
+        } else {
+	   $contrail_nfs_server = $openstack_ip_list[0]
+        }
+
+        if ($nfs_glance_path != "" and nfs_glance_path != undef) {
+
+	   $contrail_nfs_glance_path = $nfs_glance_path
+
+        } else {
+	   $contrail_nfs_glance_path = "/var/lib/glance/images"
+        }
+
+      
 
         $openstack_mgmt_ip_list_shell = inline_template('<%= @openstack_mgmt_ip_list.map{ |ip| "#{ip}" }.join(",") %>')
         $openstack_ip_list_shell = inline_template('<%= @openstack_ip_list.map{ |name2| "#{name2}" }.join(" ") %>')
@@ -190,6 +206,19 @@
                 logoutput => 'true',
             }
             ->
+            exec { "ha-mon-restart":
+                command => "service contrail-hamon restart && echo contrail-ha-mon >> /etc/contrail/contrail_openstack_exec.out",
+                provider => shell,
+                logoutput => "true",
+                unless  => "grep -qx contrail-ha-mon  /etc/contrail/contrail_openstack_exec.out",
+        #                require => File["/opt/contrail/bin/transfer_keys.py"]
+            }
+
+
+
+        }
+        #This will be skipped if there is an external nfs server
+        if ($contrail_nfs_server == $host_control_ip) {
 	    package { 'nfs-kernel-server':
 		ensure  => present,
 	    }
@@ -203,6 +232,7 @@
             }
 
         }
+
         # setup_cmon
         file { "/opt/contrail/bin/setup-cmon-schema.py" :
             ensure  => present,
@@ -219,6 +249,7 @@
             require => [ File["/opt/contrail/bin/setup-cmon-schema.py"] ],
             logoutput => 'true',
         }
+/*
         ->
         exec { "setup-cluster-monitor" :
             command => "service contrail-hamon restart && chkconfig contrail-hamon on  && echo setup-cluster-monitor >> /etc/contrail/contrail_openstack_exec.out ",
@@ -228,6 +259,7 @@
             tries => 3,
             try_sleep => 15,
         }
+*/
         ->
         exec { "fix_xinetd_conf" :
             command => "sed -i -e 's#only_from = 0.0.0.0/0#only_from = $host_control_ip 127.0.0.1#' /etc/xinetd.d/contrail-mysqlprobe && service xinetd restart && chkconfig xinetd on && echo fix_xinetd_conf >> /etc/contrail/contrail_openstack_exec.out",
@@ -284,30 +316,22 @@
                 unless  => "grep -qx exec-transfer-keys  /etc/contrail/contrail_openstack_exec.out",
                 require => File["/opt/contrail/bin/transfer_keys.py"]
         }
-        ->
-        exec { "ha-mon-restart":
-                command => "service contrail-hamon restart && echo contrail-ha-mon >> /etc/contrail/contrail_openstack_exec.out",
-                provider => shell,
-                logoutput => "true",
-                unless  => "grep -qx contrail-ha-mon  /etc/contrail/contrail_openstack_exec.out",
-#                require => File["/opt/contrail/bin/transfer_keys.py"]
-        }
-
-        if ($openstack_index != "1" ) {
+        #This wil be executed for all openstacks ,if there is an external nfs server
+        if ($contrail_nfs_server != $host_control_ip ) {
 	    package { 'nfs-common':
 		ensure  => present,
 	    }
             ->
 	    exec { "mount-nfs" :
-		command => "sudo mount $nfs_server:$glance_path /var/lib/glance/images && echo mount-nfs >> /etc/contrail/contrail_openstack_exec.out",
+		command => "sudo mount $contrail_nfs_server:$contrail_nfs_glance_path /var/lib/glance/images && echo mount-nfs >> /etc/contrail/contrail_openstack_exec.out",
 		require => [  ],
 		unless  => "grep -qx mount-nfs  /etc/contrail/contrail_openstack_exec.out",
 		provider => shell,
 		logoutput => "true"
 	    }
 	    exec { "add-fstab" :
-		command => "echo \"$nfs_server:$glance_path /var/lib/glance/images nfs nfsvers=3,hard,intr,auto 0 0\" >> /etc/fstab && echo add-fstab >> /etc/contrail/contrail_openstack_exec.out ",
-		unless  => "grep -qx add-fstab  /etc/contrail/contrail_oprenstack_exec.out",
+		command => "echo \"$contrail_nfs_server:$contrail_nfs_glance_path /var/lib/glance/images nfs nfsvers=3,hard,intr,auto 0 0\" >> /etc/fstab && echo add-fstab >> /etc/contrail/contrail_openstack_exec.out ",
+		unless  => "grep -qx add-fstab  /etc/contrail/contrail_openstack_exec.out",
 		provider => shell,
 		logoutput => "true"
 	    }
diff --git a/contrail/environment/modules/contrail/manifests/keepalived.pp b/contrail/environment/modules/contrail/manifests/keepalived.pp
index b98f1221..64f2b532 100644
--- a/contrail/environment/modules/contrail/manifests/keepalived.pp
+++ b/contrail/environment/modules/contrail/manifests/keepalived.pp
@@ -56,6 +56,7 @@
     }
 
     if ($vip != "") {
+        $num_nodes = inline_template('<%= @ip_list.length %>')
         $tmp_index = inline_template('<%= @ip_list.index(@host_control_ip) %>')
         if ($tmp_index == nil) {
             fail("Host $host_control_ip not found in servers of config roles")
@@ -64,11 +65,20 @@
 
         $config_index = $tmp_index + 1
         $keepalived_priority = $keepalived_vrid - $config_index
-        if ($config_index == 1) {
+        if ($config_index == 1 ) {
             $keepalived_state = "MASTER"
+            $contrail_garp_master_delay = 5
+            $contrail_preempt_delay = 7
+        } 
+        elsif ($config_index ==2 and $num_nodes > 2 ) {
+            $keepalived_state = "MASTER"
+            $contrail_garp_master_delay = 1
+            $contrail_preempt_delay = 1
         }
         else {
             $keepalived_state = "BACKUP"
+            $contrail_garp_master_delay = 1
+            $contrail_preempt_delay = 1
         }
 
 	include ::keepalived
@@ -100,6 +110,8 @@
 	  virtual_ipaddress => $vip,
           garp_master_refresh => 1,
           garp_master_repeat => 3,
+          garp_master_delay => $contrail_garp_master_delay,
+          preempt_delay => $contrail_preempt_delay,
           vmac_xmit_base => true,
           track_interface => $control_data_intf,
 	  track_script      => ['check_haproxy','check_peers'],
diff --git a/contrail/environment/modules/contrail/manifests/params.pp b/contrail/environment/modules/contrail/manifests/params.pp
index 24553b60..e02c9172 100644
--- a/contrail/environment/modules/contrail/manifests/params.pp
+++ b/contrail/environment/modules/contrail/manifests/params.pp
@@ -510,6 +510,7 @@
     $compute_passwd_list,
     $host_roles = "",
     $external_bgp = "",
+    $sync_db = "",
     $contrail_plugin_location  = "NEUTRON_PLUGIN_CONFIG=\'/etc/neutron/plugins/opencontrail/ContrailPlugin.ini\'"
 ) {
     # Manifests use keystone_admin_token to refer to keystone_service_token too. Hence set
diff --git a/contrail/environment/modules/contrail/manifests/profile/openstack/glance/api.pp b/contrail/environment/modules/contrail/manifests/profile/openstack/glance/api.pp
index 58c07e9a..b35eb3cd 100644
--- a/contrail/environment/modules/contrail/manifests/profile/openstack/glance/api.pp
+++ b/contrail/environment/modules/contrail/manifests/profile/openstack/glance/api.pp
@@ -4,7 +4,8 @@
 class contrail::profile::openstack::glance::api {
   $api_network = $::openstack::config::network_api
   $api_address = ip_for_network($api_network)
-  $sync_db = true
+ # $sync_db = true
+  $sync_db = $::contrail::params::sync_db
   $management_network = $::openstack::config::network_management
   $management_address = ip_for_network($management_network)
 
diff --git a/contrail/environment/modules/contrail/manifests/profile/openstack_controller.pp b/contrail/environment/modules/contrail/manifests/profile/openstack_controller.pp
index 56a927ac..e3be3e63 100644
--- a/contrail/environment/modules/contrail/manifests/profile/openstack_controller.pp
+++ b/contrail/environment/modules/contrail/manifests/profile/openstack_controller.pp
@@ -15,6 +15,11 @@
     #Contrail expects neutron to run on config nodes only
     contain ::contrail::profile::openstack::neutron::server
 
+    package { 'contrail-openstack-dashboard':
+      ensure  => present,
+    }
+
+
 #   Though neutron runs on config, setup the db in openstack node 
     exec { 'neutron-db-sync':
       command     => 'neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini upgrade head',
diff --git a/contrail/environment/modules/contrail/manifests/provision_complete.pp b/contrail/environment/modules/contrail/manifests/provision_complete.pp
index dfe23693..d46d8260 100644
--- a/contrail/environment/modules/contrail/manifests/provision_complete.pp
+++ b/contrail/environment/modules/contrail/manifests/provision_complete.pp
@@ -2,9 +2,28 @@
     $state = undef
 )
 {
+    $host_control_ip = $::contrail::params::host_ip
+    $openstack_ip_list = $::contrail::params::openstack_ip_list
+
     contrail::lib::report_status { $state: state => $state }
+    if ($host_control_ip in $openstack_ip_list) {
 
-}
+      package { 'contrail-openstack':
+	ensure  => present,
+      }
+
+
+      exec { "exec_start_supervisor_openstack" :
+	  command => "service supervisor-openstack start && echo start_supervisor_openstack >> /etc/contrail/contrail_openstack_exec.out",
+	  unless  => "grep -qx start_supervisor_openstack /etc/contrail/contrail_openstack_exec.out",
+	  provider => shell,
+	  require => [ Package["contrail-openstack"]  ],
+	  logoutput => 'true'
+      }
 
 
+    }
+
+}
+
 
diff --git a/contrail/environment/modules/horizon/templates/local_settings.py.erb b/contrail/environment/modules/horizon/templates/local_settings.py.erb
index e2b10de3..a3c0022b 100644
--- a/contrail/environment/modules/horizon/templates/local_settings.py.erb
+++ b/contrail/environment/modules/horizon/templates/local_settings.py.erb
@@ -102,6 +102,16 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__))
 # SECRET_KEY = secret_key.generate_or_read_from_file(os.path.join(LOCAL_PATH, '.secret_key_store'))
 SECRET_KEY = '<%= @secret_key %>'
 
+import hashlib
+def hash_key(key, key_prefix, version):
+    new_key = ':'.join([key_prefix, str(version), key])
+    if len(new_key) > 250:
+	m = hashlib.md5()
+	m.update(new_key)
+	new_key = m.hexdigest()
+    return new_key
+
+
 # We recommend you use memcached for development; otherwise after every reload
 # of the django development server, you will have to login again. To use
 # memcached set CACHES to something like
@@ -126,6 +136,7 @@ CACHES = {
         <% else %>
         'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'
         <% end %>
+        'KEY_FUNCTION': hash_key,
     }
 }
 
@@ -553,3 +564,13 @@ COMPRESS_OFFLINE = <%= @compress_offline.to_s.capitalize %>
 # so we add this option to change the directory where uploaded files are temporarily
 # stored until they are loaded into Glance.
 FILE_UPLOAD_TEMP_DIR = '<%= @file_upload_temp_dir %>'
+
+
+# Enable the Ubuntu theme if it is present.
+try:
+    from ubuntu_theme import *
+except ImportError:
+    pass
+
+ALLOWED_HOSTS = '*'
+HORIZON_CONFIG['customization_module'] = 'contrail_openstack_dashboard.overrides'
diff --git a/contrail/environment/modules/keepalived/templates/vrrp_instance.erb b/contrail/environment/modules/keepalived/templates/vrrp_instance.erb
index 04bce3bb..91db362e 100644
--- a/contrail/environment/modules/keepalived/templates/vrrp_instance.erb
+++ b/contrail/environment/modules/keepalived/templates/vrrp_instance.erb
@@ -17,15 +17,12 @@ vrrp_instance <%= @name %> {
   <%- if @preempt_delay -%>
   preempt_delay             <%= @preempt_delay %>
   <%- end -%>
-
   <%- if @garp_master_refresh -%>
   garp_master_refresh         <%= @garp_master_refresh %>
   <%- end -%>
-
   <%- if @garp_master_repeat -%>
   garp_master_repeat         <%= @garp_master_repeat %>
   <%- end -%>
-
   <%- if @vmac_xmit_base == true -%>
   vmac_xmit_base
   <%- end -%>
diff --git a/contrail/environment/modules/keepalived/templates/vrrp_script.erb b/contrail/environment/modules/keepalived/templates/vrrp_script.erb
index be38ad3f..3be36df9 100644
--- a/contrail/environment/modules/keepalived/templates/vrrp_script.erb
+++ b/contrail/environment/modules/keepalived/templates/vrrp_script.erb
@@ -5,7 +5,7 @@ vrrp_script <%= @name %> {
   weight    <%= @weight %>
   <%- end -%>
   <%- if @timeout-%>
-  fall      <%= @timeout%>
+  timeout      <%= @timeout%>
   <%- end -%>
   <%- if @fall -%>
   fall      <%= @fall %>
diff --git a/contrail/environment/modules/mysql/manifests/params.pp b/contrail/environment/modules/mysql/manifests/params.pp
index 3114c1a9..81fb55fc 100644
--- a/contrail/environment/modules/mysql/manifests/params.pp
+++ b/contrail/environment/modules/mysql/manifests/params.pp
@@ -196,12 +196,12 @@
       'bind-address'          => '127.0.0.1',
       'datadir'               => $mysql::params::datadir,
       'expire_logs_days'      => '10',
-      'key_buffer_size'       => '16M',
+#      'key_buffer_size'       => '16M',
       'log-error'             => $mysql::params::log_error,
-      'max_allowed_packet'    => '16M',
+#      'max_allowed_packet'    => '16M',
       'max_binlog_size'       => '100M',
-      'max_connections'       => '151',
-      'myisam_recover'        => 'BACKUP',
+      'max_connections'       => '10000',
+#      'myisam_recover'        => 'BACKUP',
       'pid-file'              => $mysql::params::pidfile,
       'port'                  => '3306',
       'query_cache_limit'     => '1M',
@@ -218,12 +218,12 @@
       'user'                  => 'mysql',
     },
     'mysqldump'             => {
-      'max_allowed_packet'  => '16M',
+#      'max_allowed_packet'  => '16M',
       'quick'               => true,
       'quote-names'         => true,
     },
     'isamchk'      => {
-      'key_buffer_size' => '16M',
+#      'key_buffer_size' => '16M',
     },
   }
 
diff --git a/contrail/environment/modules/mysql/templates/my.cnf.erb b/contrail/environment/modules/mysql/templates/my.cnf.erb
index 5aa959ba..70552575 100644
--- a/contrail/environment/modules/mysql/templates/my.cnf.erb
+++ b/contrail/environment/modules/mysql/templates/my.cnf.erb
@@ -15,4 +15,9 @@
 <%   end %>
 <% end -%>
 
+[mysqld]
+wait_timeout = 60
+interactive_timeout = 60
+lock_wait_timeout = 600
+
 !includedir /etc/mysql/conf.d/
diff --git a/contrail/environment/modules/openstack/manifests/common/cinder.pp b/contrail/environment/modules/openstack/manifests/common/cinder.pp
index cd2f0e14..c67451f7 100644
--- a/contrail/environment/modules/openstack/manifests/common/cinder.pp
+++ b/contrail/environment/modules/openstack/manifests/common/cinder.pp
@@ -3,6 +3,7 @@
 class openstack::common::cinder {
   $internal_vip = $::contrail::params::internal_vip
   $controller_management_address = $::openstack::config::controller_address_management
+  $sync_db = $::contrail::params::sync_db
 
   if ($internal_vip != "" and $internal_vip != undef) {
     cinder_config {
diff --git a/contrail/environment/modules/openstack/manifests/common/contrail/nova.pp b/contrail/environment/modules/openstack/manifests/common/contrail/nova.pp
index 2c63b959..d8a19efe 100644
--- a/contrail/environment/modules/openstack/manifests/common/contrail/nova.pp
+++ b/contrail/environment/modules/openstack/manifests/common/contrail/nova.pp
@@ -23,6 +23,9 @@
     $neutron_ip_address = $::contrail::params::config_ip_list[0]
   }
 
+#  $contrail_internal_vip = $::contrail::params::internal_vip
+#  $external_vip = $::contrail::params::internal_vip
+#  $contrail_external_vip = $::contrail::params::contrail_internal_vip
 
   class { '::nova':
     sql_connection     => $::openstack::resources::connectors::nova,
@@ -48,9 +51,17 @@
     sync_db         => $sync_db,
   }
 
-  class { '::nova::vncproxy':
-    host    => $::openstack::config::controller_address_api,
-    enabled => $is_controller,
+  if ($internal_vip != "" and $internal_vip != undef) {
+      class { '::nova::vncproxy':
+	host    => $::openstack::config::controller_address_api,
+	enabled => $is_controller,
+        port => '6999',
+      }
+  } else {
+      class { '::nova::vncproxy':
+	host    => $::openstack::config::controller_address_api,
+	enabled => $is_controller,
+      }
   }
 
   class { [
diff --git a/contrail/environment/modules/openstack/manifests/common/glance.pp b/contrail/environment/modules/openstack/manifests/common/glance.pp
index 61f73a79..4935d44d 100644
--- a/contrail/environment/modules/openstack/manifests/common/glance.pp
+++ b/contrail/environment/modules/openstack/manifests/common/glance.pp
@@ -17,6 +17,7 @@
       verbose           => $::openstack::config::verbose,
       debug             => $::openstack::config::debug,
       enabled           => $::openstack::profile::base::is_storage,
+      database_idle_timeout      => "180",
       bind_port     => '9393',
       mysql_module      => '2.2',
     }
@@ -35,6 +36,7 @@
       verbose           => $::openstack::config::verbose,
       debug             => $::openstack::config::debug,
       enabled           => $::openstack::profile::base::is_storage,
+      database_idle_timeout      => "180",
       mysql_module      => '2.2',
     }
     $contrail_rabbit_host = $::contrail::params::config_ip_list[0]
@@ -42,11 +44,29 @@
 
   }
 
+
+  glance_api_config {
+#    'database/idle_timeout':             value => "180";
+    'database/min_pool_size':            value => "100";
+    'database/max_pool_size':            value => "700";
+    'database/max_overflow':             value => "1080";
+    'database/retry_interval':           value => "5";
+    'database/max_retries':              value => "-1";
+    'database/db_max_retries':           value => "3";
+    'database/db_retry_interval':        value => "1";
+    'database/connection_debug':         value => "10";
+    'database/pool_timeout':             value => "120";
+#     notify => Service['glance-api']
+
+  }
+
+  ->
   # basic service config
   glance_api_config {'DEFAULT/rabbit_host': 
      value => $contrail_rabbit_host,
      notify => Service['glance-api']
   }
+  ->
   glance_api_config {'DEFAULT/rabbit_port': 
      value => $contrail_rabbit_port,
      notify => Service['glance-api']
diff --git a/contrail/environment/modules/openstack/manifests/common/keystone.pp b/contrail/environment/modules/openstack/manifests/common/keystone.pp
index 9ea50b3c..fba32f67 100644
--- a/contrail/environment/modules/openstack/manifests/common/keystone.pp
+++ b/contrail/environment/modules/openstack/manifests/common/keystone.pp
@@ -1,5 +1,8 @@
 class openstack::common::keystone {
   $internal_vip = $::contrail::params::internal_vip
+  $sync_db = $::contrail::params::sync_db
+ 
+  notify { "SYNC_DB = $sync_db":; }
 
   if ($internal_vip != "" and $internal_vip != undef) {
 
@@ -17,6 +20,21 @@
     rabbit_port     => '5673',
     rabbit_host     => $::openstack::config::controller_address_management,
   }
+    keystone_config {
+      'database/min_pool_size':   value => "100";
+      'database/max_pool_size':   value => "700";
+      'database/max_overflow':   value => "100";
+      'database/retry_interva':   value => "5";
+      'database/max_retries':   value => "-1";
+      'database/db_max_retries':   value => "-1";
+      'database/db_retry_interval':   value => "1";
+      'database/connection_debug':   value => "10";
+      'database/pool_timeout':   value => "120";
+  #    'sql/connection':   value => $database_connection_real, secret => true;
+  #    'database/idle_timeout': value => $database_idle_timeout_real;
+  #    'sql/idle_timeout': value => $database_idle_timeout_real;
+    }
+
 } else {
   class { '::keystone':
     admin_token     => $::openstack::config::keystone_admin_token,
@@ -30,7 +48,11 @@
     rabbit_port     => '5672',
     rabbit_host     => $::contrail::params::config_ip_list[0],
   }
+    keystone_config {
+      'identity/driver':   value => "keystone.identity.backends.sql.Identity";
+      'ec2/driver':   value => "keystone.contrib.ec2.backends.sql.Ec2";
+      'DEFAULT/onready':   value => "keystone.common.systemd";
+    }
 
-
-}
+  }
 }
diff --git a/contrail/environment/modules/openstack/manifests/common/neutron.pp b/contrail/environment/modules/openstack/manifests/common/neutron.pp
index 2db7b3ed..0933f55c 100644
--- a/contrail/environment/modules/openstack/manifests/common/neutron.pp
+++ b/contrail/environment/modules/openstack/manifests/common/neutron.pp
@@ -5,7 +5,7 @@
 # This follows the suggest deployment from the neutron Administrator Guide.
 class openstack::common::neutron {
   $controller_management_address = $::openstack::config::controller_address_management
-
+  $sync_db = $::contrail::params::sync_db
   $data_network = $::openstack::config::network_data
   $data_address = ip_for_network($data_network)
 
diff --git a/contrail/environment/modules/openstack/manifests/common/nova.pp b/contrail/environment/modules/openstack/manifests/common/nova.pp
index 7e6709ae..79a4195a 100644
--- a/contrail/environment/modules/openstack/manifests/common/nova.pp
+++ b/contrail/environment/modules/openstack/manifests/common/nova.pp
@@ -5,6 +5,8 @@
 # depends on openstack::profile::base having been added to a node
 class openstack::common::nova ($is_compute    = false) {
   $is_controller = $::openstack::profile::base::is_controller
+  $sync_db = $::contrail::params::sync_db
+
 
   $management_network = $::openstack::config::network_management
   $management_address = ip_for_network($management_network)
@@ -46,19 +48,36 @@
       'DEFAULT/osapi_compute_listen_port':     value => '9774';
       'DEFAULT/metadata_listen_port':     value => '9775';
     }
-  }
+    class { '::nova::api':
+      admin_password                       => $::openstack::config::nova_password,
+      auth_host                            => $controller_management_address,
+      enabled                              => $is_controller,
+      sync_db                              => $sync_db,
+      neutron_metadata_proxy_shared_secret => $::openstack::config::neutron_shared_secret,
+      osapi_compute_workers                => '40'
+    }
 
-  class { '::nova::api':
-    admin_password                       => $::openstack::config::nova_password,
-    auth_host                            => $controller_management_address,
-    enabled                              => $is_controller,
-    sync_db                              => $sync_db,
-    neutron_metadata_proxy_shared_secret => $::openstack::config::neutron_shared_secret,
-  }
+    class { '::nova::vncproxy':
+      host    => $::openstack::config::controller_address_api,
+      enabled => $is_controller,
+      port => '6999',
+    }
+
+
+  } else {
+    class { '::nova::api':
+      admin_password                       => $::openstack::config::nova_password,
+      auth_host                            => $controller_management_address,
+      enabled                              => $is_controller,
+      sync_db                              => $sync_db,
+      neutron_metadata_proxy_shared_secret => $::openstack::config::neutron_shared_secret,
+    }
+
+    class { '::nova::vncproxy':
+      host    => $::openstack::config::controller_address_api,
+      enabled => $is_controller,
+    }
 
-  class { '::nova::vncproxy':
-    host    => $::openstack::config::controller_address_api,
-    enabled => $is_controller,
   }
 
   class { [
diff --git a/contrail/environment/modules/openstack/manifests/profile/contrail/glance/api.pp b/contrail/environment/modules/openstack/manifests/profile/contrail/glance/api.pp
index 28af8298..79d2d76c 100644
--- a/contrail/environment/modules/openstack/manifests/profile/contrail/glance/api.pp
+++ b/contrail/environment/modules/openstack/manifests/profile/contrail/glance/api.pp
@@ -4,7 +4,7 @@
 class openstack::profile::contrail::glance::api {
   $api_network = $::openstack::config::network_api
   $api_address = ip_for_network($api_network)
-
+  $sync_db = $::contrail::params::sync_db
   $management_network = $::openstack::config::network_management
   $management_address = ip_for_network($management_network)
 
diff --git a/contrail/environment/modules/openstack/manifests/profile/glance/api.pp b/contrail/environment/modules/openstack/manifests/profile/glance/api.pp
index 4b0c55cb..c01ccbc6 100644
--- a/contrail/environment/modules/openstack/manifests/profile/glance/api.pp
+++ b/contrail/environment/modules/openstack/manifests/profile/glance/api.pp
@@ -5,6 +5,8 @@
   $api_network = $::openstack::config::network_api
   $api_address = ip_for_network($api_network)
 
+  $sync_db = $::contrail::params::sync_db
+
   $management_network = $::openstack::config::network_management
   $management_address = ip_for_network($management_network)
 
diff --git a/contrail/environment/modules/openstack/manifests/profile/horizon.pp b/contrail/environment/modules/openstack/manifests/profile/horizon.pp
index aa25f45b..1d3ef984 100644
--- a/contrail/environment/modules/openstack/manifests/profile/horizon.pp
+++ b/contrail/environment/modules/openstack/manifests/profile/horizon.pp
@@ -1,9 +1,21 @@
 # Profile to install the horizon web service
 class openstack::profile::horizon {
+  $internal_vip = $::contrail::params::internal_vip
+
+  if ($internal_vip != "" and $internal_vip != undef) {
+
+    $contrail_keystone_url = "http://${internal_vip}:5000/v2.0"
+  } else {
+
+    $contrail_keystone_url = "http://127.0.0.1:5000/v2.0"
+  }
+
+ 
   class { '::horizon':
     fqdn            => [ '127.0.0.1', $::openstack::config::controller_address_api, $::fqdn ],
     secret_key      => $::openstack::config::horizon_secret_key,
     cache_server_ip => $::openstack::config::controller_address_management,
+    keystone_url => $contrail_keystone_url
 
   }
 
diff --git a/contrail/environment/modules/openstack/manifests/resources/connectors.pp b/contrail/environment/modules/openstack/manifests/resources/connectors.pp
index a4e06b4f..621fe537 100644
--- a/contrail/environment/modules/openstack/manifests/resources/connectors.pp
+++ b/contrail/environment/modules/openstack/manifests/resources/connectors.pp
@@ -1,6 +1,14 @@
 class openstack::resources::connectors {
+  $internal_vip = $::contrail::params::internal_vip
+
+  if ($internal_vip != "" and $internal_vip != undef) {
+    $mysql_port = "33306"
+    $management_ip_address = $::openstack::config::controller_address_management 
+    $management_address = "${management_ip_address}:${mysql_port}"
+  } else { 
+    $management_address = $::openstack::config::controller_address_management
+  }
 
-  $management_address = $::openstack::config::controller_address_management
   $password = $::openstack::config::mysql_service_password
 
   $keystone = "mysql://keystone:${password}@${management_address}/keystone"