Closes-Bug: #1538357

Closes-Bug: #1538298 Closes-Bug: #1537290 config was stuck when adding a node at the begining, as ssl keys were not re-distributed from the first new node. controller gets stuck at openstack_started When a node was delete and re-added, old keepalived instance was still running. "uninstall_collector" Zookeeper was not getting restarted as result of this below commit e31d7a9 Nitish provided a fix and tested it. Change-Id: I5d77c6dae0fa527deb2331cc184b4f297ad39ff7
Juniper · Feb 3, 2016 · 7439ea5 · 7439ea5
1 parent e003aa6
commit 7439ea5
Show file tree

Hide file tree

Showing 21 changed files with 147 additions and 27 deletions.
diff --git a/contrail/environment/modules/contrail/files/galera_setup_gcomm_mod.py b/contrail/environment/modules/contrail/files/galera_setup_gcomm_mod.py
@@ -38,6 +38,7 @@ def __init__(self, args_str = None):
         self.mysql_redo_log_sz = '5242880'
 
     def is_clustered(self, galera_ip_list, self_ip):
+        print "is_clustered"
         status,output = commands.getstatusoutput("cat /etc/contrail/mysql.token")
         mysql_token = output
         status,output = commands.getstatusoutput('mysql -uroot -p%s -e "show status like \'wsrep_incoming_addresses\'"' %  mysql_token )
@@ -51,9 +52,11 @@ def is_clustered(self, galera_ip_list, self_ip):
             return False
 
     def is_cluster_synced(self):
+        print "is_cluster_synced"
         status,output = commands.getstatusoutput("cat /etc/contrail/mysql.token")
         mysql_token = output
         status,output = commands.getstatusoutput('mysql -uroot -p%s -e "show status like \'wsrep_local_state\'"' %  mysql_token )
+        print "wsrep_local_state: %s" % output
         output_list = []
         output_list = output.split('\t')
         if ('4'  in output_list):
@@ -62,13 +65,19 @@ def is_cluster_synced(self):
             return False
 
 
-    def is_cluster_updated(self, galera_ip_list):
+    def is_cluster_running(self, galera_ip_list, host='127.0.0.1'):
+        print "is_cluster_running"
         status,output = commands.getstatusoutput("cat /etc/contrail/mysql.token")
         mysql_token = output
-        status,output = commands.getstatusoutput('mysql -uroot -p%s -e "show status like \'wsrep_incoming_addresses\'"' %  mysql_token )
+        cmd = "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s \"mysql -uroot -p%s -e 'show status'\" | grep wsrep_incoming_addresses" %  (host, mysql_token)
+        print cmd
+#        status,output = commands.getstatusoutput('mysql -uroot -p%s -h %s -e "show status like \'wsrep_incoming_addresses\'"' %  (mysql_token, host) )
+        status,output = commands.getstatusoutput(cmd)
+
+
         print "wsrep_incoming_addresses: %s" % output
         output_list = []
-        wsrep_nodes =  (':3306,'.join(self._args.galera_ip_list) + ':3306')
+        wsrep_nodes =  (':3306,'.join(galera_ip_list) + ':3306')
         output_list = output.split("\t")
         if (wsrep_nodes  in output_list):
             return True
@@ -102,7 +111,7 @@ def parse_args(self, args_str):
 
     def fixup_config_files(self):
         # fix cmon_param
-
+        print "fixup_config_files"
         zk_servers_ports = ','.join(['%s:2181' %(s) for s in self._args.zoo_ip_list])
         template_vals = {'__internal_vip__' : self._args.internal_vip,
                          '__haproxy_dips__' :
@@ -281,12 +290,17 @@ def setup_cron(self):
         local('rm %s/galera_cron' % self._temp_dir_name)
 
     def run_services(self):
+        print "run_services"
         if self.is_cluster_synced( ):
             local("service %s restart" % self.mysql_svc)
         if self.is_clustered(self._args.galera_ip_list, self._args.self_ip):
             return
         self.cleanup_redo_log()
-        if self._args.openstack_index == 1:
+
+        if self._args.openstack_index == 1 and self.is_cluster_running(self._args.galera_ip_list[1:], self._args.galera_ip_list[1]):
+            print "Mysql cluster already present for '%s'" % self._args.galera_ip_list[1:]
+            local("service %s restart" % self.mysql_svc)
+        elif self._args.openstack_index == 1:
             local("service %s stop" % self.mysql_svc)
             local("service %s start --wsrep_cluster_address=gcomm://" % self.mysql_svc)
         else:

diff --git a/contrail/environment/modules/contrail/files/transfer_keys.py b/contrail/environment/modules/contrail/files/transfer_keys.py
@@ -69,6 +69,11 @@ def main(argv):
     if status != 0:
         sys.exit(1)
 
+    status,output = commands.getstatusoutput("md5sum /etc/keystone/ssl/certs/*")
+    print output
+    if status != 0:
+        sys.exit(1)
+
 
     status,output = commands.getstatusoutput("rm -rf /tmp/keystone-signing-*")
     print output

diff --git a/contrail/environment/modules/contrail/lib/facter/contrail_roles.rb b/contrail/environment/modules/contrail/lib/facter/contrail_roles.rb
@@ -9,6 +9,12 @@
             service_str = "service supervisor-" + role + " status"
             puts(service_str)
             is_service_running = system(service_str)
+            if role == "analytics"
+                role = "collector"
+            elsif role == "vrouter"
+                role = "compute"
+            end
+
             if (is_service_running)
                 contrail_roles[role] = true 
             else

diff --git a/contrail/environment/modules/contrail/manifests/database/config.pp b/contrail/environment/modules/contrail/manifests/database/config.pp
@@ -164,7 +164,7 @@
         # File['/etc/init.d/supervisord-contrail-database'] -> File['/etc/contrail/contrail_setup_utils/config-zk-files-setup.sh']
         File['/etc/init.d/supervisord-contrail-database'] -> File['/etc/zookeeper/conf/zoo.cfg'] ->
         File ['/etc/zookeeper/conf/log4j.properties'] -> File ['/etc/zookeeper/conf/environment'] ->
-        File ['/etc/zookeeper/conf/myid']
+        File ['/etc/zookeeper/conf/myid'] ~> Service['zookeeper']
     }
     # set high session timeout to survive glance led disk activity
     # Commented out call to old exec

diff --git a/contrail/environment/modules/contrail/manifests/database/service.pp b/contrail/environment/modules/contrail/manifests/database/service.pp
@@ -1,5 +1,11 @@
 class contrail::database::service {
-    service { [ 'zookeeper', 'supervisor-database', 'contrail-database'] :
+    service { 'zookeeper':
+        ensure => running,
+        enable => true,
+        subscribe => File['/etc/zookeeper/conf/zoo.cfg'],
+    }
+    ->
+    service { ['supervisor-database', 'contrail-database'] :
         ensure    => running,
         enable    => true,
     }

diff --git a/contrail/environment/modules/contrail/manifests/delete_vnc_config.pp b/contrail/environment/modules/contrail/manifests/delete_vnc_config.pp
@@ -3,7 +3,10 @@
     $config_ip_to_use,
     $host_control_ip,
     $keystone_admin_user,
-    $keystone_admin_password
+    $keystone_admin_password,
+    $keystone_admin_tenant,
+    $contrail_router_type,
+    $openstack_ip,
 ) {
     exec { 'del-vnc-config' :
 	command => "/bin/bash -c \"python /opt/contrail/utils/provision_vrouter.py --host_name $::hostname --host_ip $host_control_ip --api_server_ip $config_ip_to_use --oper del --admin_user $keystone_admin_user --admin_password $keystone_admin_password --admin_tenant_name $keystone_admin_tenant --openstack_ip $openstack_ip ${contrail_router_type} && echo del-vnc-config >> /etc/contrail/contrail_compute_exec.out\"",

diff --git a/contrail/environment/modules/contrail/manifests/ha_config.pp b/contrail/environment/modules/contrail/manifests/ha_config.pp
@@ -207,7 +207,7 @@
                 command   => $contrail_exec_vnc_galera,
                 cwd       => '/opt/contrail/bin/',
                 provider  => shell,
-                logoutput => $contrail_logoutput,
+                logoutput => true,
                 tries     => 3,
                 try_sleep => 15,
             }
@@ -245,7 +245,7 @@
                 command   => $contrail_exec_check_wsrep,
                 cwd       => '/opt/contrail/bin/',
                 provider  => shell,
-                logoutput => $contrail_logoutput,
+                logoutput => true,
             }
         }
         #This will be skipped if there is an external nfs server
@@ -330,8 +330,7 @@
         exec { 'exec-transfer-keys':
             command   => "python /opt/contrail/bin/transfer_keys.py ${os_master} \"/etc/ssl/\" ${os_username} ${os_passwd} && echo exec-transfer-keys >> /etc/contrail/contrail_openstack_exec.out",
             provider  => shell,
-            logoutput => $contrail_logoutput,
-            unless    => 'grep -qx exec-transfer-keys  /etc/contrail/contrail_openstack_exec.out',
+            logoutput => true,
         }
 
         if (enable_sequence_provisioning == false) {

diff --git a/contrail/environment/modules/contrail/manifests/params.pp b/contrail/environment/modules/contrail/manifests/params.pp
@@ -773,4 +773,21 @@
         $multi_tenancy_options = ""
     }
 
+    notify { "host_role = $host_roles and contrail_roles = $contrail_roles":; }
+    $contrail_roles_present_hash = delete_values($contrail_roles, false)
+    $contrail_roles_present_array = keys($contrail_roles_present_hash)
+    notify { "contrail_roles_present_hash = $contrail_roles_present_hash and  contrail_roles_present_array = $contrail_roles_present_array":; }
+    $roles_to_delete = difference($contrail_roles_present_array , $host_roles)
+    notify { "roles_to_delete is $roles_to_delete":; }
+    #copy of roles to delete as all run as a single manifest
+    #TODO verify its a ref or a copy
+    $cpy_roles_to_delete = $roles_to_delete
+
+    $num_roles_to_delete = size($roles_to_delete)
+    if ($num_role_to_delete) {
+       $is_there_roles_to_delete = true
+    } else {
+       $is_there_roles_to_delete = false 
+    }
+
 }
diff --git a/contrail/environment/modules/contrail/manifests/profile/collector.pp b/contrail/environment/modules/contrail/manifests/profile/collector.pp
@@ -9,11 +9,12 @@
 #
 class contrail::profile::collector (
     $enable_module = $::contrail::params::enable_collector,
+    $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
     $host_roles = $::contrail::params::host_roles
 ) {
-    if ($enable_module and "collector" in $host_roles) {
+    if ($enable_module and "collector" in $host_roles and $is_there_roles_to_delete == false) {
         contain ::contrail::collector
-    } elsif ((!("collector" in $host_roles)) and ($contrail_roles["analytics"] == true)) {
+    } elsif ((!("collector" in $host_roles)) and ($contrail_roles["collector"] == true)) {
 
         notify { "uninstalling collector":; }
         contain ::contrail::uninstall_collector

diff --git a/contrail/environment/modules/contrail/manifests/profile/compute.pp b/contrail/environment/modules/contrail/manifests/profile/compute.pp
@@ -14,9 +14,10 @@
 class contrail::profile::compute (
     $enable_module = $::contrail::params::enable_compute,
     $enable_ceilometer = $::contrail::params::enable_ceilometer,
+    $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
     $host_roles = $::contrail::params::host_roles
 ) {
-    if ($enable_module and "compute" in $host_roles) {
+    if ($enable_module and "compute" in $host_roles and $is_there_roles_to_delete == false) {
         require ::contrail::common
         require ::openstack::profile::firewall
         require ::contrail::profile::nova::compute
@@ -26,7 +27,7 @@
             include ::contrail::ceilometer::agent::auth
             include ::ceilometer::agent::compute
         }
-    } elsif ((!("compute" in $host_roles)) and ($contrail_roles["vrouter"] == true)) {
+    } elsif ((!("compute" in $host_roles)) and ($contrail_roles["compute"] == true)) {
 
         notify { "uninstalling compute":; }
         contain ::contrail::uninstall_compute

diff --git a/contrail/environment/modules/contrail/manifests/profile/config.pp b/contrail/environment/modules/contrail/manifests/profile/config.pp
@@ -9,10 +9,11 @@
 #
 class contrail::profile::config (
     $enable_module = $::contrail::params::enable_config,
+    $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
     $host_roles = $::contrail::params::host_roles
 ) {
 
-    if ($enable_module and 'config' in $host_roles) {
+    if ($enable_module and 'config' in $host_roles and $is_there_roles_to_delete == false) {
         contain ::contrail::config
         #contrail expects neutron server to run on configs
         include ::contrail::profile::neutron_server

diff --git a/contrail/environment/modules/contrail/manifests/profile/controller.pp b/contrail/environment/modules/contrail/manifests/profile/controller.pp
@@ -9,10 +9,11 @@
 #
 class contrail::profile::controller (
     $enable_module = $::contrail::params::enable_control,
+    $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
     $host_roles = $::contrail::params::host_roles
 ) {
 
-    if ($enable_module and "control" in $host_roles) {
+    if ($enable_module and "control" in $host_roles and $is_there_roles_to_delete == false) {
         contain ::contrail::control
         #contrail expects neutron server to run on controls
         include ::contrail::profile::neutron_server

diff --git a/contrail/environment/modules/contrail/manifests/profile/database.pp b/contrail/environment/modules/contrail/manifests/profile/database.pp
@@ -14,9 +14,10 @@
 class contrail::profile::database (
     $enable_module = $::contrail::params::enable_database,
     $host_roles = $::contrail::params::host_roles,
+    $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
     $enable_ceilometer = $::contrail::params::enable_ceilometer
 ) {
-    if ($enable_module and "database" in $host_roles) {
+    if ($enable_module and "database" in $host_roles and $is_there_roles_to_delete == false) {
         contain ::contrail::database
         if ($enable_ceilometer) {
             include ::contrail::profile::mongodb

diff --git a/contrail/environment/modules/contrail/manifests/profile/haproxy.pp b/contrail/environment/modules/contrail/manifests/profile/haproxy.pp
@@ -8,9 +8,20 @@
 #     (optional) - Defaults to true.
 #
 class contrail::profile::haproxy (
-    $enable_module = $::contrail::params::enable_haproxy
+    $enable_module = $::contrail::params::enable_haproxy,
+    $host_roles = $::contrail::params::host_roles
 ) {
-  if ($enable_module) {
+  if ($enable_module and ('config' in $host_roles or 'openstack' in $host_roles)) {
       contain ::contrail::haproxy
+  } elsif (((!('config' in $host_roles)) and ($contrail_roles['config'] == true)) or
+           ((!('openstack' in $host_roles)) and ($contrail_roles['openstack'] == true))
+          ) 
+  {
+        notify { 'uninstalling haproxy':; }
+        contrail::lib::report_status { 'uninstall_haproxy_started': state => 'uninstall_haproxy_started' }
+        -> class {'::contrail::uninstall_haproxy' : }
+        -> contrail::lib::report_status { 'uninstall_haproxy_completed': state => 'uninstall_haproxy_completed' }
+
   }
+
 }
diff --git a/contrail/environment/modules/contrail/manifests/profile/keepalived.pp b/contrail/environment/modules/contrail/manifests/profile/keepalived.pp
@@ -8,11 +8,22 @@
 #     (optional) - Defaults to true.
 #
 class contrail::profile::keepalived (
-    $enable_module = $::contrail::params::enable_keepalived
+    $enable_module = $::contrail::params::enable_keepalived,
+    $host_roles = $::contrail::params::host_roles
 ) {
-  if ($enable_module) {
+  if ($enable_module and ('config' in $host_roles or 'openstack' in $host_roles)) {
     contrail::lib::report_status { 'keepalived_started': state => 'keepalived_started' }
     -> class {'::contrail::keepalived' : }
     -> contrail::lib::report_status { 'keepalived_completed': state => 'keepalived_completed' }
+
+  } elsif (((!('config' in $host_roles)) and ($contrail_roles['config'] == true)) or
+           ((!('openstack' in $host_roles)) and ($contrail_roles['openstack'] == true)) 
+          ) {
+        notify { 'uninstalling keepalived':; }
+    contrail::lib::report_status { 'uinstall_keepalived_started': state => 'uninstall_keepalived_started' }
+    -> class {'::contrail::uninstall_keepalived' : }
+    -> contrail::lib::report_status { 'uninstall_keepalived_completed': state => 'uninstall_keepalived_completed' }
+
+
   }
 }
diff --git a/contrail/environment/modules/contrail/manifests/profile/openstack_controller.pp b/contrail/environment/modules/contrail/manifests/profile/openstack_controller.pp
@@ -14,10 +14,11 @@
 class contrail::profile::openstack_controller (
   $enable_module = $::contrail::params::enable_openstack,
   $enable_ceilometer = $::contrail::params::enable_ceilometer,
+  $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
   $host_roles = $::contrail::params::host_roles,
   $package_sku = $::contrail::params::package_sku
 ) {
-    if ($enable_module and 'openstack' in $host_roles) {
+    if ($enable_module and 'openstack' in $host_roles and $is_there_roles_to_delete == false) {
         contrail::lib::report_status { 'openstack_started': state => 'openstack_started' } ->
         class {'::openstack::profile::base' : } ->
         class {'::nova::quota' :

diff --git a/contrail/environment/modules/contrail/manifests/profile/webui.pp b/contrail/environment/modules/contrail/manifests/profile/webui.pp
@@ -9,10 +9,11 @@
 #
 class contrail::profile::webui (
     $enable_module = $::contrail::params::enable_webui,
+    $is_there_roles_to_delete = $::contrail::params::is_there_roles_to_delete,
     $host_roles = $::contrail::params::host_roles
 ) {
 
-    if ($enable_module and 'webui' in $host_roles) {
+    if ($enable_module and 'webui' in $host_roles and $is_there_roles_to_delete == false) {
         contain ::contrail::webui
     } elsif ((!('webui' in $host_roles)) and ($contrail_roles['webui'] == true)) {
 

diff --git a/contrail/environment/modules/contrail/manifests/uninstall_compute.pp b/contrail/environment/modules/contrail/manifests/uninstall_compute.pp
@@ -65,7 +65,10 @@
            config_ip_to_use => $config_ip_to_use,
            host_control_ip => $host_control_ip,
            keystone_admin_user => $keystone_admin_user,
-           keystone_admin_password => $keystone_admin_password
+           keystone_admin_password => $keystone_admin_password,
+           keystone_admin_tenant => $keystone_admin_tenant,
+           openstack_ip => $openstack_ip,
+           contrail_router_type => "", 
     }
     ->
     service { 'supervisor-vrouter' :

diff --git a/contrail/environment/modules/contrail/manifests/uninstall_haproxy.pp b/contrail/environment/modules/contrail/manifests/uninstall_haproxy.pp
@@ -0,0 +1,19 @@
+class contrail::uninstall_haproxy(
+    $host_control_ip = $::contrail::params::host_ip,
+    $host_roles = $::contrail::params::host_roles,
+) inherits ::contrail::params {
+
+    service { 'haproxy' :
+	enable => false,
+	ensure => stopped,
+    }
+    ->
+#    package { 'haproxy' : ensure => purged}
+#    ->
+    file { [           
+            '/etc/haproxy/haproxy.cfg'
+           ]:
+        ensure  => absent,
+    }
+
+}
diff --git a/contrail/environment/modules/contrail/manifests/uninstall_keepalived.pp b/contrail/environment/modules/contrail/manifests/uninstall_keepalived.pp
@@ -0,0 +1,19 @@
+class contrail::uninstall_keepalived (
+    $host_control_ip = $::contrail::params::host_ip,
+    $host_roles = $::contrail::params::host_roles,
+) inherits ::contrail::params {
+
+    service { "keepalived" :
+	enable => false,
+	ensure => stopped,
+    }
+    ->
+   # package { 'keepalived' : ensure => purged}
+   # ->
+    file { [           
+            '/etc/keepalived/keepalived.conf'
+           ]:
+        ensure  => absent,
+    }
+
+}
diff --git a/contrail/environment/modules/contrail/manifests/uninstall_openstack.pp b/contrail/environment/modules/contrail/manifests/uninstall_openstack.pp
@@ -53,7 +53,7 @@
     class {'::contrail::remove_mysql_flush_privileges':
         mysql_root_password => $mysql_root_password
     }
-
+    ->
     service { ['contrail-hamon','cmon', 'apache2', 'mysql'] :
         ensure    => false ,
         enable    => false,