Closes-Bug: #1545618 : Upgrade of contrail from R2.23 to R3.0 fixed

Changes:
I. Cassandra upgrade thru intermediate 2.0 version
Cassandra configured using file_line (not templates)
Cassandra templates removed

II. Python-kafka-python package replaced by python-kafka package

III. Contrail-config package brings new configs, causes dependency issues - fixed

IV. Removed secret key from nova api (metadata services bug)

Closes-Bug: #1543333

Change-Id: I1a655b129c9878f499aef58cb1b9057c7769d53c

contrail/environment/modules/contrail/files/upgrade_cassandra.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -x
+this_host=$1; shift
+database_dir=$1
+cassandra_version=`dpkg -s cassandra | grep Version | awk '{print $2}'`
+if [ "$cassandra_version" != "1.2.11" ]; then
+  # Don't do upgrade
+  exit
+fi
+nodetool upgradesstables
+service supervisor-database stop
+wget http://puppet/contrail/repo/dgautam_uj_mainline_2713/cassandra_2.0.17_all.deb
+dpkg --force-depends --force-overwrite --force-confnew --install cassandra_2.0.17_all.deb
+service cassandra stop
+chown -R cassandra:cassandra ${database_dir}
+chown -R cassandra:cassandra /var/log/cassandra
+sed -i -e "s|listen_address.*|listen_address: $this_host|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|cluster_name.*|cluster_name: \'Contrail\'|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|rpc_address.*|rpc_address: $this_host|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|# num_tokens.*|num_tokens: 256|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|initial_token.*|# initial_token:|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|saved_caches_directory.*|saved_caches_directory: $database_dir/saved_caches|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|commitlog_directory.*|commitlog_directory: $database_dir/commitlog|" /etc/cassandra/cassandra.yaml
+sed -i -e "s|    - /var/lib/cassandra/data|    - $database_dir/data|" /etc/cassandra/cassandra.yaml
+sed -i 's/JVM_OPTS=\"\$JVM_OPTS -Xss.*\"/JVM_OPTS=\"\$JVM_OPTS -Xss512k\"/g' /etc/cassandra/cassandra-env.sh
+service cassandra start; sleep 10
+cassandra_cli_status="0"
+while [ $cassandra_cli_status != "0" ] && [ $cassandra_cli_cmd != "" ]
+do
+cassandra_cli_cmd=$(cassandra-cli --host $this_host --batch < /dev/null | grep 'Connected to:')
+cassandra_cli_status=$?
+sleep 5
+done
+echo "Successful cassandra connect to 2.0.17: $cassandra_cli_cmd"
+nodetool upgradesstables
+service cassandra stop

contrail/environment/modules/contrail/manifests/collector/install.pp

@@ -1,5 +1,16 @@
 class contrail::collector::install {
-    package { ['contrail-openstack-analytics', 'contrail-docs'] :
+    exec { 'Temporarily delete contrail-analytics to upgrade python-kafka' :
+          command   => "apt-get -y --force-yes purge contrail-analytics python-kafka-python",
+          provider  => shell,
+          logoutput => $contrail_logoutput,
+          before => Package['python-kafka'],
+    }
+    package {'python-kafka':
+        ensure => latest,
+        before => Package['contrail-analytics']
+    }
+    package { ['contrail-analytics','contrail-openstack-analytics', 'contrail-docs'] :
         ensure => latest,
+        configfiles => "replace",
     }
-}
+}

contrail/environment/modules/contrail/manifests/config/install.pp

@@ -26,8 +26,25 @@
       Package['keepalived'] -> Package['contrail-openstack-config']
   }
 
+  exec { 'Temporarily delete contrail-openstack-config, contrail-config-openstack' :
+          command   => "apt-get -y --force-yes purge contrail-openstack-config contrail-config-openstack",
+          provider  => shell,
+          logoutput => $contrail_logoutput,
+          before => Package['contrail-config'],
+  }
+  package { 'contrail-config':
+    ensure => latest,
+    configfiles => "replace",
+    before => Package['contrail-openstack-config']
+  }
   package { 'contrail-openstack-config' :
-    ensure => latest
+    ensure => latest,
+    configfiles => "replace",
+    before => Package['contrail-config-openstack']
+  }
+  package { 'contrail-config-openstack' :
+    ensure => latest,
+    configfiles => "replace",
   }
 
 }

contrail/environment/modules/contrail/manifests/database/config.pp

@@ -77,15 +77,9 @@
         group   => cassandra,
     }
     ->
-
-    file { "${contrail_cassandra_dir}/cassandra.yaml" :
-        ensure  => present,
-        content => template("${module_name}/cassandra.yaml.erb"),
-    }
-    ->
-    file { "${contrail_cassandra_dir}/cassandra-env.sh" :
-        ensure  => present,
-        content => template("${module_name}/cassandra-env.sh.erb"),
+    class {'::contrail::database::config_cassandra':
+            cassandra_seeds => $cassandra_seeds,
+            contrail_cassandra_dir => $contrail_cassandra_dir
     }
 
     # Ensure kafka/config/server.properties file is present with right content.
@@ -156,7 +150,6 @@
         file { '/etc/init.d/supervisord-contrail-database':
             ensure  => link,
             target  => '/lib/init/upstart-job',
-            require => File["${contrail_cassandra_dir}/cassandra-env.sh"],
         }
         # Replaced the below script with augeas
         # File['/etc/init.d/supervisord-contrail-database'] -> File['/etc/contrail/contrail_setup_utils/config-zk-files-setup.sh']

contrail/environment/modules/contrail/manifests/database/config_cassandra.pp

@@ -0,0 +1,75 @@
+class contrail::database::config_cassandra (
+  $contrail_logoutput = $::contrail::params::contrail_logoutput,
+  $host_control_ip = $::contrail::params::host_ip,
+  $database_dir = $::contrail::params::database_dir,
+  $cassandra_seeds,
+  $contrail_cassandra_dir,
+) {
+
+      # Moved Cassandra config to augeas from templates
+      $cassandra_config_file ="${contrail_cassandra_dir}/cassandra.yaml"
+      $cassandra_env_file="${contrail_cassandra_dir}/cassandra-env.sh"
+      #$cassandra_yaml_config = { 'cassandra_config' => {
+      #        'listen_address' => $host_control_ip,
+      #        'cluster_name' => "\'Contrail\'",
+      #        'rpc_address' => $host_control_ip,
+      #        'num_tokens' => "256",
+      #        'saved_caches_directory' => "${database_dir}/saved_caches",
+      #        'commitlog_directory' => "${database_dir}/commitlog",
+      #    },
+      #}
+      package {'cassandra':
+                ensure => latest,
+                configfiles => "replace",
+      } ->
+      file_line { 'Config Cassandra listen_address':
+          path => $cassandra_config_file,
+          line => "listen_address: ${host_control_ip}",
+          match   => "^listen_address:.*$",
+      } ->
+      file_line { 'Config Cassandra rpc_address':
+          path => $cassandra_config_file,
+          line => "rpc_address: ${host_control_ip}",
+          match   => "^rpc_address:.*$",
+      } ->
+      file_line { 'Config Cassandra num_tokens':
+          path => $cassandra_config_file,
+          line => 'num_tokens: 256',
+          match   => "# num_tokens:.*$",
+      } ->
+      file_line { 'Config Cassandra cluster_name':
+          path => $cassandra_config_file,
+          line => 'cluster_name: \'Contrail\'',
+          match   => "^cluster_name:.*$",
+      } ->
+      file_line { 'Config Cassandra saved_caches_dir':
+          path => $cassandra_config_file,
+          line => "saved_caches_directory: ${database_dir}/saved_caches",
+          match   => "^saved_caches_directory:.*$",
+      } ->
+      file_line { 'Config Cassandra commitlog_dir':
+          path => $cassandra_config_file,
+          line => "commitlog_directory: ${database_dir}/commitlog",
+          match   => "^commitlog_directory:.*$",
+      } ->
+      file_line { 'Removing Cassandra initial_token':
+          path => $cassandra_config_file,
+          line => '# initial_token',
+          match   => "^initial_token:.*$",
+      } ->
+      file_line { 'Config Cassandra Seeds':
+        path => $cassandra_config_file,
+        line => "          - seeds: \"${cassandra_seeds}\"",
+        match   => "          - seeds:.*$",
+      } ->
+      file_line { 'Config File Directories':
+        path => $cassandra_config_file,
+        line => "    - ${database_dir}/data",
+        match   => "    - /var/lib/cassandra/data",
+      } ->
+      file_line { 'ENV Cassandra file setting':
+        path => $cassandra_env_file,
+        line => 'JVM_OPTS="$JVM_OPTS -Xss512k"',
+        match   => "JVM_OPTS=\"\$JVM_OPTS -Xss.*\"",
+      }
+}

contrail/environment/modules/contrail/manifests/database/install.pp

@@ -1,13 +1,32 @@
 class contrail::database::install (
   $contrail_logoutput = $::contrail::params::contrail_logoutput,
+  $host_ip = $::contrail::params::host_ip,
+  $database_dir = $::contrail::params::database_dir,
 ) {
   if ($lsbdistrelease == "14.04") {
       package { 'default-jre-headless' :
       ensure => latest,
       before => Package['contrail-openstack-database']
     }
   }
+  $cassandra_upgrade_cmd = "/bin/bash /etc/contrail/contrail_setup_utils/upgrade_cassandra.sh ${host_ip} ${database_dir}"
+  file { '/etc/contrail/contrail_setup_utils/upgrade_cassandra.sh':
+              ensure  => present,
+              mode    => '0755',
+              owner   => root,
+              group   => root,
+              source  => "puppet:///modules/${module_name}/upgrade_cassandra.sh"
+  }
+  ->
+  exec { 'Upgrade Cassandra to version 2.1 through intermediate version':
+      command => $cassandra_upgrade_cmd,
+      provider => shell,
+      logoutput => $contrail_logoutput,
+      before => Package['contrail-openstack-database'],
+  }
+  ->
+  notify { "executed contrail contrail_zk_exec_cmd : ${cassandra_upgrade_cmd}":; }
   package { 'contrail-openstack-database' :
     ensure => latest
   }
-}
+}