Storage provision fix merge

Closes-Bug: #1446391
Closes-Bug: #1447707
Closes-Bug: #1456880
Closes-Bug: #1455259
Closes-Bug: #1458281
Closes-Bug: #1459485
Issues:
    OSDs flaps because of insufficient heartbeat timeout on
        large clusters
    Replica configured is overwritten when upgrade or setup_storage
        is run again.
    If no replica configuration is provided in the testbed
        Setup storage fails.
    Current code only checks for the release numbers, doesn't check
        for build numbers.
Fix:
    Configured heartbeat based on the replica size.
    Added a configuration variable 'storage_replica_size' in testbed.py
        to specify the replica
    Fixed the replica parse logic.
    Added checks to avoid provisioning live migration on nodes
        that have hypervisor other than kvm.
    Added check for release number and build number and used
        storage-package instead of contrail-install-package for the
        versions.
    Added a separate command for base openstack nova live-migration.

Change-Id: I06279b0d1b25ac0888fab9e6831533a58976eb68

fabfile/tasks/storage/install.py

@@ -126,11 +126,19 @@ def install_storage():
 @task
 @roles('build')
 def upgrade_storage(from_rel, pkg):
-    """upgrades all the contrail pkgs in all nodes."""
-    to_rel = get_release()
-    if Decimal(to_rel) > Decimal(from_rel):
+    """upgrades contrail-storage pkgs in all nodes."""
+    from_rel = get_release('contrail-storage-packages')
+    to_rel = sudo('dpkg --info %s |grep Version: | cut -d\':\' -f 2'
+                    %(pkg)).split('-')[0]
+    from_build = get_build('contrail-storage-packages').split('~')[0]
+    to_build = sudo('dpkg --info %s |grep Version: | cut -d\':\' -f 2'
+                    %(pkg)).split('-')[1].split('~')[0]
+    if (Decimal(to_rel) > Decimal(from_rel)) or \
+        (Decimal(to_rel) == Decimal(from_rel) and \
+        Decimal(to_build) > Decimal(from_build)):
         execute('install_storage_pkg_all', pkg)
         execute('install_storage')
         execute('setup_upgrade_storage')
     else:
-        raise RuntimeError("Upgrade not supported from release %s to %s" % (from_rel, to_rel))
+        raise RuntimeError("Downgrade not supported. Current version - %s~%s, New version - %s~%s"
+            %(from_rel, from_build, to_rel, to_build))

fabfile/tasks/storage/provision.py

@@ -123,7 +123,7 @@ def create_storage_setup_cmd(mode):
     cfm_ip = get_data_ip(cfm)[0]
     storage_master_password=get_env_passwords(env.roledefs['storage-master'][0])
     # Argument details
-    # storage-setup-mode - setup/unconfigure/reconfigure - First time 
+    # storage-setup-mode - setup/unconfigure/reconfigure - First time
     #                      setup/Remove all configuration/Do a reconfigure
     # storage-master - Storage master IP
     # storage-hostnames - hostnames of all the nodes (storage master +
@@ -150,10 +150,11 @@ def create_storage_setup_cmd(mode):
     # storage-mon-hosts - storage hosts with monitors
     # cfg-vip -- cfg internal vip address
     # storage-compute-hostnames - hostnames of all storage compute nodes
+    # storage-replica-size - Replica size for Ceph storage
     # WARNING: If anything is added in the arguments, make sure it
     # doesn't break add_storage_node task.
-    cmd = "PASSWORD=%s setup-vnc-storage --storage-setup-mode %s --storage-master %s --storage-hostnames %s --storage-hosts %s --storage-host-tokens %s --storage-disk-config %s --storage-ssd-disk-config %s --storage-journal-config %s --storage-local-disk-config %s --storage-local-ssd-disk-config %s --storage-nfs-disk-config %s --storage-directory-config %s --storage-chassis-config %s --live-migration %s --collector-hosts %s --collector-host-tokens %s --cfg-host %s --cinder-vip %s --config-hosts %s --storage-os-hosts %s --storage-os-host-tokens %s --storage-mon-hosts %s --cfg-vip %s --storage-compute-hostnames %s" \
-            %(storage_master_password, mode, storage_master_ip, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_ssd_disk_config()), ' '.join(get_storage_journal_config()), ' '.join(get_storage_local_disk_config()), ' '.join(get_storage_local_ssd_disk_config()), ' '.join(get_storage_nfs_disk_config()), ' '.join(get_storage_directory_config()), ' '.join(get_storage_chassis_config()), get_live_migration_opts(), ' '.join(collector_host_list), ' '.join(collector_pass_list), cfm_ip, get_cinder_ha_vip(), ' '.join(cfg_host_list), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), ' '.join(get_storage_mon_hosts()), get_cfg_ha_vip(), ' '.join(storage_compute_hostnames))
+    cmd = "PASSWORD=%s setup-vnc-storage --storage-setup-mode %s --storage-master %s --storage-hostnames %s --storage-hosts %s --storage-host-tokens %s --storage-disk-config %s --storage-ssd-disk-config %s --storage-journal-config %s --storage-local-disk-config %s --storage-local-ssd-disk-config %s --storage-nfs-disk-config %s --storage-directory-config %s --storage-chassis-config %s --live-migration %s --collector-hosts %s --collector-host-tokens %s --cfg-host %s --cinder-vip %s --config-hosts %s --storage-os-hosts %s --storage-os-host-tokens %s --storage-mon-hosts %s --cfg-vip %s --storage-compute-hostnames %s --storage-replica-size %s" \
+            %(storage_master_password, mode, storage_master_ip, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_ssd_disk_config()), ' '.join(get_storage_journal_config()), ' '.join(get_storage_local_disk_config()), ' '.join(get_storage_local_ssd_disk_config()), ' '.join(get_storage_nfs_disk_config()), ' '.join(get_storage_directory_config()), ' '.join(get_storage_chassis_config()), get_live_migration_opts(), ' '.join(collector_host_list), ' '.join(collector_pass_list), cfm_ip, get_cinder_ha_vip(), ' '.join(cfg_host_list), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), ' '.join(get_storage_mon_hosts()), get_cfg_ha_vip(), ' '.join(storage_compute_hostnames), get_storage_replica_size())
     return cmd
 
 
@@ -189,6 +190,11 @@ def setup_nfs_live_migration(mode):
         for entry in env.roledefs['compute']:
             for sthostname, sthostentry in zip(env.hostnames['all'], env.roledefs['all']):
                 if entry == sthostentry:
+                    #Add only for qemu-kvm hypervisor
+                    hypervisor = get_hypervisor(entry)
+                    if hypervisor != 'libvirt' and hypervisor != 'qemu' and \
+                            hypervisor != 'kvm' and hypervisor != 'qemu-kvm':
+                        continue
                     storage_hostnames.append(sthostname)
                     storage_host_password=get_env_passwords(entry)
                     storage_pass_list.append(storage_host_password)
@@ -201,6 +207,16 @@ def setup_nfs_live_migration(mode):
         storage_master_password=get_env_passwords(env.roledefs['openstack'][0])
         cfm = env.roledefs['cfgm'][0]
         cfm_ip = get_data_ip(cfm)[0]
+
+        # if mode is 'setup_lm', just setup openstack nova live-migration
+        # configuration alone. Ignore all NFS live-migration settings from
+        # testbed.py.
+        if mode == 'setup_lm':
+            mode = 'setup'
+            no_nfs = 1
+        else:
+            no_nfs = 0
+
         for entry in env.roledefs['openstack']:
             for sthostname, sthostentry in zip(env.hostnames['all'],
                                                     env.roledefs['all']):
@@ -236,7 +252,7 @@ def setup_nfs_live_migration(mode):
                 # storage-os-hosts - storage openstack hosts (except storage-master)
                 # storage-os-host-tokens - storage openstack hosts passwd list
                 cmd= "PASSWORD=%s setup-vnc-livemigration --storage-setup-mode %s --storage-master %s --storage-master-token %s --storage-hostnames %s --storage-hosts %s --storage-host-tokens %s --storage-disk-config %s --storage-directory-config %s --live-migration %s --nfs-live-migration %s  --storage-os-hosts %s --storage-os-host-tokens %s --fix-nova-uid %s" \
-                    %(storage_master_password, mode, storage_master_ip, storage_master_password, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_directory_config()), get_live_migration_opts(), get_nfs_live_migration_opts(), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), get_nova_uid_fix_opt())
+                    %(storage_master_password, mode, storage_master_ip, storage_master_password, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_directory_config()), get_live_migration_opts(), get_nfs_live_migration_opts(no_nfs), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), get_nova_uid_fix_opt())
                 print cmd
                 sudo(cmd)
 #end setup_nfs_live_migration_services
@@ -304,6 +320,14 @@ def setup_upgrade_storage():
     execute("setup_webui_storage", "upgrade")
 #end setup_storage
 
+# base openstack nova live-migration configuration
+@task
+@roles('build')
+def setup_livemigration():
+    """Provisions required contrail services in all nodes as per the role definition.
+    """
+    execute("setup_nfs_live_migration", "setup_op_lm")
+#end setup_livemigration
 
 @task
 @roles('build')

fabfile/testbeds/testbed_multibox_example.py

@@ -199,6 +199,8 @@
 #    host6 : { 'disks' : ['/dev/sdc', '/dev/sdd'], 'local-disks' : ['/dev/sde'], 'local-ssd-disks' : ['/dev/sdf'] },
 #    host7 : { 'nfs' : ['10.10.10.10:/nfs', '11.11.11.11:/nfs']},
 #}
+#Set Storage replica
+#storage_replica_size = 3
 
 #Base Openstack live migration configuration.
 #live_migration = True

fabfile/utils/migration.py

@@ -3,49 +3,79 @@
 from fabfile.config import testbed
 from fabfile.utils.host import *
 
+# reads and returns nova live-migration config from testbed.py
 def get_live_migration_enable():
     return getattr(testbed, 'live_migration', False)
+#end get_live_migration_enable
 
+# reads and returns nfs live-migration config from testbed.py
 def get_ceph_nfs_migration_enable():
     return getattr(testbed, 'ceph_nfs_livem', False)
+#end get_ceph_nfs_migration_enable
 
+# reads and returns external nfs live-migration config from testbed.py
 def get_ext_nfs_migration_enable():
     return getattr(testbed, 'ext_nfs_livem', False)
+#end get_ext_nfs_migration_enable
 
+# reads and returns Ceph vm nfs server subnet from testbed.py
 def get_ceph_nfs_migration_subnet():
     return getattr(testbed, 'ceph_nfs_livem_subnet', None)
+#end get_ceph_nfs_migration_subnet
 
+# reads and returns Ceph vm nfs server qcow2 image from testbed.py
 def get_ceph_nfs_migration_image():
     return getattr(testbed, 'ceph_nfs_livem_image', None)
+#end get_ceph_nfs_migration_image
 
+# reads and returns Ceph vm nfs server host from testbed.py
+# The host should be part of the compute nodes
 def get_ceph_nfs_migration_host():
     entry = getattr(testbed, 'ceph_nfs_livem_host', None)
     for sthostname, sthostentry in zip(env.hostnames['all'], env.roledefs['all']):
         if entry == sthostentry:
             return sthostname
+#end get_ceph_nfs_migration_host
 
+# reads and returns external nfs mount point from testbed.py
 def get_ext_nfs_migration_mount():
     return getattr(testbed, 'ext_nfs_livem_mount', None)
+#end get_ext_nfs_migration_mount
 
+# reads and returns nova uid fix config from testbed.py
 def get_nova_uid_fix_enabled():
     return getattr(testbed, 'nova_uid_fix', None)
+#end get_nova_uid_fix_enabled
 
-def get_nfs_live_migration_opts():
+# Returns all the nfs live-migration related configuration
+# for provisioning.
+def get_nfs_live_migration_opts(no_nfs):
     nfs_live_migration_opts = "disabled"
+    # if no_nfs is set, then ignore all
+    # Ceph_nfs live-migration configuration and
+    # external nfs live-migration configuration
+    if no_nfs == 1:
+        return nfs_live_migration_opts
     if get_ext_nfs_migration_enable():
         nfs_live_migration_opts = "enabled --nfs-livem-mount %s" %(get_ext_nfs_migration_mount())
     elif get_ceph_nfs_migration_enable():
         nfs_live_migration_opts = "enabled --nfs-livem-subnet %s --nfs-livem-image %s --nfs-livem-host %s" %(get_ceph_nfs_migration_subnet(), get_ceph_nfs_migration_image(), get_ceph_nfs_migration_host())
     return nfs_live_migration_opts
+#end get_nfs_live_migration_opts
 
+# Returns nova live-migration configuration for
+# provisioning
 def get_live_migration_opts():
     live_migration_opts = "disabled"
     if get_live_migration_enable():
         live_migration_opts = "enabled"
     return live_migration_opts
+#end get_live_migration_opts
 
+# Returns whether nova uid fix is required or not
 def get_nova_uid_fix_opt():
     nova_uid_fix_opt = "disabled"
     if get_nova_uid_fix_enabled():
         nova_uid_fix_opt = "enabled"
     return nova_uid_fix_opt
+#end get_nova_uid_fix_opt

fabfile/utils/storage.py

@@ -205,8 +205,20 @@ def get_storage_mon_hosts():
     return (storage_mon_list)
 #end get_storage_mon_hosts config
 
+# Returns interal HA vip
 def get_cfg_ha_vip():
     ha_vip = get_from_testbed_dict('ha', 'contrail_internal_vip', None)
     if ha_vip:
         return ha_vip
     return 'none'
+#end get_cfg_ha_vip
+
+# Returns replica size
+def get_storage_replica_size():
+    replica_size = getattr(testbed, 'storage_replica_size', None)
+    if replica_size != None:
+        if int(replica_size) < 1:
+            print 'Replica should be >= 1'
+            sys.exit(0)
+    return (replica_size)
+#end get_storage_replica_size