From 637303e2f12adcc40f5ccd28451ea5a5e403681a Mon Sep 17 00:00:00 2001 From: Jeya ganesh babu J Date: Tue, 2 Jun 2015 14:27:18 -0700 Subject: [PATCH] Storage provision fix merge Closes-Bug: #1446391 Closes-Bug: #1447707 Closes-Bug: #1456880 Closes-Bug: #1455259 Closes-Bug: #1458281 Closes-Bug: #1459485 Issues: OSDs flaps because of insufficient heartbeat timeout on large clusters Replica configured is overwritten when upgrade or setup_storage is run again. If no replica configuration is provided in the testbed Setup storage fails. Current code only checks for the release numbers, doesn't check for build numbers. Fix: Configured heartbeat based on the replica size. Added a configuration variable 'storage_replica_size' in testbed.py to specify the replica Fixed the replica parse logic. Added checks to avoid provisioning live migration on nodes that have hypervisor other than kvm. Added check for release number and build number and used storage-package instead of contrail-install-package for the versions. Added a separate command for base openstack nova live-migration. Change-Id: I06279b0d1b25ac0888fab9e6831533a58976eb68 --- fabfile/tasks/storage/install.py | 16 +++++++--- fabfile/tasks/storage/provision.py | 32 +++++++++++++++++--- fabfile/testbeds/testbed_multibox_example.py | 2 ++ fabfile/utils/migration.py | 32 +++++++++++++++++++- fabfile/utils/storage.py | 12 ++++++++ 5 files changed, 85 insertions(+), 9 deletions(-) diff --git a/fabfile/tasks/storage/install.py b/fabfile/tasks/storage/install.py index d9333b47c..d3d07c652 100644 --- a/fabfile/tasks/storage/install.py +++ b/fabfile/tasks/storage/install.py @@ -126,11 +126,19 @@ def install_storage(): @task @roles('build') def upgrade_storage(from_rel, pkg): - """upgrades all the contrail pkgs in all nodes.""" - to_rel = get_release() - if Decimal(to_rel) > Decimal(from_rel): + """upgrades contrail-storage pkgs in all nodes.""" + from_rel = get_release('contrail-storage-packages') + to_rel = sudo('dpkg --info %s |grep Version: | cut -d\':\' -f 2' + %(pkg)).split('-')[0] + from_build = get_build('contrail-storage-packages').split('~')[0] + to_build = sudo('dpkg --info %s |grep Version: | cut -d\':\' -f 2' + %(pkg)).split('-')[1].split('~')[0] + if (Decimal(to_rel) > Decimal(from_rel)) or \ + (Decimal(to_rel) == Decimal(from_rel) and \ + Decimal(to_build) > Decimal(from_build)): execute('install_storage_pkg_all', pkg) execute('install_storage') execute('setup_upgrade_storage') else: - raise RuntimeError("Upgrade not supported from release %s to %s" % (from_rel, to_rel)) + raise RuntimeError("Downgrade not supported. Current version - %s~%s, New version - %s~%s" + %(from_rel, from_build, to_rel, to_build)) diff --git a/fabfile/tasks/storage/provision.py b/fabfile/tasks/storage/provision.py index 62c7ffc15..40effadb6 100644 --- a/fabfile/tasks/storage/provision.py +++ b/fabfile/tasks/storage/provision.py @@ -123,7 +123,7 @@ def create_storage_setup_cmd(mode): cfm_ip = get_data_ip(cfm)[0] storage_master_password=get_env_passwords(env.roledefs['storage-master'][0]) # Argument details - # storage-setup-mode - setup/unconfigure/reconfigure - First time + # storage-setup-mode - setup/unconfigure/reconfigure - First time # setup/Remove all configuration/Do a reconfigure # storage-master - Storage master IP # storage-hostnames - hostnames of all the nodes (storage master + @@ -150,10 +150,11 @@ def create_storage_setup_cmd(mode): # storage-mon-hosts - storage hosts with monitors # cfg-vip -- cfg internal vip address # storage-compute-hostnames - hostnames of all storage compute nodes + # storage-replica-size - Replica size for Ceph storage # WARNING: If anything is added in the arguments, make sure it # doesn't break add_storage_node task. - cmd = "PASSWORD=%s setup-vnc-storage --storage-setup-mode %s --storage-master %s --storage-hostnames %s --storage-hosts %s --storage-host-tokens %s --storage-disk-config %s --storage-ssd-disk-config %s --storage-journal-config %s --storage-local-disk-config %s --storage-local-ssd-disk-config %s --storage-nfs-disk-config %s --storage-directory-config %s --storage-chassis-config %s --live-migration %s --collector-hosts %s --collector-host-tokens %s --cfg-host %s --cinder-vip %s --config-hosts %s --storage-os-hosts %s --storage-os-host-tokens %s --storage-mon-hosts %s --cfg-vip %s --storage-compute-hostnames %s" \ - %(storage_master_password, mode, storage_master_ip, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_ssd_disk_config()), ' '.join(get_storage_journal_config()), ' '.join(get_storage_local_disk_config()), ' '.join(get_storage_local_ssd_disk_config()), ' '.join(get_storage_nfs_disk_config()), ' '.join(get_storage_directory_config()), ' '.join(get_storage_chassis_config()), get_live_migration_opts(), ' '.join(collector_host_list), ' '.join(collector_pass_list), cfm_ip, get_cinder_ha_vip(), ' '.join(cfg_host_list), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), ' '.join(get_storage_mon_hosts()), get_cfg_ha_vip(), ' '.join(storage_compute_hostnames)) + cmd = "PASSWORD=%s setup-vnc-storage --storage-setup-mode %s --storage-master %s --storage-hostnames %s --storage-hosts %s --storage-host-tokens %s --storage-disk-config %s --storage-ssd-disk-config %s --storage-journal-config %s --storage-local-disk-config %s --storage-local-ssd-disk-config %s --storage-nfs-disk-config %s --storage-directory-config %s --storage-chassis-config %s --live-migration %s --collector-hosts %s --collector-host-tokens %s --cfg-host %s --cinder-vip %s --config-hosts %s --storage-os-hosts %s --storage-os-host-tokens %s --storage-mon-hosts %s --cfg-vip %s --storage-compute-hostnames %s --storage-replica-size %s" \ + %(storage_master_password, mode, storage_master_ip, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_ssd_disk_config()), ' '.join(get_storage_journal_config()), ' '.join(get_storage_local_disk_config()), ' '.join(get_storage_local_ssd_disk_config()), ' '.join(get_storage_nfs_disk_config()), ' '.join(get_storage_directory_config()), ' '.join(get_storage_chassis_config()), get_live_migration_opts(), ' '.join(collector_host_list), ' '.join(collector_pass_list), cfm_ip, get_cinder_ha_vip(), ' '.join(cfg_host_list), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), ' '.join(get_storage_mon_hosts()), get_cfg_ha_vip(), ' '.join(storage_compute_hostnames), get_storage_replica_size()) return cmd @@ -189,6 +190,11 @@ def setup_nfs_live_migration(mode): for entry in env.roledefs['compute']: for sthostname, sthostentry in zip(env.hostnames['all'], env.roledefs['all']): if entry == sthostentry: + #Add only for qemu-kvm hypervisor + hypervisor = get_hypervisor(entry) + if hypervisor != 'libvirt' and hypervisor != 'qemu' and \ + hypervisor != 'kvm' and hypervisor != 'qemu-kvm': + continue storage_hostnames.append(sthostname) storage_host_password=get_env_passwords(entry) storage_pass_list.append(storage_host_password) @@ -201,6 +207,16 @@ def setup_nfs_live_migration(mode): storage_master_password=get_env_passwords(env.roledefs['openstack'][0]) cfm = env.roledefs['cfgm'][0] cfm_ip = get_data_ip(cfm)[0] + + # if mode is 'setup_lm', just setup openstack nova live-migration + # configuration alone. Ignore all NFS live-migration settings from + # testbed.py. + if mode == 'setup_lm': + mode = 'setup' + no_nfs = 1 + else: + no_nfs = 0 + for entry in env.roledefs['openstack']: for sthostname, sthostentry in zip(env.hostnames['all'], env.roledefs['all']): @@ -236,7 +252,7 @@ def setup_nfs_live_migration(mode): # storage-os-hosts - storage openstack hosts (except storage-master) # storage-os-host-tokens - storage openstack hosts passwd list cmd= "PASSWORD=%s setup-vnc-livemigration --storage-setup-mode %s --storage-master %s --storage-master-token %s --storage-hostnames %s --storage-hosts %s --storage-host-tokens %s --storage-disk-config %s --storage-directory-config %s --live-migration %s --nfs-live-migration %s --storage-os-hosts %s --storage-os-host-tokens %s --fix-nova-uid %s" \ - %(storage_master_password, mode, storage_master_ip, storage_master_password, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_directory_config()), get_live_migration_opts(), get_nfs_live_migration_opts(), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), get_nova_uid_fix_opt()) + %(storage_master_password, mode, storage_master_ip, storage_master_password, ' '.join(storage_hostnames), ' '.join(storage_host_list), ' '.join(storage_pass_list), ' '.join(get_storage_disk_config()), ' '.join(get_storage_directory_config()), get_live_migration_opts(), get_nfs_live_migration_opts(no_nfs), ' '.join(storage_os_host_list), ' '.join(storage_os_pass_list), get_nova_uid_fix_opt()) print cmd sudo(cmd) #end setup_nfs_live_migration_services @@ -304,6 +320,14 @@ def setup_upgrade_storage(): execute("setup_webui_storage", "upgrade") #end setup_storage +# base openstack nova live-migration configuration +@task +@roles('build') +def setup_livemigration(): + """Provisions required contrail services in all nodes as per the role definition. + """ + execute("setup_nfs_live_migration", "setup_op_lm") +#end setup_livemigration @task @roles('build') diff --git a/fabfile/testbeds/testbed_multibox_example.py b/fabfile/testbeds/testbed_multibox_example.py index 3fb8cbed5..4cfa49816 100644 --- a/fabfile/testbeds/testbed_multibox_example.py +++ b/fabfile/testbeds/testbed_multibox_example.py @@ -199,6 +199,8 @@ # host6 : { 'disks' : ['/dev/sdc', '/dev/sdd'], 'local-disks' : ['/dev/sde'], 'local-ssd-disks' : ['/dev/sdf'] }, # host7 : { 'nfs' : ['10.10.10.10:/nfs', '11.11.11.11:/nfs']}, #} +#Set Storage replica +#storage_replica_size = 3 #Base Openstack live migration configuration. #live_migration = True diff --git a/fabfile/utils/migration.py b/fabfile/utils/migration.py index 8013daab6..2c63a333c 100644 --- a/fabfile/utils/migration.py +++ b/fabfile/utils/migration.py @@ -3,49 +3,79 @@ from fabfile.config import testbed from fabfile.utils.host import * +# reads and returns nova live-migration config from testbed.py def get_live_migration_enable(): return getattr(testbed, 'live_migration', False) +#end get_live_migration_enable +# reads and returns nfs live-migration config from testbed.py def get_ceph_nfs_migration_enable(): return getattr(testbed, 'ceph_nfs_livem', False) +#end get_ceph_nfs_migration_enable +# reads and returns external nfs live-migration config from testbed.py def get_ext_nfs_migration_enable(): return getattr(testbed, 'ext_nfs_livem', False) +#end get_ext_nfs_migration_enable +# reads and returns Ceph vm nfs server subnet from testbed.py def get_ceph_nfs_migration_subnet(): return getattr(testbed, 'ceph_nfs_livem_subnet', None) +#end get_ceph_nfs_migration_subnet +# reads and returns Ceph vm nfs server qcow2 image from testbed.py def get_ceph_nfs_migration_image(): return getattr(testbed, 'ceph_nfs_livem_image', None) +#end get_ceph_nfs_migration_image +# reads and returns Ceph vm nfs server host from testbed.py +# The host should be part of the compute nodes def get_ceph_nfs_migration_host(): entry = getattr(testbed, 'ceph_nfs_livem_host', None) for sthostname, sthostentry in zip(env.hostnames['all'], env.roledefs['all']): if entry == sthostentry: return sthostname +#end get_ceph_nfs_migration_host +# reads and returns external nfs mount point from testbed.py def get_ext_nfs_migration_mount(): return getattr(testbed, 'ext_nfs_livem_mount', None) +#end get_ext_nfs_migration_mount +# reads and returns nova uid fix config from testbed.py def get_nova_uid_fix_enabled(): return getattr(testbed, 'nova_uid_fix', None) +#end get_nova_uid_fix_enabled -def get_nfs_live_migration_opts(): +# Returns all the nfs live-migration related configuration +# for provisioning. +def get_nfs_live_migration_opts(no_nfs): nfs_live_migration_opts = "disabled" + # if no_nfs is set, then ignore all + # Ceph_nfs live-migration configuration and + # external nfs live-migration configuration + if no_nfs == 1: + return nfs_live_migration_opts if get_ext_nfs_migration_enable(): nfs_live_migration_opts = "enabled --nfs-livem-mount %s" %(get_ext_nfs_migration_mount()) elif get_ceph_nfs_migration_enable(): nfs_live_migration_opts = "enabled --nfs-livem-subnet %s --nfs-livem-image %s --nfs-livem-host %s" %(get_ceph_nfs_migration_subnet(), get_ceph_nfs_migration_image(), get_ceph_nfs_migration_host()) return nfs_live_migration_opts +#end get_nfs_live_migration_opts +# Returns nova live-migration configuration for +# provisioning def get_live_migration_opts(): live_migration_opts = "disabled" if get_live_migration_enable(): live_migration_opts = "enabled" return live_migration_opts +#end get_live_migration_opts +# Returns whether nova uid fix is required or not def get_nova_uid_fix_opt(): nova_uid_fix_opt = "disabled" if get_nova_uid_fix_enabled(): nova_uid_fix_opt = "enabled" return nova_uid_fix_opt +#end get_nova_uid_fix_opt diff --git a/fabfile/utils/storage.py b/fabfile/utils/storage.py index fe33b4513..d9e52d501 100644 --- a/fabfile/utils/storage.py +++ b/fabfile/utils/storage.py @@ -205,8 +205,20 @@ def get_storage_mon_hosts(): return (storage_mon_list) #end get_storage_mon_hosts config +# Returns interal HA vip def get_cfg_ha_vip(): ha_vip = get_from_testbed_dict('ha', 'contrail_internal_vip', None) if ha_vip: return ha_vip return 'none' +#end get_cfg_ha_vip + +# Returns replica size +def get_storage_replica_size(): + replica_size = getattr(testbed, 'storage_replica_size', None) + if replica_size != None: + if int(replica_size) < 1: + print 'Replica should be >= 1' + sys.exit(0) + return (replica_size) +#end get_storage_replica_size