Skip to content

Commit

Permalink
Fix nova/libvirt uid for live migration
Browse files Browse the repository at this point in the history 
Closes-bug: #1335344
Issue: With different nova/libvirt uid/gid, the livemigration from
    one node to another will not work with permission issues.
Fix: Change the nova/libvirt uid/gid to same across all the compute
    and openstack nodes. This requires 'nova_uid_fix = True' to be
    added to the testbed.py

Change-Id: Ic83d8fac8a639904238ffbbd1dde9d2b42aef23d
  • Loading branch information
Jeya ganesh babu J committed Apr 14, 2015
1 parent 8271f67 commit 548e625
Show file tree
Hide file tree
Showing 2 changed files with 148 additions and 2 deletions.
142 changes: 140 additions & 2 deletions contrail_provisioning/storage/compute/livemigration.py 100755 → 100644
View file
Expand Up @@ -32,8 +32,8 @@ def __init__(self, args_str = None):
LIBVIRTD_UBUNTU_BIN_CONF='/etc/default/libvirt-bin'
LIBVIRTD_TMP_BIN_CONF='/tmp/libvirtd.tmp'

for hostname, entries, entry_token in zip(self._args.storage_hostnames, self._args.storage_hosts, self._args.storage_host_tokens):
with settings(host_string = 'root@%s' %(entries), password = entry_token):
for hostname, entry, entry_token in zip(self._args.storage_hostnames, self._args.storage_hosts, self._args.storage_host_tokens):
with settings(host_string = 'root@%s' %(entry), password = entry_token):
run('openstack-config --set %s DEFAULT live_migration_flag VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE' %(NOVA_CONF))
run('openstack-config --set %s DEFAULT vncserver_listen 0.0.0.0' %(NOVA_CONF))
run('cat %s | sed s/"#listen_tls = 0"/"listen_tls = 0"/ | sed s/"#listen_tcp = 1"/"listen_tcp = 1"/ | sed s/\'#auth_tcp = "sasl"\'/\'auth_tcp = "none"\'/ > %s' %(LIBVIRTD_CONF, LIBVIRTD_TMP_CONF), shell='/bin/bash')
Expand All @@ -54,6 +54,140 @@ def __init__(self, args_str = None):
run('service nova-compute restart')
run('service libvirt-bin restart')

# Fix nova uid
if self._args.fix_nova_uid == 'enabled':
uid_fix_nodes = []
uid_fix_node_tokens = []

#Form a list of all hosts and host_tokens
for entry, entry_token in zip(self._args.storage_hosts, self._args.storage_host_tokens):
uid_fix_nodes.append(entry)
uid_fix_node_tokens.append(entry_token)
uid_fix_nodes.append(self._args.storage_master)
uid_fix_node_tokens.append(self._args.storage_master_token)
if self._args.storage_os_hosts[0] != 'none':
for entry, entry_token in zip(self._args.storage_os_hosts,
self._args.storage_os_host_tokens):
uid_fix_nodes.append(entry)
uid_fix_node_tokens.append(entry_token)

nova_id = local('sudo id -u nova', capture=True,
shell='/bin/bash')
qemu_id = local('sudo id -u libvirt-qemu', capture=True,
shell='/bin/bash')
uid_fix_required = 0

#Check if nova/libvirt uid is different in each node
for entry, entry_token in zip(uid_fix_nodes,
uid_fix_node_tokens):
with settings(host_string = 'root@%s' %(entry), password = entry_token):
nova_id_check = run('sudo id -u nova')
qemu_id_check = run('sudo id -u libvirt-qemu')
if nova_id != nova_id_check or \
qemu_id != qemu_id_check:
uid_fix_required = 1
break
if uid_fix_required == 0:
return

new_nova_uid = 500
new_nova_gid = 500

new_qemu_uid = 501
new_qemu_gid = 501

# Start from 500 and find the id that is not used in the system
while True:
recheck = 0
for entry, entry_token in zip(uid_fix_nodes,
uid_fix_node_tokens):
with settings(host_string = 'root@%s' %(entry),
password = entry_token):
id_check = run('sudo cat /etc/passwd | \
cut -d \':\' -f 3 | \
grep -w %d | wc -l'
%(new_nova_uid))
if id_check != '0':
new_nova_uid += 1
new_qemu_uid += 1
recheck = 1
id_check = run('sudo cat /etc/passwd | \
cut -d \':\' -f 3 | \
grep -w %d | wc -l'
%(new_qemu_uid))
if id_check != '0':
new_nova_uid += 1
new_qemu_uid += 1
recheck = 1
id_check = run('sudo cat /etc/group | \
cut -d \':\' -f 3 | \
grep -w %d | wc -l'
%(new_nova_gid))
if id_check != '0':
new_nova_gid += 1
new_qemu_gid += 1
recheck = 1
id_check = run('sudo cat /etc/group | \
cut -d \':\' -f 3 | \
grep -w %d | wc -l'
%(new_qemu_gid))
if id_check != '0':
new_nova_gid += 1
new_qemu_gid += 1
recheck = 1
if recheck == 1:
break
if recheck == 0:
break

# Stop nova services
# Change nova/libvirt uid and gid.
# Chown/chgrp on all the files from old uid/gid to new uid/gid
# Start nova services back
for entry, entry_token in zip(uid_fix_nodes,
uid_fix_node_tokens):
with settings(host_string = 'root@%s' %(entry), password = entry_token):
nova_services = []
services = run('ps -Af | grep nova | grep -v grep | \
awk \'{print $9}\' | cut -d \'/\' -f 4 | \
grep nova | uniq -d')
for service in services.split('\r\n'):
if service != '':
nova_services.append(service)
services = run('ps -Af | grep nova | grep -v grep | \
awk \'{print $9}\' | cut -d \'/\' -f 4 | \
grep nova | uniq -u')
for service in services.split('\r\n'):
if service != '':
nova_services.append(service)

print nova_services

for service in nova_services:
if service[0] != '':
run('service %s stop' %(service))
cur_nova_uid = run('sudo id -u nova')
cur_qemu_uid = run('sudo id -u libvirt-qemu')
cur_nova_gid = run('sudo id -g nova')
cur_qemu_gid = run('sudo id -g libvirt-qemu')
run('sudo usermod -u %d nova' %(new_nova_uid))
run('sudo groupmod -g %d nova' %(new_nova_gid))
run('sudo usermod -u %d libvirt-qemu' %(new_qemu_uid))
run('sudo groupmod -g %d kvm' %(new_qemu_gid))
run('sudo find / -uid %s -exec chown nova {} \; 2> /dev/null; echo done'
%(cur_nova_uid))
run('sudo find / -gid %s -exec chgrp nova {} \; 2> /dev/null; echo done'
%(cur_nova_gid))
run('sudo find / -uid %s -exec chown libvirt-qemu {} \; 2> /dev/null; echo done'
%(cur_qemu_uid))
run('sudo find / -gid %s -exec chgrp kvm {} \; 2> /dev/null; echo done'
%(cur_qemu_gid))
for service in nova_services:
if service[0] != '':
run('service %s start' %(service))

return

def _parse_args(self, args_str):
'''
Eg. compute-live-migration-setup --storage-master 10.157.43.171 --storage-hostnames cmbu-dt05 cmbu-ixs6-2 --storage-hosts 10.157.43.171 10.157.42.166 --storage-host-tokens n1keenA n1keenA
Expand Down Expand Up @@ -90,11 +224,15 @@ def _parse_args(self, args_str):
parser.set_defaults(**all_defaults)

parser.add_argument("--storage-master", help = "IP Address of storage master node")
parser.add_argument("--storage-master-token", help = "password of storage master node")
parser.add_argument("--storage-hostnames", help = "Host names of storage nodes", nargs='+', type=str)
parser.add_argument("--storage-hosts", help = "IP Addresses of storage nodes", nargs='+', type=str)
parser.add_argument("--storage-host-tokens", help = "Passwords of storage nodes", nargs='+', type=str)
parser.add_argument("--add-storage-node", help = "Add a new storage node")
parser.add_argument("--storage-setup-mode", help = "Storage configuration mode")
parser.add_argument("--storage-os-hosts", help = "Host names of openstack nodes other than master", nargs='+', type=str)
parser.add_argument("--storage-os-host-tokens", help = "passwords of openstack nodes other than master", nargs='+', type=str)
parser.add_argument("--fix-nova-uid", help = "Enable/disable uid fix")

self._args = parser.parse_args(remaining_argv)

Expand Down
8 changes: 8 additions & 0 deletions contrail_provisioning/storage/livemigration_setup.py
View file
Expand Up @@ -53,6 +53,9 @@ def parse_args(self, args_str):
parser.add_argument("--nfs-livem-host", help = "Image for the NFS Live migration VM", nargs="+", type=str)
parser.add_argument("--nfs-livem-mount", help = "mount point of external NFS server", nargs="+", type=str)
parser.add_argument("--storage-setup-mode", help = "Storage configuration mode")
parser.add_argument("--storage-os-hosts", help = "Host names of openstack nodes other than master", nargs='+', type=str)
parser.add_argument("--storage-os-host-tokens", help = "passwords of openstack nodes other than master", nargs='+', type=str)
parser.add_argument("--fix-nova-uid", help = "Enable/disable uid fix")

self._args = parser.parse_args(self.remaining_argv)

Expand Down Expand Up @@ -94,10 +97,15 @@ def enable_openstack_live_migration(self):
live_migration_enabled = self._args.live_migration
if live_migration_enabled == 'enabled':
livem_setup_args = " --storage-master %s" %(self._args.storage_master)
livem_setup_args = livem_setup_args + " --storage-master-token %s" %(self._args.storage_master_token)
livem_setup_args = livem_setup_args + " --storage-setup-mode %s" % (self._args.storage_setup_mode)
livem_setup_args = livem_setup_args + " --storage-hostnames %s" %(' '.join(self._args.storage_hostnames))
livem_setup_args = livem_setup_args + " --storage-hosts %s" %(' '.join(self._args.storage_hosts))
livem_setup_args = livem_setup_args + " --storage-host-tokens %s" %(' '.join(self._args.storage_host_tokens))
livem_setup_args = livem_setup_args + " --storage-os-hosts %s" %(' '.join(self._args.storage_os_hosts))
livem_setup_args = livem_setup_args + " --storage-os-host-tokens %s" %(' '.join(self._args.storage_os_host_tokens))
livem_setup_args = livem_setup_args + " --fix-nova-uid %s" %(self._args.fix_nova_uid)

with settings(host_string=self._args.storage_master, password=self._args.storage_master_token):
run("sudo compute-live-migration-setup %s" %(livem_setup_args))

Expand Down

0 comments on commit 548e625

Please sign in to comment.