From 89549526db727d59c8f8a5e837217c62c2e16cc0 Mon Sep 17 00:00:00 2001 From: Raja Sivaramakrishnan Date: Thu, 23 Jun 2016 14:56:22 -0700 Subject: [PATCH] Merge below 3.0 changes into master. - When provisioning a compute node with VFs, use the same command as setup_interface. - Avoid bringing down the PF when the VF comes up. Otherwise, addition of the VF to a bond fails, probably due a bug in the ixgbe driver. - Modify /etc/network/interfaces to add the right commands for VFs - Enslave VFs explicitly in /etc/network/interfaces to work around flaky behavior in ixgbe diver. - Prevent prompt for user/password when running fab setup_all - vlan interface creation fails if the pre-up does "ifconfig up" of the vlan interface that is being brought up. This is not an issue for physical interfaces, but fails if the VLAN is created on top of SR-IOV VFs. post-up should be used instead of pre-up as the interface would have been created by then. - Bringing up interfaces one at a time intstead of all. Change-Id: I55a583e8e89a31c3b313ad223bae2976e3f344f3 Closes-Bug: 1573255 1585876 --- contrail_provisioning/common/base.py | 11 ++- .../common/interface_setup.py | 79 ++++++++++++++++++- contrail_provisioning/compute/network.py | 13 ++- 3 files changed, 95 insertions(+), 8 deletions(-) diff --git a/contrail_provisioning/common/base.py b/contrail_provisioning/common/base.py index 9deb3854..389e126f 100644 --- a/contrail_provisioning/common/base.py +++ b/contrail_provisioning/common/base.py @@ -198,9 +198,16 @@ def setup_sriov_vfs(self): intf_list = sriov_string.split(",") for intf_details in intf_list: info = intf_details.split(":") - str = 'sudo sed -i \'/^exit/i \\echo ' + info[1] + ' > /sys/class/net/' + info[0] + '/device/sriov_numvfs \' /etc/rc.local' + # Keep this command consistent with provision.py in fabric utils + str = 'echo %s > /sys/class/net/%s/device/sriov_numvfs; sleep 2; ifup -a' % (info[1], info[0]) + # Do nothing if the entry already present in /etc/rc.local with settings(warn_only = True): - local(str) + if local('grep -w \'%s\' /etc/rc.local' % str).succeeded: + continue + + sed = 'sudo sed -i \'/^\s*exit/i ' + str + '\' /etc/rc.local' + with settings(warn_only = True): + local(sed) def disable_iptables(self): diff --git a/contrail_provisioning/common/interface_setup.py b/contrail_provisioning/common/interface_setup.py index 8e2877f8..9bd25f8c 100755 --- a/contrail_provisioning/common/interface_setup.py +++ b/contrail_provisioning/common/interface_setup.py @@ -22,6 +22,10 @@ from netaddr import IPNetwork from tempfile import NamedTemporaryFile from distutils.version import LooseVersion +try: + import lsb_release +except ImportError: + pass logging.basicConfig(format='%(asctime)-15s:: %(funcName)s:%(levelname)s::\ %(message)s', @@ -259,7 +263,27 @@ def restart_service(self): if LooseVersion(VERSION) < LooseVersion("14.04"): subprocess.call('sudo /etc/init.d/networking restart', shell=True) else: - subprocess.call('sudo ifdown -a && sudo ifup -a', shell=True) + # Avoid bringing down the PF of VF together with the VFs. + # Otherwise, the addition of the VF to a bond fails (probably due to + # a bug in the ixgbe driver) + if self.no_ip: + subprocess.call('sudo ifdown %s && ifup %s' % (self.device, + self.device), shell=True) + else: + output = os.popen("sudo ifquery -l --allow=auto").read() + intfs = output.split() + lsb_info = lsb_release.get_lsb_information() + lsb_version = lsb_info['DESCRIPTION'].split()[1] + if LooseVersion(lsb_version) >= LooseVersion('14.04.4'): + for intf in intfs: + subprocess.call('sudo ifdown %s && ifup %s' % + (intf, intf), shell=True) + subprocess.call('sudo ifup -a', shell=True) + + else: + for intf in intfs: + subprocess.call('sudo ifdown %s && ifup -a' % (intf), + shell=True) time.sleep(5) def remove_lines(self, ifaces, filename): @@ -288,13 +312,15 @@ def remove_lines(self, ifaces, filename): iface_pattern = '^\s*iface ' + " |^\s*iface ".join(ifaces) + ' ' auto_pattern = '^\s*auto ' + "|^\s*auto ".join(ifaces) + allow_pattern = '^\s*allow-hotplug ' + "|^\s*allow-hotplug ".join(ifaces) # write new file with open(self.tempfile.name, 'w') as fd: fd.write('%s\n' %cfg_file[0:indices[0]]) for each in matches: each = each.strip() if re.match(auto_pattern, each) or\ - re.match(iface_pattern, each): + re.match(iface_pattern, each) or\ + re.match(allow_pattern, each): continue else: fd.write('%s\n' %each) @@ -428,6 +454,42 @@ def _get_mac_of_vf_parent(dev): return mac + @staticmethod + def _get_pf(dev): + '''Get PF of specified VF + ''' + dir_list = os.listdir('/sys/class/net/%s/device/physfn/net/' % dev) + if not dir_list: + return '' + + return dir_list[0] + + def _get_vf_index(self, dev): + '''Get index of given VF on its PF, -1 on error + ''' + pf = self._get_pf(dev) + if pf: + str = "/sys/class/net/%s/device/virtfn*" %pf + vfd = "/sys/class/net/%s/device" % dev + for file in glob.glob(str): + if (os.path.realpath(file) == os.path.realpath(vfd)): + num = re.search(r'\d+$', file) + return num.group() + return '' + + def _cfg_append_spoof_vlan(self, dev, cfg): + '''Append a line to the config to turn off spoof check Also add VLAN 0 + to the given VF as ixgbe seems to require it. + ''' + vfi = self._get_vf_index(dev) + pf = self._get_pf(dev) + if (vfi and pf): + cfg.append('post-up ip link set %s vf %s spoof off' + %(pf, vfi)) + if (self.vlan): + cfg.append('pre-up ip link set %s vf %s vlan 0' + %(pf, vfi)) + def create_interface(self): '''Create interface config for normal interface for Ubuntu''' log.info('Creating Interface: %s' % self.device) @@ -449,7 +511,13 @@ def create_interface(self): if correct_mac: cfg.append('post-up ip link set %s address %s' % (self.device, correct_mac)) - + self._cfg_append_spoof_vlan(self.device, cfg) + elif self.no_ip: + # Set PF to allow-hotplug instead of auto to distinguish it from + # interfaces which are brought up and down every time by create_interface + cfg = ['allow-hotplug %s' %self.device, + 'iface %s inet manual' %self.device, + 'down ip addr flush dev %s' %self.device] self.write_network_script(self.device, cfg) if self.vlan: self.create_vlan_interface() @@ -463,6 +531,11 @@ def create_bond_members(self): 'iface %s inet manual' %each, 'down ip addr flush dev %s' %each, 'bond-master %s' %self.device] + if self._dev_is_vf(each): + self._cfg_append_spoof_vlan(each, cfg) + # work around a bug with bonding VFs on ixgbe by repeating ifenslave + cfg.append('post-up ifenslave %s %s > /dev/null 2>&1; ifconfig %s up' + %(self.device, each, each)) self.write_network_script(each, cfg) def create_vlan_interface(self): diff --git a/contrail_provisioning/compute/network.py b/contrail_provisioning/compute/network.py index 551ba47e..692d393d 100644 --- a/contrail_provisioning/compute/network.py +++ b/contrail_provisioning/compute/network.py @@ -311,8 +311,12 @@ def _rewrite_net_interfaces_file(self, dev, mac, vhost_ip, netmask, gateway_ip, # add manual entry for dev local("echo 'auto %s' >> %s" %(dev, temp_intf_file)) local("echo 'iface %s inet manual' >> %s" %(dev, temp_intf_file)) - local("echo ' pre-up ifconfig %s up' >> %s" %(dev, temp_intf_file)) - local("echo ' post-down ifconfig %s down' >> %s" %(dev, temp_intf_file)) + if vlan: + local("echo ' post-up ifconfig %s up' >> %s" %(dev, temp_intf_file)) + local("echo ' pre-down ifconfig %s down' >> %s" %(dev, temp_intf_file)) + else: + local("echo ' pre-up ifconfig %s up' >> %s" %(dev, temp_intf_file)) + local("echo ' post-down ifconfig %s down' >> %s" %(dev, temp_intf_file)) if (esxi_vm): local("echo ' pre-up ifconfig %s up mtu %s' >> %s" % (dev, datapg_mtu, temp_intf_file)) device_driver = local("ethtool -i %s | grep driver | cut -f 2 -d ' '" %dev, capture=True) @@ -346,7 +350,10 @@ def _rewrite_net_interfaces_file(self, dev, mac, vhost_ip, netmask, gateway_ip, if (device_driver == "vmxnet3"): local("echo ' pre-up ethtool --offload %s rx off' >> %s" %(dev, temp_intf_file)) local("echo ' pre-up ethtool --offload %s tx off' >> %s" %(dev, temp_intf_file)) - local("sed -i '/auto %s/ a\iface %s inet manual\\n pre-up ifconfig %s up\\n post-down ifconfig %s down\' %s"% (dev, dev, dev, dev, temp_intf_file)) + if vlan: + local("sed -i '/auto %s/ a\iface %s inet manual\\n post-up ifconfig %s up\\n pre-down ifconfig %s down\' %s"% (dev, dev, dev, dev, temp_intf_file)) + else: + local("sed -i '/auto %s/ a\iface %s inet manual\\n pre-up ifconfig %s up\\n post-down ifconfig %s down\' %s"% (dev, dev, dev, dev, temp_intf_file)) if esxi_vm and vmpg_mtu: intf = self.get_secondary_device(self.dev) mac_addr = self.get_if_mac(intf)