From ac54dc34bebdfb3f7da1aa7b5a99cbdcd91fa57c Mon Sep 17 00:00:00 2001 From: Raja Sivaramakrishnan Date: Thu, 19 May 2016 00:17:50 -0700 Subject: [PATCH] - When provisioning a compute node with VFs, use the same command as setup_interface. - Avoid bringing down the PF when the VF comes up. Otherwise, addition of the VF to a bond fails, probably due a bug in the ixgbe driver. - Modify /etc/network/interfaces to add the right commands for VFs - Enslave VFs explicitly in /etc/network/interfaces to work around flaky behavior in ixgbe diver. Change-Id: Ia2d85d0fb19f2d4b30d1f4eac8602913c7f2ea0a Partial-bug: 1573255 --- contrail_provisioning/common/base.py | 11 +++- .../common/interface_setup.py | 66 ++++++++++++++++++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/contrail_provisioning/common/base.py b/contrail_provisioning/common/base.py index 9deb3854..9958f275 100644 --- a/contrail_provisioning/common/base.py +++ b/contrail_provisioning/common/base.py @@ -198,9 +198,16 @@ def setup_sriov_vfs(self): intf_list = sriov_string.split(",") for intf_details in intf_list: info = intf_details.split(":") - str = 'sudo sed -i \'/^exit/i \\echo ' + info[1] + ' > /sys/class/net/' + info[0] + '/device/sriov_numvfs \' /etc/rc.local' + #Keep this command consistent with provision.py in fabric utils + str = 'echo %s > /sys/class/net/%s/device/sriov_numvfs; sleep 2; ifup -a' % (info[1], info[0]) + # Do nothing if the entry already present in /etc/rc.local + if sudo('grep -w \'%s\' /etc/rc.local' % str, + quiet=True).succeeded: + continue + + sed = 'sudo sed -i \'/^\s*exit/i ' + str + '\' /etc/rc.local' with settings(warn_only = True): - local(str) + local(sed) def disable_iptables(self): diff --git a/contrail_provisioning/common/interface_setup.py b/contrail_provisioning/common/interface_setup.py index 8e2877f8..232896c3 100755 --- a/contrail_provisioning/common/interface_setup.py +++ b/contrail_provisioning/common/interface_setup.py @@ -259,7 +259,18 @@ def restart_service(self): if LooseVersion(VERSION) < LooseVersion("14.04"): subprocess.call('sudo /etc/init.d/networking restart', shell=True) else: - subprocess.call('sudo ifdown -a && sudo ifup -a', shell=True) + # Avoid bringing down the PF of VF together with the VFs. + # Otherwise, the addition of the VF to a bond fails (probably due to + # a bug in the ixgbe driver) + if self.no_ip: + subprocess.call('sudo ifdown %s && ifup %s' %(self.device, \ + self.device), shell=True) + else: + output = os.popen("sudo ifquery -l --allow=auto").read() + intfs = output.split() + for intf in intfs: + subprocess.call('sudo ifdown %s && ifup -a' %(intf), \ + shell=True) time.sleep(5) def remove_lines(self, ifaces, filename): @@ -288,13 +299,15 @@ def remove_lines(self, ifaces, filename): iface_pattern = '^\s*iface ' + " |^\s*iface ".join(ifaces) + ' ' auto_pattern = '^\s*auto ' + "|^\s*auto ".join(ifaces) + allow_pattern = '^\s*allow-hotplug ' + "|^\s*allow-hotplug ".join(ifaces) # write new file with open(self.tempfile.name, 'w') as fd: fd.write('%s\n' %cfg_file[0:indices[0]]) for each in matches: each = each.strip() if re.match(auto_pattern, each) or\ - re.match(iface_pattern, each): + re.match(iface_pattern, each) or\ + re.match(allow_pattern, each): continue else: fd.write('%s\n' %each) @@ -428,6 +441,42 @@ def _get_mac_of_vf_parent(dev): return mac + @staticmethod + def _get_pf(dev): + '''Get PF of specified VF + ''' + dir_list = os.listdir('/sys/class/net/%s/device/physfn/net/' % dev) + if not dir_list: + return '' + + return dir_list[0] + + def _get_vf_index(self, dev): + '''Get index of given VF on its PF, -1 on error + ''' + pf = self._get_pf(dev) + if pf: + str = "/sys/class/net/%s/device/virtfn*" %pf + vfd = "/sys/class/net/%s/device" % dev + for file in glob.glob(str): + if (os.path.realpath(file) == os.path.realpath(vfd)): + num = re.search(r'\d+$', file) + return num.group() + return '' + + def _cfg_append_spoof_vlan(self, dev, cfg): + '''Append a line to the config to turn off spoof check Also add VLAN 0 + to the given VF as ixgbe seems to require it. + ''' + vfi = self._get_vf_index(dev) + pf = self._get_pf(dev) + if (vfi and pf): + cfg.append('post-up ip link set %s vf %s spoof off' + %(pf, vfi)) + if (self.vlan): + cfg.append('pre-up ip link set %s vf %s vlan 0' + %(pf, vfi)) + def create_interface(self): '''Create interface config for normal interface for Ubuntu''' log.info('Creating Interface: %s' % self.device) @@ -449,7 +498,13 @@ def create_interface(self): if correct_mac: cfg.append('post-up ip link set %s address %s' % (self.device, correct_mac)) - + self._cfg_append_spoof_vlan(self.device, cfg) + elif self.no_ip: + # Set PF to allow-hotplug instead of auto to distinguish it from + # interfaces which are brought up and down every time by create_interface + cfg = ['allow-hotplug %s' %self.device, + 'iface %s inet manual' %self.device, + 'down ip addr flush dev %s' %self.device] self.write_network_script(self.device, cfg) if self.vlan: self.create_vlan_interface() @@ -463,6 +518,11 @@ def create_bond_members(self): 'iface %s inet manual' %each, 'down ip addr flush dev %s' %each, 'bond-master %s' %self.device] + if self._dev_is_vf(each): + self._cfg_append_spoof_vlan(each, cfg) + # work around a bug with bonding VFs on ixgbe by repeating ifenslave + cfg.append('post-up ifenslave %s %s > /dev/null 2>&1; ifconfig %s up' + %(self.device, each, each)) self.write_network_script(each, cfg) def create_vlan_interface(self):