Merge "Update Netronome Agilio SmartNIC Python code for Mitaka compat…

…ibility" into R3.1

src/vnsw/opencontrail-vrouter-netronome/netronome/vrouter/apps/port_control.py

@@ -102,16 +102,136 @@ def _init_Session(conf, logger):
     return sessionmaker(bind=engine)
 
 
+def _get_nova_object_data(v, namespace, name):
+    """
+    Check for an acceptable value of nova_object.namespace and
+    nova_object.name (i.e., the type name sent over from Nova). If the check
+    passes, return nova_object.data, else None.
+
+    For now, we only allow one expected type (namespace.name) at a time (unlike
+    config_editor.deserialize()).
+    """
+
+    if 'nova_object.version' not in v:
+        # Note that we currently check only check that nova_object.version is
+        # present; we don't check its value.
+        return
+
+    ns, n = v['nova_object.namespace'], v['nova_object.name']
+    if ns != namespace or n != name:
+        raise ValueError(
+            'nova_object type {}.{} is not allowed; expected type {}.{}'.
+            format(ns, n, namespace, name)
+        )
+
+    return v['nova_object.data']
+
+
+def _decode_nova_object(value, decoders, error_msg):
+    """
+    Attempt to decode `value` using each of a sequence of `decoders`.
+    Raise ValueError(error_msg) if no decoder succeeds.
+    """
+
+    for d in decoders:
+        decoded = d(value)
+        if decoded is not None:
+            return decoded
+
+    raise ValueError(error_msg)
+
+
+def _decode_image_metadata_mitaka(v):
+    try:
+        d = _get_nova_object_data(v, 'nova', 'ImageMeta')
+        if d is None:
+            return
+
+        # All of name, id, properties are optional. Let's attempt to sanity
+        # check # the data by making sure that it has a disk_format and
+        # container_format.
+        if 'disk_format' not in d or 'container_format' not in d:
+            return
+
+        if 'properties' in d:
+            props = _get_nova_object_data(
+                d['properties'], 'nova', 'ImageMetaProps'
+            )
+        else:
+            props = {}
+
+        return {
+            'name': d.get('name'),
+            'id': d.get('id'),
+            'properties': props,
+        }
+
+    except KeyError:
+        pass  # decode failed
+
+
+def _decode_image_metadata_kilo(v):
+    # All of name, id, properties are optional. Let's attempt to sanity check
+    # the data by making sure that it has a disk_format and container_format.
+    if 'disk_format' not in v or 'container_format' not in v:
+        return
+
+    return {
+        'name': v.get('name'),
+        'id': v.get('id'),
+        'properties': v.get('properties', {}),
+    }
+
+
+def _decode_image_metadata(v):
+    return _decode_nova_object(
+        value=v,
+        decoders=(
+            _decode_image_metadata_mitaka,
+            _decode_image_metadata_kilo,
+        ),
+        error_msg='unable to decode image metadata',
+    )
+
+
 def _image_metadata_to_allowed_modes(image_metadata, logger):
     if image_metadata is None:
         return
 
     logger.debug('image metadata: %s', image_metadata)
     t, v = config_editor.deserialize(image_metadata, allowed_types=('dict',))
+    decoded = _decode_image_metadata(v)
 
     return glance.allowed_hw_acceleration_modes(
-        image_properties=v.get('properties', '{}'), name=v.get('name'),
-        id=v.get('id')
+        name=decoded['name'],
+        id=decoded['id'],
+        image_properties=decoded['properties'],
+    )
+
+
+def _decode_flavor_metadata_kilo(v):
+    if 'nova_object.version' not in v:
+        return
+
+    try:
+        d = _get_nova_object_data(v, 'nova', 'Flavor')
+
+        return {
+            'name': d['name'],
+            'extra_specs': d['extra_specs'],
+        }
+
+    except KeyError:
+        pass  # decode failed
+
+
+def _decode_flavor_metadata(v):
+    return _decode_nova_object(
+        value=v,
+        decoders=(
+            _decode_flavor_metadata_kilo,
+        ),
+        error_msg='unable to decode flavor metadata',
     )
 
 
@@ -121,10 +241,12 @@ def _flavor_metadata_to_allowed_modes(flavor_metadata, logger):
 
     logger.debug('flavor information: %s', flavor_metadata)
     t, v = config_editor.deserialize(
-        flavor_metadata, allowed_types=('Flavor',)
+        flavor_metadata, allowed_types=('dict',)
     )
+    decoded = _decode_flavor_metadata(v)
+
     return flavor.allowed_hw_acceleration_modes(
-        extra_specs=v.get('_extra_specs', {}), name=v.get('_name')
+        extra_specs=decoded['extra_specs'], name=decoded['name']
     )
 
 
@@ -134,10 +256,12 @@ def _assert_flavor_supports_mode(flavor_metadata, mode):
 
     if mode == PM.VirtIO:
         t, v = config_editor.deserialize(
-            flavor_metadata, allowed_types=('Flavor',)
+            flavor_metadata, allowed_types=('dict',)
         )
+        decoded = _decode_flavor_metadata(v)
+
         flavor.assert_flavor_supports_virtio(
-            extra_specs=v.get('_extra_specs', {}), name=v.get('_name')
+            extra_specs=decoded['extra_specs'], name=decoded['name']
         )
 
 
@@ -650,6 +774,13 @@ class AddCmd(OsloConfigSubcmd):
         help='System name of TAP device for this Neutron port',
     )
 
+    MultiqueueOpt = cfg.Opt(
+        'multiqueue',
+        type=types.Boolean(),
+        default=False,
+        help=default_help('Configure TAP devices in multiqueue mode', False)
+    )
+
     PortTypeOpt_choices = ('NovaVMPort', 'NameSpacePort')
     PortTypeOpt = cfg.Opt(
         'port_type',
@@ -722,6 +853,7 @@ def HwAccelerationModeOpt():
         VmNameOpt,
         MacOpt,
         TapNameOpt,
+        MultiqueueOpt,
         PortTypeOpt,
         TxVlanIdOpt,
         RxVlanIdOpt,

src/vnsw/opencontrail-vrouter-netronome/netronome/vrouter/plug.py

@@ -23,11 +23,11 @@
 import logging
 import os
 import re
+import requests
 import six
 import stat
 import sys
 import time
-import urllib3
 import urlparse
 import zmq
 
@@ -197,16 +197,21 @@ def apply_root_dir(o, root_dir):
 
 class _CreateTAP(_Step):
     @config_section('linux_net')
-    def configure(self, dry_run=False):
+    def configure(self, dry_run=False, multiqueue=False):
         self._dry_run = dry_run
+        self._multiqueue = multiqueue
 
     @staticmethod
     def translate_conf(conf):
         ans = _Step.translate_conf(conf)
 
+        section = ans.setdefault('linux_net', {})
+
         g = getattr(conf, 'iproute2', None)
         if g is not None and g.dry_run:
-            ans.update({'linux_net': {'dry_run': True}})
+            section['dry_run'] = True
+        if getattr(conf, 'multiqueue', False):
+            section['multiqueue'] = True
 
         return ans
 
@@ -215,9 +220,13 @@ def forward_action(self, session, port, journal):
             # VRT-604 garbage collection
             return _NullAction
 
+        if self._multiqueue:
+            fmt = 'Create multiqueue TAP interface: {}'
+        else:
+            fmt = 'Create standard TAP interface: {}'
+
         return _Action(
-            lambda: self._do(session, port, journal),
-            'Create TAP interface: {}'.format(port.tap_name)
+            lambda: self._do(session, port, journal), fmt.format(port.tap_name)
         )
 
     def reverse_action(self, session, port, journal):
@@ -231,8 +240,9 @@ def reverse_action(self, session, port, journal):
         )
 
     def _do(self, session, port, journal):
+        kwds = {'multiqueue': True} if self._multiqueue else {}
         if not self._dry_run:
-            linux_net.create_tap_dev(port.tap_name)
+            linux_net.create_tap_dev(port.tap_name, **kwds)
 
         return _StepStatus.OK
 
@@ -365,7 +375,6 @@ def __init__(self, get_vif_devname=lambda p: None, vif_sync=None, **kwds):
 
         self.get_vif_devname = get_vif_devname
         self.vif_sync = vif_sync
-        self.http = urllib3.PoolManager(timeout=5, retries=1)
 
     @config_section('contrail-vrouter-agent')
     def configure(self, base_url=DEFAULT_AGENT_API_EP, vif_sync=None):
@@ -423,28 +432,28 @@ def _do(self, session, port, journal, tap_name):
         body = json.dumps(port.dump(tap_name=tap_name))
 
         try:
-            r = self.http.urlopen(
-                'POST', ep, body=body,
+            r = requests.post(
+                ep, data=body,
                 headers={'Content-Type': 'application/json'},
-                retries=None
             )
 
-        except urllib3.exceptions.HTTPError as e:
+        except requests.exceptions.RequestException as e:
             logger.error('POST error: %s', _exception_msg(e))
             return _StepStatus.ERROR
 
         try:
             log_content_type = True
             tc = vrouter_rest.HTTPSuccessTc()
-            tc.assert_valid(r.status)
+            tc.assert_valid(r.status_code)
 
             log_content_type = False
             tc = vrouter_rest.ContentTypeTc('application/json')
             tc.assert_valid(r.headers)
 
         except ValueError as e:
             _log_error_response(
-                'POST', e, r.headers, r.data, log_content_type=log_content_type
+                'POST', e, r.headers, r.content,
+                log_content_type=log_content_type
             )
 
             # vrouter-port-control also masks this error. Since it represents
@@ -461,18 +470,18 @@ def _undo(self, session, port, journal):
         ep = urlparse.urljoin(self.base_url, 'port/{}'.format(port.uuid))
 
         try:
-            r = self.http.urlopen('DELETE', ep, retries=None)
+            r = requests.delete(ep)
 
-        except urllib3.exceptions.HTTPError as e:
+        except requests.exceptions.RequestException as e:
             logger.error('DELETE error: %s', _exception_msg(e))
             return _StepStatus.ERROR
 
         try:
             tc = vrouter_rest.HTTPSuccessTc()
-            tc.assert_valid(r.status)
+            tc.assert_valid(r.status_code)
 
         except ValueError as e:
-            if port.tap_name is None and r.status == 404:
+            if port.tap_name is None and r.status_code == 404:
                 # VRT-604 garbage collection
                 pass
 
@@ -482,9 +491,9 @@ def _undo(self, session, port, journal):
                 # original vrouter-port-control script from Contrail masked all
                 # HTTP DELETE errors.
 
-                _log_error_response('DELETE', e, r.headers, r.data)
+                _log_error_response('DELETE', e, r.headers, r.content)
 
-                if r.status == 404:
+                if r.status_code == 404:
                     return _StepStatus.ERROR
                 else:
                     raise

src/vnsw/opencontrail-vrouter-netronome/netronome/vrouter/tests/helpers/plug.py

@@ -43,6 +43,16 @@
 from netronome.vrouter.tests.unit import *
 from netronome.vrouter.tests.helpers.config import _random_pci_address
 
+# urllib3 log messages can appear under various names depending on the versions
+# of the requests and/or urllib3 libraries (e.g., as they may change between
+# different releases of Ubuntu OpenStack).
+URLLIB3_LOGGERS = frozenset((
+    'urllib3.connectionpool',
+    'urllib3.util.retry',
+    'requests.packages.urllib3.connectionpool',
+    'requests.packages.urllib3.util.retry',
+))
+
 
 def _enable_fake_intel_iommu(root_dir, pt=False):
     """Fakes out netronome.iommu_check to believe that the IOMMU is on."""

src/vnsw/opencontrail-vrouter-netronome/netronome/vrouter/tests/unit/__init__.py

@@ -25,6 +25,7 @@
 
 __all__ = (
     'LogMessageCounter',
+    'SetLogLevel',
     'attachLogHandler',
     'make_vfset',
     'make_getLogger',
@@ -62,6 +63,38 @@ def __str__(self):
         return json.dumps(self.logs, indent=4)
 
 
+class SetLogLevel(object):
+    """
+    Sets the log level for a set of loggers, typically at file scope.
+
+    This is originally intended for urllib3 and/or "requests" library logging.
+    The DEBUG and INFO messages for these loggers are radically different
+    between different releases of OpenStack Kilo and OpenStack Mitaka (the
+    loggers have different names, etc). We have no interest in checking these.
+
+    We do leave warnings, errors, etc. enabled however, as we aren't expecting
+    any of these.
+    """
+
+    def __init__(self, loggers, level, **kwds):
+        super(SetLogLevel, self).__init__(**kwds)
+        self.loggers = loggers
+        self.level = level
+        self.old = {}
+
+    def setUp(self):
+        assert not self.old, 'setUp called multiple times'
+        for k in self.loggers:
+            logger = logging.getLogger(k)
+            self.old[k] = logger.level
+            logger.setLevel(self.level)
+
+    def tearDown(self):
+        old, self.old = self.old, {}
+        for k, v in old.iteritems():
+            logging.getLogger(k).setLevel(v)
+
+
 @contextlib.contextmanager
 def attachLogHandler(logger, handler=None, level=logging.DEBUG):
     if handler is None: