Merge "1. python-keystoneclient latest has moved middleware to keysto…

…nemiddleware. fixing it in unit-test and code. Also detecting failure of .venv building is broken so invoke testr run with pipefail."

src/analytics/contrail-broadview/test-requirements.txt

@@ -16,7 +16,6 @@ mock==1.0.1
 kafka-python==0.9.2
 kazoo
 stevedore
-distribute>=0.7.3
 ../opserver
 ../sandesh/common/dist/sandesh-common-0.1dev.tar.gz
 ../tools/sandesh/library/python/dist/sandesh-0.1dev.tar.gz

src/analytics/contrail-snmp-collector/test-requirements.txt

@@ -9,7 +9,6 @@ mock==1.0.1
 kafka-python==0.9.2
 kazoo
 stevedore
-distribute>=0.7.3
 ../opserver
 ../sandesh/common/dist/sandesh-common-0.1dev.tar.gz
 ../tools/sandesh/library/python/dist/sandesh-0.1dev.tar.gz

src/analytics/contrail-topology/test-requirements.txt

@@ -16,7 +16,6 @@ mock==1.0.1
 kafka-python==0.9.2
 kazoo
 stevedore
-distribute>=0.7.3
 ../opserver
 ../sandesh/common/dist/sandesh-common-0.1dev.tar.gz
 ../tools/sandesh/library/python/dist/sandesh-0.1dev.tar.gz

src/api-lib/run_tests.sh

@@ -148,9 +148,9 @@ function run_tests {
   then
     # subunit-2to1 is present, testr subunit stream should be in version 2
     # format. Convert to version one before colorizing.
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
   else
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
   fi
   RESULT=$?
   set -e

src/api-lib/setup.py

@@ -22,7 +22,9 @@ def run(self):
         if self.coverage:
             logfname = 'coveragetest.log'
             args += ' -c'
-        os.system('./run_tests.sh %s' % args)
+        rc_sig = os.system('./run_tests.sh %s' % args)
+        if rc_sig >> 8:
+            os._exit(rc_sig>>8)
         with open(logfname) as f:
             if not re.search('\nOK', ''.join(f.readlines())):
                 os._exit(1)

src/api-lib/test-requirements.txt

@@ -4,5 +4,4 @@ flexmock
 coverage
 pyaml
 httpretty==0.8.10
-distribute>=0.7.3
 

src/config/api-server/run_tests.sh

@@ -153,9 +153,9 @@ function run_tests {
   then
     # subunit-2to1 is present, testr subunit stream should be in version 2
     # format. Convert to version one before colorizing.
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
   else
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
   fi
   RESULT=$?
   set -e

src/config/api-server/setup.py

@@ -22,7 +22,9 @@ def run(self):
         if self.coverage:
             logfname = 'coveragetest.log'
             args += ' -c'
-        os.system('./run_tests.sh %s' % args)
+        rc_sig = os.system('./run_tests.sh %s' % args)
+        if rc_sig >> 8:
+            os._exit(rc_sig>>8)
         with open(logfname) as f:
             if not re.search('\nOK', ''.join(f.readlines())):
                 os._exit(1)

src/config/api-server/test-requirements.txt

@@ -5,4 +5,4 @@ flexmock
 coverage
 webtest
 pyaml
-distribute>=0.7.3
+keystonemiddleware

src/config/api-server/tests/test_case.py

@@ -4,8 +4,8 @@
 import test_common
 
 class ApiServerTestCase(test_common.TestCase):
-    def setUp(self, extra_mocks = None, extra_config_knobs = None):
-        super(ApiServerTestCase, self).setUp(extra_mocks, extra_config_knobs)
+    def setUp(self):
+        super(ApiServerTestCase, self).setUp()
         self.ignore_err_in_log = False
 
     def tearDown(self):

src/config/api-server/tests/test_crud_basic.py

@@ -1023,7 +1023,7 @@ def test_sandesh_trace(self):
         traces = requests.get('http://localhost:%s/Snh_SandeshTraceRequest?x=IfmapTraceBuf' %(introspect_port))
         self.assertThat(traces.status_code, Equals(200))
         top_elem = etree.fromstring(traces.text)
-        print top_elem[0][0][-1].text
+        logger.info("Top Elem: %s" % top_elem[0][0][-1].text)
         self.assertThat(top_elem[0][0][-1].text, Contains('delete'))
         self.assertThat(top_elem[0][0][-1].text, Contains(test_obj.name))
 
@@ -1933,7 +1933,7 @@ def setUpClass(cls):
         super(TestVncCfgApiServerRequests, cls).setUpClass(
             extra_config_knobs=[('DEFAULTS', 'max_requests', 10)])
 
-    def api_requests(self, orig_vn_read, count):
+    def api_requests(self, orig_vn_read, count, vn_name):
         api_server = test_common.vnc_cfg_api_server.server
         self.blocked = True
         def slow_response_on_vn_read(obj_type, *args, **kwargs):
@@ -1945,7 +1945,7 @@ def slow_response_on_vn_read(obj_type, *args, **kwargs):
         api_server._db_conn._cassandra_db.object_read = slow_response_on_vn_read
 
         logger.info("Creating a test VN object.")
-        test_obj = self._create_test_object()
+        test_obj = self.create_virtual_network(vn_name, '1.1.1.0/24')
         logger.info("Making max_requests(%s) to api server" % (count - 1))
         def vn_read():
             self._vnc_lib.virtual_network_read(id=test_obj.uuid)
@@ -1955,7 +1955,8 @@ def vn_read():
             gevent.spawn(vn_read)
         gevent.sleep(1)
 
-    def test_within_max_api_requests(self):
+    def test_max_api_requests(self):
+        # Test to make sure api-server accepts requests within max_api_requests
         self.wait_till_api_server_idle()
 
         # when there are pipe-lined requests, responses have content-length
@@ -1971,9 +1972,10 @@ def reset_response_content_length():
         api_server = test_common.vnc_cfg_api_server.server
         orig_vn_read = api_server._db_conn._cassandra_db.object_read
         try:
-            self.api_requests(orig_vn_read, 5)
+            vn_name = self.id() + '5testvn1'
+            self.api_requests(orig_vn_read, 5, vn_name)
             logger.info("Making one more requests well within the max_requests to api server")
-            vn_name = self.id() + 'testvn'
+            vn_name = self.id() + 'testvn1'
             try:
                 greenlet = gevent.spawn(self.create_virtual_network, vn_name, '10.1.1.0/24')
                 gevent.sleep(0)
@@ -1986,15 +1988,17 @@ def reset_response_content_length():
             api_server._db_conn._cassandra_db.object_read = orig_vn_read
             self.blocked = False
 
-    def test_err_on_max_api_requests(self):
+        # Test to make sure api-server rejects requests over max_api_requests
         self.wait_till_api_server_idle()
         api_server = test_common.vnc_cfg_api_server.server
         orig_vn_read = api_server._db_conn._cassandra_db.object_read
         try:
-            self.api_requests(orig_vn_read, 11)
+            vn_name = self.id() + '11testvn2'
+            self.api_requests(orig_vn_read, 11, vn_name)
             logger.info("Making one more requests (max_requests + 1) to api server")
             try:
-                greenlet = gevent.spawn(self.create_virtual_network, 'testvn', '10.1.1.0/24')
+                vn_name = self.id() + 'testvn2'
+                greenlet = gevent.spawn(self.create_virtual_network, vn_name, '10.1.1.0/24')
                 gevent.sleep(0)
                 greenlet.get(timeout=3)
             except gevent.timeout.Timeout as e:
@@ -2013,7 +2017,7 @@ class TestLocalAuth(test_case.ApiServerTestCase):
     _rbac_role = 'admin'
     @classmethod
     def setUpClass(cls):
-        from keystoneclient.middleware import auth_token
+        from keystonemiddleware import auth_token
         class FakeAuthProtocol(object):
             _test_case = cls
             def __init__(self, app, *args, **kwargs):
@@ -2362,10 +2366,10 @@ def test_set_in_object(self):
             id=vmi_obj.uuid)
         rd_ff_proto = rd_vmi_obj.virtual_machine_interface_fat_flow_protocols
         self.assertIsNone(rd_ff_proto)
-        cols = uuid_cf.get(vmi_obj.uuid,
-            column_start='propl:virtual_machine_interface_fat_flow_protocols:',
-            column_finish='propl:virtual_machine_interface_fat_flow_protocols;')
-        self.assertEqual(len(cols), 0)
+        with ExpectedException(pycassa.NotFoundException) as e:
+            cols = uuid_cf.get(vmi_obj.uuid,
+                    column_start='propl:virtual_machine_interface_fat_flow_protocols:',
+                    column_finish='propl:virtual_machine_interface_fat_flow_protocols;')
     # end test_set_in_object
 
     def test_add_del_in_object(self):
@@ -2773,10 +2777,10 @@ def test_set_in_object(self):
             id=vmi_obj.uuid)
         rd_bindings = rd_vmi_obj.virtual_machine_interface_bindings
         self.assertIsNone(rd_bindings)
-        cols = uuid_cf.get(vmi_obj.uuid,
-            column_start='propm:virtual_machine_interface_bindings:',
-            column_finish='propm:virtual_machine_interface_bindings;')
-        self.assertEqual(len(cols), 0)
+        with ExpectedException(pycassa.NotFoundException) as e:
+            cols = uuid_cf.get(vmi_obj.uuid,
+                    column_start='propm:virtual_machine_interface_bindings:',
+                    column_finish='propm:virtual_machine_interface_bindings;')
     # end test_set_in_object
 
     def test_element_add_del_in_object(self):

src/config/api-server/tests/test_logical_router.py

@@ -81,14 +81,14 @@ def test_lr_v4_subnets(self):
         vn1 = VirtualNetwork('my-vn-1', project)
         vn1.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn1, ipam_sn_v6_vn1]))
         self._vnc_lib.virtual_network_create(vn1)
-        logger.debug('Created Virtual Network object for my-vn-1 ', vn1.uuid)
+        logger.debug('Created Virtual Network object for my-vn-1: %s', vn1.uuid)
         net_obj1 = self._vnc_lib.virtual_network_read(id = vn1.uuid)
 
         # Create VN my-vn-2
         vn2 = VirtualNetwork('my-vn-2', project)
         vn2.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn2, ipam_sn_v6_vn2]))
         self._vnc_lib.virtual_network_create(vn2)
-        logger.debug('Created Virtual Network object for my-vn-2 ', vn2.uuid)
+        logger.debug('Created Virtual Network object for my-vn-2: %s', vn2.uuid)
         net_obj2 = self._vnc_lib.virtual_network_read(id = vn2.uuid)
 
         # Create Logical Router
@@ -229,14 +229,14 @@ def test_lr_v6_subnets(self):
         vn1 = VirtualNetwork('my-vn-1', project)
         vn1.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn1, ipam_sn_v6_vn1]))
         self._vnc_lib.virtual_network_create(vn1)
-        logger.debug('Created Virtual Network object for my-vn-1 ', vn1.uuid)
+        logger.debug('Created Virtual Network object for my-vn-1: %s', vn1.uuid)
         net_obj1 = self._vnc_lib.virtual_network_read(id = vn1.uuid)
 
         # Create VN my-vn-2
         vn2 = VirtualNetwork('my-vn-2', project)
         vn2.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn2, ipam_sn_v6_vn2]))
         self._vnc_lib.virtual_network_create(vn2)
-        logger.debug('Created Virtual Network object for my-vn-2 ', vn2.uuid)
+        logger.debug('Created Virtual Network object for my-vn-2: %s', vn2.uuid)
         net_obj2 = self._vnc_lib.virtual_network_read(id = vn2.uuid)
 
         # Create Logical Router

src/config/api-server/tests/test_perms2.py

@@ -27,9 +27,9 @@
 import stevedore
 
 from vnc_api.vnc_api import *
-import keystoneclient.apiclient.exceptions as kc_exceptions
+import keystoneclient.exceptions as kc_exceptions
 import keystoneclient.v2_0.client as keystone
-from keystoneclient.middleware import auth_token
+from keystonemiddleware import auth_token
 from cfgm_common import rest, utils
 import cfgm_common
 
@@ -322,7 +322,8 @@ class TestPermissions(test_case.ApiServerTestCase):
     fqdn = [domain_name]
     vn_name='alice-vn'
 
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         extra_mocks = [(keystone.Client,
                             '__new__', test_utils.FakeKeystoneClient),
                        (vnc_api.vnc_api.VncApi,
@@ -333,9 +334,11 @@ def setUp(self):
             ('DEFAULTS', 'multi_tenancy_with_rbac', 'True'),
             ('DEFAULTS', 'auth', 'keystone'),
         ]
-        super(TestPermissions, self).setUp(extra_mocks=extra_mocks,
+        super(TestPermissions, cls).setUpClass(extra_mocks=extra_mocks,
             extra_config_knobs=extra_config_knobs)
 
+    def setUp(self):
+        super(TestPermissions, self).setUp()
         ip = self._api_server_ip
         port = self._api_server_port
         # kc = test_utils.get_keystone_client()

src/config/api-server/tests/test_perms2_target.py

@@ -5,7 +5,7 @@
 from vnc_api.vnc_api import *
 import uuid
 import argparse
-import keystoneclient.apiclient.exceptions as kc_exceptions
+import keystoneclient.exceptions as kc_exceptions
 import cfgm_common
 import pprint
 

src/config/api-server/vnc_auth_keystone.py

@@ -17,6 +17,8 @@
 
 try:
     from keystoneclient.middleware import auth_token
+except ImportError:
+    from keystonemiddleware import auth_token
 except Exception:
     pass
 
@@ -119,9 +121,9 @@ def __call__(self, env, start_response):
         if 'HTTP_X_ROLE' in env:
             roles = env['HTTP_X_ROLE'].split(',')
         if not 'admin' in [x.lower() for x in roles]:
-            resp = auth_token.MiniResp('Permission Denied', env)
-            start_response('403 Permission Denied', resp.headers)
-            return resp.body
+            start_response('403 Permission Denied',
+                [('Content-type', 'text/plain')])
+            return ['403 Permission Denied']
 
         return self.app(env, start_response)
 

src/config/common/test-requirements.txt

@@ -1,2 +1 @@
 mock
-distribute>=0.7.3

src/config/common/tests/test_utils.py

@@ -1000,7 +1000,7 @@ def __init__(self, app, conf, *args, **kwargs):
         auth_protocol = conf['auth_protocol']
         auth_host = conf['auth_host']
         auth_port = conf['auth_port']
-        self.delay_auth_decision = conf['delay_auth_decision']
+        self.delay_auth_decision = conf.get('delay_auth_decision', False)
         self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, auth_port)
         self.auth_uri = self.request_uri
         # print 'FakeAuthProtocol init: auth-uri %s, conf %s' % (self.auth_uri, self.conf)

src/config/common/tests/tools/install_venv_common.py

@@ -30,6 +30,15 @@
 import os
 import subprocess
 import sys
+import fnmatch
+import re
+
+
+def find_files(directory, file_pattern):
+    for root, dirs, files in os.walk(directory):
+        for afile in files:
+            if fnmatch.fnmatch(afile, file_pattern):
+                yield os.path.join(root, afile)
 
 
 class InstallVenv(object):
@@ -115,6 +124,33 @@ def pip_install(self, find_links, *args):
         self.run_command(cmd_array + list(args),
                           redirect_output=False)
 
+    def get_requirements(self):
+        with open(self.requirements, 'r+') as fd:
+            reqs = [req.strip() for req in fd.readlines()]
+        gteq_or_lteq_or_eqeq = re.compile('(.*)[>=<]+[>=<]+')
+        gt_or_lt_or_eq = re.compile('(.*)[>=<]+')
+        requirements = []
+        for req in reqs:
+            for regexp in [gteq_or_lteq_or_eqeq, gt_or_lt_or_eq]:
+                match = regexp.match(req)
+                if match:
+                    requirements.append(match.group(1))
+                    continue
+            else:
+                requirements.append(req)
+        return requirements
+
+    def pip_clear_cache(self, find_links):
+        contrail_reqs = []
+        for req in self.get_requirements():
+            for find_link in find_links:
+                if find_files(find_link, req):
+                    contrail_reqs.append(req)
+        for contrail_req in contrail_reqs:
+            cache_dir = os.path.expanduser('~/.cache/pip')
+            for cachefile in find_files(cache_dir, contrail_req):
+                os.remove(cachefile)
+
     def install_dependencies(self, find_links):
         print('Installing dependencies with pip (this can take a while)...')
 
@@ -124,6 +160,7 @@ def install_dependencies(self, find_links):
         self.pip_install(find_links, 'setuptools')
         self.pip_install(find_links, 'pbr')
 
+        self.pip_clear_cache(find_links)
         self.pip_install(find_links, '-r', self.requirements,
                          '-r', self.test_requirements, '--pre')
 

src/config/device-manager/run_tests.sh

@@ -154,9 +154,9 @@ function run_tests {
   then
     # subunit-2to1 is present, testr subunit stream should be in version 2
     # format. Convert to version one before colorizing.
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
   else
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
   fi
   RESULT=$?
   set -e

src/config/device-manager/setup.py

@@ -23,7 +23,9 @@ def run(self):
         if self.coverage:
             logfname = 'coveragetest.log'
             args += ' -c'
-        os.system('./run_tests.sh %s' % args)
+        rc_sig = os.system('./run_tests.sh %s' % args)
+        if rc_sig >> 8:
+            os._exit(rc_sig>>8)
         with open(logfname) as f:
             if not re.search('\nOK', ''.join(f.readlines())):
                 os._exit(1)