1. python-keystoneclient latest has moved middleware to keystonemiddl…

…eware.

fixing it in unit-test and code. Also detecting failure of .venv building
is broken so invoke testr run with pipefail.

2. Also clearing pip cache entry for contrail python packages before installing
requiements in VENV.

3. Removing distribute from test-requirements, becaause distribute is deprecated
in favour of setuptools

4. fixed test failues in test_perms2 test module

Closes-Bug: 1549065
Change-Id: Iabc8231eb1b6e0051e07744621440ddb0f828164

src/analytics/contrail-broadview/test-requirements.txt

@@ -16,7 +16,6 @@ mock==1.0.1
 kafka-python==0.9.2
 kazoo
 stevedore
-distribute>=0.7.3
 ../opserver
 ../sandesh/common/dist/sandesh-common-0.1dev.tar.gz
 ../tools/sandesh/library/python/dist/sandesh-0.1dev.tar.gz

src/analytics/contrail-snmp-collector/test-requirements.txt

@@ -9,7 +9,6 @@ mock==1.0.1
 kafka-python==0.9.2
 kazoo
 stevedore
-distribute>=0.7.3
 ../opserver
 ../sandesh/common/dist/sandesh-common-0.1dev.tar.gz
 ../tools/sandesh/library/python/dist/sandesh-0.1dev.tar.gz

src/analytics/contrail-topology/test-requirements.txt

@@ -16,7 +16,6 @@ mock==1.0.1
 kafka-python==0.9.2
 kazoo
 stevedore
-distribute>=0.7.3
 ../opserver
 ../sandesh/common/dist/sandesh-common-0.1dev.tar.gz
 ../tools/sandesh/library/python/dist/sandesh-0.1dev.tar.gz

src/api-lib/run_tests.sh

@@ -148,9 +148,9 @@ function run_tests {
   then
     # subunit-2to1 is present, testr subunit stream should be in version 2
     # format. Convert to version one before colorizing.
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
   else
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
   fi
   RESULT=$?
   set -e

src/api-lib/setup.py

@@ -22,7 +22,9 @@ def run(self):
         if self.coverage:
             logfname = 'coveragetest.log'
             args += ' -c'
-        os.system('./run_tests.sh %s' % args)
+        rc_sig = os.system('./run_tests.sh %s' % args)
+        if rc_sig >> 8:
+            os._exit(rc_sig>>8)
         with open(logfname) as f:
             if not re.search('\nOK', ''.join(f.readlines())):
                 os._exit(1)

src/api-lib/test-requirements.txt

@@ -4,5 +4,4 @@ flexmock
 coverage
 pyaml
 httpretty==0.8.10
-distribute>=0.7.3
 

src/config/api-server/run_tests.sh

@@ -153,9 +153,9 @@ function run_tests {
   then
     # subunit-2to1 is present, testr subunit stream should be in version 2
     # format. Convert to version one before colorizing.
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
   else
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
   fi
   RESULT=$?
   set -e

src/config/api-server/setup.py

@@ -22,7 +22,9 @@ def run(self):
         if self.coverage:
             logfname = 'coveragetest.log'
             args += ' -c'
-        os.system('./run_tests.sh %s' % args)
+        rc_sig = os.system('./run_tests.sh %s' % args)
+        if rc_sig >> 8:
+            os._exit(rc_sig>>8)
         with open(logfname) as f:
             if not re.search('\nOK', ''.join(f.readlines())):
                 os._exit(1)

src/config/api-server/test-requirements.txt

@@ -5,4 +5,4 @@ flexmock
 coverage
 webtest
 pyaml
-distribute>=0.7.3
+keystonemiddleware

src/config/api-server/tests/test_case.py

@@ -4,8 +4,8 @@
 import test_common
 
 class ApiServerTestCase(test_common.TestCase):
-    def setUp(self, extra_mocks = None, extra_config_knobs = None):
-        super(ApiServerTestCase, self).setUp(extra_mocks, extra_config_knobs)
+    def setUp(self):
+        super(ApiServerTestCase, self).setUp()
         self.ignore_err_in_log = False
 
     def tearDown(self):

src/config/api-server/tests/test_crud_basic.py

@@ -1023,7 +1023,7 @@ def test_sandesh_trace(self):
         traces = requests.get('http://localhost:%s/Snh_SandeshTraceRequest?x=IfmapTraceBuf' %(introspect_port))
         self.assertThat(traces.status_code, Equals(200))
         top_elem = etree.fromstring(traces.text)
-        print top_elem[0][0][-1].text
+        logger.info("Top Elem: %s" % top_elem[0][0][-1].text)
         self.assertThat(top_elem[0][0][-1].text, Contains('delete'))
         self.assertThat(top_elem[0][0][-1].text, Contains(test_obj.name))
 
@@ -1933,7 +1933,7 @@ def setUpClass(cls):
         super(TestVncCfgApiServerRequests, cls).setUpClass(
             extra_config_knobs=[('DEFAULTS', 'max_requests', 10)])
 
-    def api_requests(self, orig_vn_read, count):
+    def api_requests(self, orig_vn_read, count, vn_name):
         api_server = test_common.vnc_cfg_api_server.server
         self.blocked = True
         def slow_response_on_vn_read(obj_type, *args, **kwargs):
@@ -1945,7 +1945,7 @@ def slow_response_on_vn_read(obj_type, *args, **kwargs):
         api_server._db_conn._cassandra_db.object_read = slow_response_on_vn_read
 
         logger.info("Creating a test VN object.")
-        test_obj = self._create_test_object()
+        test_obj = self.create_virtual_network(vn_name, '1.1.1.0/24')
         logger.info("Making max_requests(%s) to api server" % (count - 1))
         def vn_read():
             self._vnc_lib.virtual_network_read(id=test_obj.uuid)
@@ -1955,7 +1955,8 @@ def vn_read():
             gevent.spawn(vn_read)
         gevent.sleep(1)
 
-    def test_within_max_api_requests(self):
+    def test_max_api_requests(self):
+        # Test to make sure api-server accepts requests within max_api_requests
         self.wait_till_api_server_idle()
 
         # when there are pipe-lined requests, responses have content-length
@@ -1971,9 +1972,10 @@ def reset_response_content_length():
         api_server = test_common.vnc_cfg_api_server.server
         orig_vn_read = api_server._db_conn._cassandra_db.object_read
         try:
-            self.api_requests(orig_vn_read, 5)
+            vn_name = self.id() + '5testvn1'
+            self.api_requests(orig_vn_read, 5, vn_name)
             logger.info("Making one more requests well within the max_requests to api server")
-            vn_name = self.id() + 'testvn'
+            vn_name = self.id() + 'testvn1'
             try:
                 greenlet = gevent.spawn(self.create_virtual_network, vn_name, '10.1.1.0/24')
                 gevent.sleep(0)
@@ -1986,15 +1988,17 @@ def reset_response_content_length():
             api_server._db_conn._cassandra_db.object_read = orig_vn_read
             self.blocked = False
 
-    def test_err_on_max_api_requests(self):
+        # Test to make sure api-server rejects requests over max_api_requests
         self.wait_till_api_server_idle()
         api_server = test_common.vnc_cfg_api_server.server
         orig_vn_read = api_server._db_conn._cassandra_db.object_read
         try:
-            self.api_requests(orig_vn_read, 11)
+            vn_name = self.id() + '11testvn2'
+            self.api_requests(orig_vn_read, 11, vn_name)
             logger.info("Making one more requests (max_requests + 1) to api server")
             try:
-                greenlet = gevent.spawn(self.create_virtual_network, 'testvn', '10.1.1.0/24')
+                vn_name = self.id() + 'testvn2'
+                greenlet = gevent.spawn(self.create_virtual_network, vn_name, '10.1.1.0/24')
                 gevent.sleep(0)
                 greenlet.get(timeout=3)
             except gevent.timeout.Timeout as e:
@@ -2013,7 +2017,7 @@ class TestLocalAuth(test_case.ApiServerTestCase):
     _rbac_role = 'admin'
     @classmethod
     def setUpClass(cls):
-        from keystoneclient.middleware import auth_token
+        from keystonemiddleware import auth_token
         class FakeAuthProtocol(object):
             _test_case = cls
             def __init__(self, app, *args, **kwargs):
@@ -2362,10 +2366,10 @@ def test_set_in_object(self):
             id=vmi_obj.uuid)
         rd_ff_proto = rd_vmi_obj.virtual_machine_interface_fat_flow_protocols
         self.assertIsNone(rd_ff_proto)
-        cols = uuid_cf.get(vmi_obj.uuid,
-            column_start='propl:virtual_machine_interface_fat_flow_protocols:',
-            column_finish='propl:virtual_machine_interface_fat_flow_protocols;')
-        self.assertEqual(len(cols), 0)
+        with ExpectedException(pycassa.NotFoundException) as e:
+            cols = uuid_cf.get(vmi_obj.uuid,
+                    column_start='propl:virtual_machine_interface_fat_flow_protocols:',
+                    column_finish='propl:virtual_machine_interface_fat_flow_protocols;')
     # end test_set_in_object
 
     def test_add_del_in_object(self):
@@ -2773,10 +2777,10 @@ def test_set_in_object(self):
             id=vmi_obj.uuid)
         rd_bindings = rd_vmi_obj.virtual_machine_interface_bindings
         self.assertIsNone(rd_bindings)
-        cols = uuid_cf.get(vmi_obj.uuid,
-            column_start='propm:virtual_machine_interface_bindings:',
-            column_finish='propm:virtual_machine_interface_bindings;')
-        self.assertEqual(len(cols), 0)
+        with ExpectedException(pycassa.NotFoundException) as e:
+            cols = uuid_cf.get(vmi_obj.uuid,
+                    column_start='propm:virtual_machine_interface_bindings:',
+                    column_finish='propm:virtual_machine_interface_bindings;')
     # end test_set_in_object
 
     def test_element_add_del_in_object(self):

src/config/api-server/tests/test_logical_router.py

@@ -81,14 +81,14 @@ def test_lr_v4_subnets(self):
         vn1 = VirtualNetwork('my-vn-1', project)
         vn1.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn1, ipam_sn_v6_vn1]))
         self._vnc_lib.virtual_network_create(vn1)
-        logger.debug('Created Virtual Network object for my-vn-1 ', vn1.uuid)
+        logger.debug('Created Virtual Network object for my-vn-1: %s', vn1.uuid)
         net_obj1 = self._vnc_lib.virtual_network_read(id = vn1.uuid)
 
         # Create VN my-vn-2
         vn2 = VirtualNetwork('my-vn-2', project)
         vn2.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn2, ipam_sn_v6_vn2]))
         self._vnc_lib.virtual_network_create(vn2)
-        logger.debug('Created Virtual Network object for my-vn-2 ', vn2.uuid)
+        logger.debug('Created Virtual Network object for my-vn-2: %s', vn2.uuid)
         net_obj2 = self._vnc_lib.virtual_network_read(id = vn2.uuid)
 
         # Create Logical Router
@@ -229,14 +229,14 @@ def test_lr_v6_subnets(self):
         vn1 = VirtualNetwork('my-vn-1', project)
         vn1.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn1, ipam_sn_v6_vn1]))
         self._vnc_lib.virtual_network_create(vn1)
-        logger.debug('Created Virtual Network object for my-vn-1 ', vn1.uuid)
+        logger.debug('Created Virtual Network object for my-vn-1: %s', vn1.uuid)
         net_obj1 = self._vnc_lib.virtual_network_read(id = vn1.uuid)
 
         # Create VN my-vn-2
         vn2 = VirtualNetwork('my-vn-2', project)
         vn2.add_network_ipam(ipam, VnSubnetsType([ipam_sn_v4_vn2, ipam_sn_v6_vn2]))
         self._vnc_lib.virtual_network_create(vn2)
-        logger.debug('Created Virtual Network object for my-vn-2 ', vn2.uuid)
+        logger.debug('Created Virtual Network object for my-vn-2: %s', vn2.uuid)
         net_obj2 = self._vnc_lib.virtual_network_read(id = vn2.uuid)
 
         # Create Logical Router

src/config/api-server/tests/test_perms2.py

@@ -27,9 +27,9 @@
 import stevedore
 
 from vnc_api.vnc_api import *
-import keystoneclient.apiclient.exceptions as kc_exceptions
+import keystoneclient.exceptions as kc_exceptions
 import keystoneclient.v2_0.client as keystone
-from keystoneclient.middleware import auth_token
+from keystonemiddleware import auth_token
 from cfgm_common import rest, utils
 import cfgm_common
 
@@ -322,7 +322,8 @@ class TestPermissions(test_case.ApiServerTestCase):
     fqdn = [domain_name]
     vn_name='alice-vn'
 
-    def setUp(self):
+    @classmethod
+    def setUpClass(cls):
         extra_mocks = [(keystone.Client,
                             '__new__', test_utils.FakeKeystoneClient),
                        (vnc_api.vnc_api.VncApi,
@@ -333,9 +334,11 @@ def setUp(self):
             ('DEFAULTS', 'multi_tenancy_with_rbac', 'True'),
             ('DEFAULTS', 'auth', 'keystone'),
         ]
-        super(TestPermissions, self).setUp(extra_mocks=extra_mocks,
+        super(TestPermissions, cls).setUpClass(extra_mocks=extra_mocks,
             extra_config_knobs=extra_config_knobs)
 
+    def setUp(self):
+        super(TestPermissions, self).setUp()
         ip = self._api_server_ip
         port = self._api_server_port
         # kc = test_utils.get_keystone_client()

src/config/api-server/tests/test_perms2_target.py

@@ -5,7 +5,7 @@
 from vnc_api.vnc_api import *
 import uuid
 import argparse
-import keystoneclient.apiclient.exceptions as kc_exceptions
+import keystoneclient.exceptions as kc_exceptions
 import cfgm_common
 import pprint
 

src/config/api-server/vnc_auth_keystone.py

@@ -17,6 +17,8 @@
 
 try:
     from keystoneclient.middleware import auth_token
+except ImportError:
+    from keystonemiddleware import auth_token
 except Exception:
     pass
 
@@ -119,9 +121,9 @@ def __call__(self, env, start_response):
         if 'HTTP_X_ROLE' in env:
             roles = env['HTTP_X_ROLE'].split(',')
         if not 'admin' in [x.lower() for x in roles]:
-            resp = auth_token.MiniResp('Permission Denied', env)
-            start_response('403 Permission Denied', resp.headers)
-            return resp.body
+            start_response('403 Permission Denied',
+                [('Content-type', 'text/plain')])
+            return ['403 Permission Denied']
 
         return self.app(env, start_response)
 

src/config/common/test-requirements.txt

@@ -1,2 +1 @@
 mock
-distribute>=0.7.3

src/config/common/tests/test_utils.py

@@ -1000,7 +1000,7 @@ def __init__(self, app, conf, *args, **kwargs):
         auth_protocol = conf['auth_protocol']
         auth_host = conf['auth_host']
         auth_port = conf['auth_port']
-        self.delay_auth_decision = conf['delay_auth_decision']
+        self.delay_auth_decision = conf.get('delay_auth_decision', False)
         self.request_uri = '%s://%s:%s' % (auth_protocol, auth_host, auth_port)
         self.auth_uri = self.request_uri
         # print 'FakeAuthProtocol init: auth-uri %s, conf %s' % (self.auth_uri, self.conf)

src/config/common/tests/tools/install_venv_common.py

@@ -30,6 +30,15 @@
 import os
 import subprocess
 import sys
+import fnmatch
+import re
+
+
+def find_files(directory, file_pattern):
+    for root, dirs, files in os.walk(directory):
+        for afile in files:
+            if fnmatch.fnmatch(afile, file_pattern):
+                yield os.path.join(root, afile)
 
 
 class InstallVenv(object):
@@ -115,6 +124,33 @@ def pip_install(self, find_links, *args):
         self.run_command(cmd_array + list(args),
                           redirect_output=False)
 
+    def get_requirements(self):
+        with open(self.requirements, 'r+') as fd:
+            reqs = [req.strip() for req in fd.readlines()]
+        gteq_or_lteq_or_eqeq = re.compile('(.*)[>=<]+[>=<]+')
+        gt_or_lt_or_eq = re.compile('(.*)[>=<]+')
+        requirements = []
+        for req in reqs:
+            for regexp in [gteq_or_lteq_or_eqeq, gt_or_lt_or_eq]:
+                match = regexp.match(req)
+                if match:
+                    requirements.append(match.group(1))
+                    continue
+            else:
+                requirements.append(req)
+        return requirements
+
+    def pip_clear_cache(self, find_links):
+        contrail_reqs = []
+        for req in self.get_requirements():
+            for find_link in find_links:
+                if find_files(find_link, req):
+                    contrail_reqs.append(req)
+        for contrail_req in contrail_reqs:
+            cache_dir = os.path.expanduser('~/.cache/pip')
+            for cachefile in find_files(cache_dir, contrail_req):
+                os.remove(cachefile)
+
     def install_dependencies(self, find_links):
         print('Installing dependencies with pip (this can take a while)...')
 
@@ -124,6 +160,7 @@ def install_dependencies(self, find_links):
         self.pip_install(find_links, 'setuptools')
         self.pip_install(find_links, 'pbr')
 
+        self.pip_clear_cache(find_links)
         self.pip_install(find_links, '-r', self.requirements,
                          '-r', self.test_requirements, '--pre')
 

src/config/device-manager/run_tests.sh

@@ -154,9 +154,9 @@ function run_tests {
   then
     # subunit-2to1 is present, testr subunit stream should be in version 2
     # format. Convert to version one before colorizing.
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} subunit-2to1 | ${wrapper} ${tools_path}/tools/colorizer.py"
   else
-    bash -c "${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
+    bash -c "set -o pipefail && ${wrapper} $TESTRTESTS | ${wrapper} ${tools_path}/tools/colorizer.py"
   fi
   RESULT=$?
   set -e

src/config/device-manager/setup.py

@@ -23,7 +23,9 @@ def run(self):
         if self.coverage:
             logfname = 'coveragetest.log'
             args += ' -c'
-        os.system('./run_tests.sh %s' % args)
+        rc_sig = os.system('./run_tests.sh %s' % args)
+        if rc_sig >> 8:
+            os._exit(rc_sig>>8)
         with open(logfname) as f:
             if not re.search('\nOK', ''.join(f.readlines())):
                 os._exit(1)